diff --git a/.github/workflows/dokken-system-tests.yml b/.github/workflows/dokken-system-tests.yml index 32731168a1..0ab49662db 100644 --- a/.github/workflows/dokken-system-tests.yml +++ b/.github/workflows/dokken-system-tests.yml @@ -35,6 +35,8 @@ jobs: - ubuntu2204 - rhel8 - rocky8 + - rhel9 + - rocky9 fail-fast: false steps: - uses: actions/checkout@main diff --git a/CHANGELOG.md b/CHANGELOG.md index 048b20c54a..0a075da8d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ This file is used to list changes made in each version of the AWS ParallelCluste ------ **ENHANCEMENTS** +- Add support for RHEL9. +- Add support for Rocky Linux 9 as `CustomAmi` created through `build-image` process. No public official ParallelCluster Rocky9 Linux AMI is made available at this time. - Add the configuration parameter `DeploymentSettings/DefaultUserHome` to allow users to move the default user's home directory to `/local/home` instead of `/home` (default). - Add possibility to choose between Open and Closed Source Nvidia Drivers when building an AMI, through the ```['cluster']['nvidia']['kernel_open']``` cookbook node attribute. diff --git a/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/ec2_dev_2_volid.py b/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/ec2_dev_2_volid.py index e8fe19a64e..7a1ff437b1 100644 --- a/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/ec2_dev_2_volid.py +++ b/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/ec2_dev_2_volid.py @@ -1,31 +1,11 @@ # FIXME: Fix Code Duplication # pylint: disable=R0801 -import configparser +import json import os import re import sys import syslog -import time - -import boto3 -import requests -from botocore.config import Config - -METADATA_REQUEST_TIMEOUT = 60 - - -def get_imdsv2_token(): - # Try with getting IMDSv2 token, fall back to IMDSv1 if can not get the token - token = requests.put( - "http://169.254.169.254/latest/api/token", - headers={"X-aws-ec2-metadata-token-ttl-seconds": "300"}, - timeout=METADATA_REQUEST_TIMEOUT, - ) - headers = {} - if token.status_code == requests.codes.ok: - headers["X-aws-ec2-metadata-token"] = token.content - return headers def validate_device_name(device_name): @@ -66,47 +46,6 @@ def adapt_device_name(dev): return dev -def parse_proxy_config(): - config = configparser.RawConfigParser() - config.read("/etc/boto.cfg") - proxy_config = Config() - if config.has_option("Boto", "proxy") and config.has_option("Boto", "proxy_port"): - proxy = config.get("Boto", "proxy") - proxy_port = config.get("Boto", "proxy_port") - proxy_config = Config(proxies={"https": f"{proxy}:{proxy_port}"}) - return proxy_config - - -def get_device_volume_id(ec2, dev, instance_id): - # Poll for blockdevicemapping - devices = ec2.describe_instance_attribute(InstanceId=instance_id, Attribute="blockDeviceMapping").get( - "BlockDeviceMappings" - ) - dev_map = dict((d.get("DeviceName"), d) for d in devices) - loop_count = 0 - while dev not in dev_map: - if loop_count == 36: - syslog.syslog(f"Dev {dev} did not appears in 180 seconds.") - sys.exit(1) - syslog.syslog(f"Looking for dev {dev} in dev_map {dev_map}") - time.sleep(5) - devices = ec2.describe_instance_attribute(InstanceId=instance_id, Attribute="blockDeviceMapping").get( - "BlockDeviceMappings" - ) - dev_map = dict((d.get("DeviceName"), d) for d in devices) - loop_count += 1 - - return dev_map.get(dev).get("Ebs").get("VolumeId") - - -def get_metadata_value(token, metadata_path): - return requests.get( - metadata_path, - headers=token, - timeout=METADATA_REQUEST_TIMEOUT, - ).text - - def main(): syslog.syslog("Starting ec2_dev_2_volid.py script") try: @@ -115,29 +54,14 @@ def main(): syslog.syslog(f"Input block device is {dev}") except IndexError: syslog.syslog(syslog.LOG_ERR, "Provide block device i.e. xvdf") - dev = adapt_device_name(dev) - - token = get_imdsv2_token() - - instance_id = get_metadata_value(token, "http://169.254.169.254/latest/meta-data/instance-id") - - region = get_metadata_value(token, "http://169.254.169.254/latest/meta-data/placement/availability-zone") - region = region[:-1] - - proxy_config = parse_proxy_config() - - # Configure the AWS CA bundle. - # In US isolated regions the dedicated CA bundle will be used. - # In any other region, the default bundle will be used (None stands for the default settings). - # Note: We want to apply a more general solution that applies to every region, - # but for the time being this is enough to support US isolated regions without - # impacting the other ones. - ca_bundle = f"/etc/pki/{region}/certs/ca-bundle.pem" if region.startswith("us-iso") else None - - ec2 = boto3.client("ec2", region_name=region, config=proxy_config, verify=ca_bundle) - - volume_id = get_device_volume_id(ec2, dev, instance_id) + mapping_file_path = "/dev/disk/by-ebs-volumeid/parallelcluster_dev_id_mapping" + if os.path.isfile(mapping_file_path): + with open(mapping_file_path, "r", encoding="utf-8") as mapping_file: + mapping = json.load(mapping_file) + else: + mapping = {} + volume_id = mapping.get(dev) print(volume_id) diff --git a/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/ec2blkdev-init b/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/ec2blkdev-init deleted file mode 100644 index 76ed0f4c3f..0000000000 --- a/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/ec2blkdev-init +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# -# cloud-init remote -# -# chkconfig: 345 12 99 -# description: Script to call udevadm to setup any links to devices already attached at instance launch - -### BEGIN INIT INFO -# Provides: ec2blkdev -# Required-Start: $network -# Required-Stop: $network -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: ec2blkdev -# Description: Script to call udevadm to setup any links to devices already attached at instance launch -### END INIT INFO - -. /etc/rc.d/init.d/functions - -prog=/sbin/udevadm - -start() { - echo -n $"Starting ec2blkdev: " - $prog trigger --action=change --subsystem-match=block - retval=$? - sleep 15 # Wait for EC2 API calls to complete - echo - return $retval -} - -stop() { - return -} - -case "$1" in - start) - $1 - ;; - stop) - $1 - ;; - *) - echo $"Usage: $0 {start|stop}" - exit 2 -esac -exit $? diff --git a/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/manageVolume.py b/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/manageVolume.py index 13ee4afa63..5db0b6096b 100644 --- a/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/manageVolume.py +++ b/cookbooks/aws-parallelcluster-environment/files/default/ec2_udev_rules/manageVolume.py @@ -7,6 +7,7 @@ import argparse import configparser +import json import os import re import subprocess # nosec B404 @@ -133,6 +134,17 @@ def attach_volume(volume_id, instance_id, ec2): dev = available_devices[0] response = ec2.attach_volume(VolumeId=volume_id, InstanceId=instance_id, Device=dev) + mapping_file_path = "/dev/disk/by-ebs-volumeid/parallelcluster_dev_id_mapping" + if os.path.isfile(mapping_file_path): + with open(mapping_file_path, "r", encoding="utf-8") as mapping_file: + mapping = json.load(mapping_file) + else: + mapping = {} + mapping[dev] = volume_id + os.makedirs(os.path.dirname(mapping_file_path), exist_ok=True) + with open(mapping_file_path, "w", encoding="utf-8") as mapping_file: + json.dump(mapping, mapping_file) + # Poll for volume to attach state = response.get("State") delay = 5 # seconds diff --git a/cookbooks/aws-parallelcluster-environment/files/ubuntu/ec2_udev_rules/ec2blkdev-init b/cookbooks/aws-parallelcluster-environment/files/ubuntu/ec2_udev_rules/ec2blkdev-init deleted file mode 100644 index b06c0e6db1..0000000000 --- a/cookbooks/aws-parallelcluster-environment/files/ubuntu/ec2_udev_rules/ec2blkdev-init +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# -# cloud-init remote -# -# chkconfig: 345 12 99 -# description: Script to call udevadm to setup any links to devices already attached at instance launch - -### BEGIN INIT INFO -# Provides: ec2blkdev -# Required-Start: $network -# Required-Stop: $network -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: ec2blkdev -# Description: Script to call udevadm to setup any links to devices already attached at instance launch -### END INIT INFO - -. /lib/lsb/init-functions - -prog=$(which udevadm) - -start() { - echo -n $"Starting ec2blkdev: " - $prog trigger --action=change --subsystem-match=block - retval=$? - sleep 15 # Wait for EC2 API calls to complete - echo - return $retval -} - -stop() { - return -} - -case "$1" in - start) - $1 - ;; - stop) - $1 - ;; - *) - echo $"Usage: $0 {start|stop}" - exit 2 -esac -exit $? diff --git a/cookbooks/aws-parallelcluster-environment/files/ubuntu/ec2_udev_rules/udev-override.conf b/cookbooks/aws-parallelcluster-environment/files/ubuntu/ec2_udev_rules/udev-override.conf deleted file mode 100644 index db08b6a437..0000000000 --- a/cookbooks/aws-parallelcluster-environment/files/ubuntu/ec2_udev_rules/udev-override.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -IPAddressAllow=any diff --git a/cookbooks/aws-parallelcluster-environment/recipes/config/imds.rb b/cookbooks/aws-parallelcluster-environment/recipes/config/imds.rb index 2209514841..8a845c0f96 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/config/imds.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/config/imds.rb @@ -60,8 +60,8 @@ command "mkdir -p $(dirname #{ip6tables_rules_file}) && ip6tables-save > #{ip6tables_rules_file}" end - template '/etc/init.d/parallelcluster-iptables' do - source 'imds/parallelcluster-iptables.erb' + template '/usr/local/sbin/restore_tables.sh' do + source 'imds/restore_tables.sh.erb' user 'root' group 'root' mode '0744' @@ -71,6 +71,25 @@ ) end + template '/usr/local/sbin/save_tables.sh' do + source 'imds/save_tables.sh.erb' + user 'root' + group 'root' + mode '0744' + variables( + iptables_rules_file: iptables_rules_file, + ip6tables_rules_file: ip6tables_rules_file + ) + end + + template '/etc/systemd/system/parallelcluster-iptables.service' do + source 'imds/parallelcluster-iptables.service.erb' + cookbook 'aws-parallelcluster-environment' + owner 'root' + group 'root' + mode '0644' + end + service "parallelcluster-iptables" do action %i(enable start) end diff --git a/cookbooks/aws-parallelcluster-environment/resources/cloudwatch/cloudwatch_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/cloudwatch/cloudwatch_redhat8.rb index 565bbc7933..8c094e1ac3 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/cloudwatch/cloudwatch_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/cloudwatch/cloudwatch_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :cloudwatch, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_cloudwatch_common' diff --git a/cookbooks/aws-parallelcluster-environment/resources/cloudwatch/cloudwatch_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/cloudwatch/cloudwatch_rocky8.rb index 90a89ac0f3..e60a70f960 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/cloudwatch/cloudwatch_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/cloudwatch/cloudwatch_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :cloudwatch, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_cloudwatch_common' diff --git a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_redhat8.rb index 8cad395a66..015d0af629 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_redhat8.rb @@ -13,7 +13,7 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. provides :ec2_udev_rules, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true diff --git a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_rocky8.rb index f3c538aa1c..0a15d59730 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_rocky8.rb @@ -13,7 +13,7 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. provides :ec2_udev_rules, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true diff --git a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_ubuntu20+.rb b/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_ubuntu20+.rb index 74cab3d13b..cb3be3ad4e 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_ubuntu20+.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/ec2_udev_rules_ubuntu20+.rb @@ -18,12 +18,10 @@ unified_mode true use 'partial/_common_udev_configuration' -use 'partial/_debian_udev_configuration' default_action :setup action :setup do action_create_common_udev_files - action_set_udev_autoreload action_start_ec2blk end diff --git a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/partial/_common_udev_configuration.rb b/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/partial/_common_udev_configuration.rb index 9a87d65bb3..37b2a165f9 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/partial/_common_udev_configuration.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/partial/_common_udev_configuration.rb @@ -47,15 +47,6 @@ mode '0744' end - cookbook_file 'ec2blkdev-init' do - source 'ec2_udev_rules/ec2blkdev-init' - cookbook 'aws-parallelcluster-environment' - path '/etc/init.d/ec2blkdev' - user 'root' - group 'root' - mode '0744' - end - cookbook_file 'manageVolume.py' do source 'ec2_udev_rules/manageVolume.py' cookbook 'aws-parallelcluster-environment' @@ -67,8 +58,7 @@ end action :start_ec2blk do - service "ec2blkdev" do - supports restart: true - action %i(enable start) + execute "Refresh UdevAdmin" do + command "udevadm trigger --action=change --subsystem-match=block" end unless on_docker? end diff --git a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/partial/_debian_udev_configuration.rb b/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/partial/_debian_udev_configuration.rb deleted file mode 100644 index a9c0b34ac7..0000000000 --- a/cookbooks/aws-parallelcluster-environment/resources/ec2_udev_rules/partial/_debian_udev_configuration.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -# -# Copyright:: 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "LICENSE.txt" file accompanying this file. -# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. -# See the License for the specific language governing permissions and limitations under the License. - -action :set_udev_autoreload do - # allow udev to do network call - execute 'udev-daemon-reload' do - command 'udevadm control --reload' - # This means the action is not executed until something triggers it with a 'notifies', as below - # https://docs.chef.io/resources/service/#actions - action :nothing - end - - directory '/etc/systemd/system/systemd-udevd.service.d' - - # Disable udev network sandbox and notify udev to reload configuration - cookbook_file 'udev-override.conf' do - source 'ec2_udev_rules/udev-override.conf' - cookbook 'aws-parallelcluster-environment' - path '/etc/systemd/system/systemd-udevd.service.d/override.conf' - user 'root' - group 'root' - mode '0644' - notifies :run, "execute[udev-daemon-reload]", :immediately unless on_docker? - end -end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efa/efa_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efa/efa_redhat8.rb index 1382cb97e4..13f60c5411 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efa/efa_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efa/efa_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :efa, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true default_action :setup diff --git a/cookbooks/aws-parallelcluster-environment/resources/efa/efa_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/efa/efa_rocky8.rb index cd26878620..5b57732fae 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efa/efa_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efa/efa_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :efa, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true default_action :setup diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index 5d0d26a55b..a0a905a3ae 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :efs, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_get_package_version_rpm' diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_rocky8.rb index 4bb51a9d0a..2924ff4fc7 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :efs, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_get_package_version_rpm' diff --git a/cookbooks/aws-parallelcluster-environment/resources/ephemeral_drives/ephemeral_drives_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/ephemeral_drives/ephemeral_drives_redhat8.rb index 9c89a78a4c..62e2afc80c 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/ephemeral_drives/ephemeral_drives_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/ephemeral_drives/ephemeral_drives_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :ephemeral_drives, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_ephemeral_drives_common.rb' diff --git a/cookbooks/aws-parallelcluster-environment/resources/ephemeral_drives/ephemeral_drives_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/ephemeral_drives/ephemeral_drives_rocky8.rb index 1f1108a784..2b6e442ac1 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/ephemeral_drives/ephemeral_drives_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/ephemeral_drives/ephemeral_drives_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :ephemeral_drives, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_ephemeral_drives_common.rb' diff --git a/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_redhat8.rb index c219c73dd9..a33d92849c 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_redhat8.rb @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :lustre, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true diff --git a/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_rocky8.rb index 656ff3134f..7a723bf03f 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_rocky8.rb @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :lustre, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true diff --git a/cookbooks/aws-parallelcluster-environment/resources/network_service/network_service_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/network_service/network_service_redhat8.rb index 57ebab380f..9151c6b4c8 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/network_service/network_service_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/network_service/network_service_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :network_service, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_network_service' diff --git a/cookbooks/aws-parallelcluster-environment/resources/network_service/network_service_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/network_service/network_service_rocky8.rb index c9d5e08f20..e0344ddbda 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/network_service/network_service_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/network_service/network_service_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :network_service, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_network_service' diff --git a/cookbooks/aws-parallelcluster-environment/resources/nfs/nfs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/nfs/nfs_redhat8.rb index ba1f951784..34f7addc37 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/nfs/nfs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/nfs/nfs_redhat8.rb @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :nfs, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true diff --git a/cookbooks/aws-parallelcluster-environment/resources/nfs/nfs_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/nfs/nfs_rocky8.rb index 6879a1c77a..4019715e88 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/nfs/nfs_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/nfs/nfs_rocky8.rb @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :nfs, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true diff --git a/cookbooks/aws-parallelcluster-environment/resources/raid/raid_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/raid/raid_redhat8.rb index 0d0b98330f..9c92c11da3 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/raid/raid_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/raid/raid_redhat8.rb @@ -1,5 +1,5 @@ provides :raid, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_raid_common' diff --git a/cookbooks/aws-parallelcluster-environment/resources/raid/raid_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/raid/raid_rocky8.rb index 302160810d..df69dbf995 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/raid/raid_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/raid/raid_rocky8.rb @@ -1,5 +1,5 @@ provides :raid, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_raid_common' diff --git a/cookbooks/aws-parallelcluster-environment/resources/spack/spack_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/spack/spack_redhat8.rb index ed9a2612f3..732129c2e7 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/spack/spack_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/spack/spack_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :spack, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_spack_common.rb' diff --git a/cookbooks/aws-parallelcluster-environment/resources/spack/spack_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/spack/spack_rocky8.rb index 3647cfde89..0ed686c890 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/spack/spack_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/spack/spack_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :spack, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_spack_common.rb' diff --git a/cookbooks/aws-parallelcluster-environment/resources/system_authentication/system_authentication_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/system_authentication/system_authentication_redhat8.rb index 30230e1d83..35396e2f91 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/system_authentication/system_authentication_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/system_authentication/system_authentication_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :system_authentication, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_system_authentication_common' diff --git a/cookbooks/aws-parallelcluster-environment/resources/system_authentication/system_authentication_rocky8.rb b/cookbooks/aws-parallelcluster-environment/resources/system_authentication/system_authentication_rocky8.rb index db4e6f7d0b..1068c03ea3 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/system_authentication/system_authentication_rocky8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/system_authentication/system_authentication_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :system_authentication, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_system_authentication_common' diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/imds_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/imds_spec.rb index 11fab0d40e..b6c63fb64c 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/imds_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/imds_spec.rb @@ -56,9 +56,19 @@ is_expected.to run_execute("Save ip6tables rules").with(command: /ip6tables-save/) end - it 'creates iptables init.d file' do - is_expected.to create_template("/etc/init.d/parallelcluster-iptables") - .with(source: 'imds/parallelcluster-iptables.erb') + it 'creates iptables systemd unit file' do + is_expected.to create_template("/etc/systemd/system/parallelcluster-iptables.service") + .with(source: 'imds/parallelcluster-iptables.service.erb') + end + + it 'creates restore table script' do + is_expected.to create_template("/usr/local/sbin/restore_tables.sh") + .with(source: 'imds/restore_tables.sh.erb') + end + + it 'creates save table script' do + is_expected.to create_template("/usr/local/sbin/save_tables.sh") + .with(source: 'imds/save_tables.sh.erb') end it 'starts parallelcluster-iptables service' do diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/ec2_udev_rules_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/ec2_udev_rules_spec.rb index 22b4be9cc6..7d194effb3 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/ec2_udev_rules_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/ec2_udev_rules_spec.rb @@ -46,13 +46,6 @@ def self.setup(chef_run) .with(group: 'root') .with(mode: '0744') - is_expected.to create_cookbook_file('ec2blkdev-init') - .with(source: 'ec2_udev_rules/ec2blkdev-init') - .with(path: '/etc/init.d/ec2blkdev') - .with(user: 'root') - .with(group: 'root') - .with(mode: '0744') - is_expected.to create_cookbook_file('manageVolume.py') .with(source: 'ec2_udev_rules/manageVolume.py') .with(path: '/usr/local/sbin/manageVolume.py') @@ -60,31 +53,6 @@ def self.setup(chef_run) .with(group: 'root') .with(mode: '0755') end - - if platform == 'ubuntu' - it 'sets udev autoreload' do - is_expected.to nothing_execute('udev-daemon-reload') - .with(command: 'udevadm control --reload') - - is_expected.to create_directory('/etc/systemd/system/systemd-udevd.service.d') - - is_expected.to create_cookbook_file('udev-override.conf') - .with(source: 'ec2_udev_rules/udev-override.conf') - .with(path: '/etc/systemd/system/systemd-udevd.service.d/override.conf') - .with(user: 'root') - .with(group: 'root') - .with(mode: '0644') - - expect(chef_run.cookbook_file('udev-override.conf')).to notify('execute[udev-daemon-reload]').to(:run).immediately - end - end - - it 'enables and starts ec2blk service' do - is_expected.to enable_service('ec2blkdev') - .with(supports: { restart: true }) - is_expected.to start_service('ec2blkdev') - .with(supports: { restart: true }) - end end end end diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb index e7466ee6e0..da452dbf29 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb @@ -158,6 +158,9 @@ def mock_already_installed(package, expected_version, installed) for_oses([ %w(centos 7), %w(redhat 8), + %w(rocky 8), + %w(redhat 9), + %w(rocky 9), ]) do |platform, version| context "on #{platform}#{version}" do cached(:source_dir) { 'SOURCE DIR' } @@ -178,6 +181,7 @@ def mock_already_installed(package, expected_version, installed) { "centos" => 'rpm-build', "redhat" => %w(rpm-build make), + "rocky" => %w(rpm-build make), } end diff --git a/cookbooks/aws-parallelcluster-environment/templates/ec2_udev_rules/ec2-volid.rules.erb b/cookbooks/aws-parallelcluster-environment/templates/ec2_udev_rules/ec2-volid.rules.erb index 884df10849..1df1820bdd 100644 --- a/cookbooks/aws-parallelcluster-environment/templates/ec2_udev_rules/ec2-volid.rules.erb +++ b/cookbooks/aws-parallelcluster-environment/templates/ec2_udev_rules/ec2-volid.rules.erb @@ -1,4 +1,4 @@ -KERNEL=="xvd*", ENV{DEVTYPE}=="disk", PROGRAM="<%= @cookbook_virtualenv_path %>/bin/python /sbin/ec2_dev_2_volid.py %k", SYMLINK+="disk/by-ebs-volumeid/%c" -KERNEL=="xvd*", ENV{DEVTYPE}=="partition", PROGRAM="<%= @cookbook_virtualenv_path %>/bin/python /sbin/ec2_dev_2_volid.py %k", SYMLINK+="disk/by-ebs-volumeid/%c-p%n" +KERNEL=="xvd*", KERNEL!="xvda*", ENV{DEVTYPE}=="disk", PROGRAM="<%= @cookbook_virtualenv_path %>/bin/python /sbin/ec2_dev_2_volid.py %k", SYMLINK+="disk/by-ebs-volumeid/%c" +KERNEL=="xvd*", KERNEL!="xvda*", ENV{DEVTYPE}=="partition", PROGRAM="<%= @cookbook_virtualenv_path %>/bin/python /sbin/ec2_dev_2_volid.py %k", SYMLINK+="disk/by-ebs-volumeid/%c-p%n" KERNEL=="nvme*", ENV{DEVTYPE}=="disk", PROGRAM="<%= @cookbook_virtualenv_path %>/bin/python /sbin/ec2_dev_2_volid.py %k", SYMLINK+="disk/by-ebs-volumeid/%c" KERNEL=="nvme*", ENV{DEVTYPE}=="partition", PROGRAM="<%= @cookbook_virtualenv_path %>/bin/python /sbin/ec2_dev_2_volid.py %k", SYMLINK+="disk/by-ebs-volumeid/%c-p%n" diff --git a/cookbooks/aws-parallelcluster-environment/templates/imds/parallelcluster-iptables.erb b/cookbooks/aws-parallelcluster-environment/templates/imds/parallelcluster-iptables.erb deleted file mode 100755 index 273a4d84a8..0000000000 --- a/cookbooks/aws-parallelcluster-environment/templates/imds/parallelcluster-iptables.erb +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# parallelcluster-iptables -# -# chkconfig: 12345 99 99 -# description: Backup and restore iptables rules (both for IPv4 and IPv6) - -### BEGIN INIT INFO -# Provides: $parallelcluster-iptables -# Required-Start: $network -# Required-Stop: $network -# Default-Start: 1 2 3 4 5 -# Default-Stop: 0 6 -# Short-Description: Backup and restore iptables rules -# Description: Backup and restore iptables rules -### END INIT INFO - -IPTABLES_RULES_FILE="<%= @iptables_rules_file %>" -IP6TABLES_RULES_FILE="<%= @ip6tables_rules_file %>" - -function save_tables() { - local iptables_command=$1 - local iptables_file=$2 - echo "saving iptables rules to file: $iptables_file" - mkdir -p $(dirname $iptables_file) - $iptables_command > $iptables_file - echo "iptables rules saved to file: $iptables_file" -} - -function restore_tables() { - local iptables_command=$1 - local iptables_file=$2 - if [[ -f $iptables_file ]]; then - $iptables_command < $iptables_file - echo "iptables rules restored from file: $iptables_file" - else - echo "iptables rules left unchanged as file was not found: $iptables_file" - fi -} - -function start() { - restore_tables iptables-restore $IPTABLES_RULES_FILE - restore_tables ip6tables-restore $IP6TABLES_RULES_FILE -} - -function stop() { - save_tables iptables-save $IPTABLES_RULES_FILE - save_tables ip6tables-save $IP6TABLES_RULES_FILE -} - -case "$1" in -start|stop) - $1 - ;; -*) - echo "Usage: $0 {start|stop}" - exit 2 -esac - -exit $? diff --git a/cookbooks/aws-parallelcluster-environment/templates/imds/parallelcluster-iptables.service.erb b/cookbooks/aws-parallelcluster-environment/templates/imds/parallelcluster-iptables.service.erb new file mode 100644 index 0000000000..e1684e2de2 --- /dev/null +++ b/cookbooks/aws-parallelcluster-environment/templates/imds/parallelcluster-iptables.service.erb @@ -0,0 +1,10 @@ +[Unit] +Description=Backup and restore iptables rules (both for IPv4 and IPv6) +After=network-online.target + +[Service] +ExecStart=/usr/local/sbin/restore_tables.sh +ExecStop=/usr/local/sbin/save_tables.sh + +[Install] +WantedBy=multi-user.target diff --git a/cookbooks/aws-parallelcluster-environment/templates/imds/restore_tables.sh.erb b/cookbooks/aws-parallelcluster-environment/templates/imds/restore_tables.sh.erb new file mode 100644 index 0000000000..45ab1c6523 --- /dev/null +++ b/cookbooks/aws-parallelcluster-environment/templates/imds/restore_tables.sh.erb @@ -0,0 +1,31 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the +# License. A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and +# limitations under the License. + +IPTABLES_RULES_FILE="<%= @iptables_rules_file %>" +IP6TABLES_RULES_FILE="<%= @ip6tables_rules_file %>" + +function restore_tables() { + local iptables_command=$1 + local iptables_file=$2 + if [[ -f $iptables_file ]]; then + $iptables_command < $iptables_file + echo "iptables rules restored from file: $iptables_file" + else + echo "iptables rules left unchanged as file was not found: $iptables_file" + fi +} + +function main { + restore_tables iptables-restore $IPTABLES_RULES_FILE + restore_tables ip6tables-restore $IP6TABLES_RULES_FILE +} + +main diff --git a/cookbooks/aws-parallelcluster-environment/templates/imds/save_tables.sh.erb b/cookbooks/aws-parallelcluster-environment/templates/imds/save_tables.sh.erb new file mode 100644 index 0000000000..fc8d1bc7c6 --- /dev/null +++ b/cookbooks/aws-parallelcluster-environment/templates/imds/save_tables.sh.erb @@ -0,0 +1,29 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the +# License. A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and +# limitations under the License. + +IPTABLES_RULES_FILE="<%= @iptables_rules_file %>" +IP6TABLES_RULES_FILE="<%= @ip6tables_rules_file %>" + +function save_tables() { + local iptables_command=$1 + local iptables_file=$2 + echo "saving iptables rules to file: $iptables_file" + mkdir -p $(dirname $iptables_file) + $iptables_command > $iptables_file + echo "iptables rules saved to file: $iptables_file" +} + +function main { + save_tables iptables-save $IPTABLES_RULES_FILE + save_tables ip6tables-save $IP6TABLES_RULES_FILE +} + +main diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/ec2_udev_rules_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/ec2_udev_rules_spec.rb index 5b7ad1c6f9..199fd35850 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/ec2_udev_rules_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/ec2_udev_rules_spec.rb @@ -9,7 +9,7 @@ its('mode') { should cmp '0644' } end - paths = %w(/usr/local/sbin/parallelcluster-ebsnvme-id /sbin/ec2_dev_2_volid.py /etc/init.d/ec2blkdev) + paths = %w(/usr/local/sbin/parallelcluster-ebsnvme-id /sbin/ec2_dev_2_volid.py) paths.each do |path| describe file(path) do it { should exist } @@ -29,28 +29,8 @@ end end -control 'tag:install_ec2_udev_rules_debian_udevd_reload_configuration' do - title "Configuration to reload the udevd daemon when the override.conf changes" - - only_if { os.debian? } - - describe file('/etc/systemd/system/systemd-udevd.service.d/override.conf') do - it { should exist } - its('content') { should_not be_empty } - its('owner') { should eq 'root' } - its('group') { should eq 'root' } - its('mode') { should cmp '0644' } - end -end - control 'tag:install_tag:config_ec2_udev_rules_ec2blkdev_service_installation' do title "Installation of the ec2blkdev service" only_if { !os_properties.on_docker? } - - describe service('ec2blkdev') do - it { should be_installed } - it { should be_enabled } - it { should be_running } - end end diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/imds_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/imds_spec.rb index afc95c2df9..b6a27ac02d 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/imds_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/imds_spec.rb @@ -38,21 +38,6 @@ describe service('parallelcluster-iptables') do it { should be_installed } it { should be_enabled } - it { should be_running } - end - - %w(1 2 3 4 5).each do |level| - describe "Check parallelcluster-iptables run level #{level} on" do - subject { bash("ls /etc/rc#{level}.d/ | egrep '^S[0-9]+parallelcluster-iptables$'") } - its('exit_status') { should eq(0) } - end - end - - %w(0 6).each do |level| - describe "Check parallelcluster-iptables run level #{level} off" do - subject { bash("ls /etc/rc#{level}.d/ | egrep '^K[0-9]+parallelcluster-iptables$'") } - its('exit_status') { should eq(0) } - end end describe file("#{node['cluster']['etc_dir']}/sysconfig/iptables.rules") do diff --git a/cookbooks/aws-parallelcluster-platform/files/dcv/pcluster_dcv_connect.sh b/cookbooks/aws-parallelcluster-platform/files/dcv/pcluster_dcv_connect.sh index 50122d4c58..337d10ab2a 100644 --- a/cookbooks/aws-parallelcluster-platform/files/dcv/pcluster_dcv_connect.sh +++ b/cookbooks/aws-parallelcluster-platform/files/dcv/pcluster_dcv_connect.sh @@ -112,7 +112,7 @@ main() { os=$(< /etc/chef/dna.json jq -r .cluster.base_os) _log "Input parameters: user: ${user}, OS: ${os}, shared_folder_path: ${shared_folder_path}." - if ! [[ "${os}" =~ ^(alinux2|ubuntu2004|ubuntu2204|centos[7-8]|rhel8|rocky8)$ ]]; then + if ! [[ "${os}" =~ ^(alinux2|ubuntu2004|ubuntu2204|centos[7-8]|rhel8|rocky8|rhel9|rocky9)$ ]]; then _fail "OS not supported." fi diff --git a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_amazon2.rb b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_amazon2.rb index 2618232ec5..8641f659d6 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_amazon2.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_amazon2.rb @@ -18,6 +18,10 @@ action_class do def armpl_platform - 'RHEL-8' + 'AmazonLinux-2' + end + + def gcc_major_minor_version + '9.3' end end diff --git a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_centos7.rb b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_centos7.rb index 3931d44373..bc575b65e3 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_centos7.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_centos7.rb @@ -30,4 +30,8 @@ def armpl_platform 'RHEL-7' end + + def gcc_major_minor_version + '9.3' + end end diff --git a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_redhat8.rb index 96ea4b9826..8be3828836 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_redhat8.rb @@ -13,13 +13,22 @@ # See the License for the specific language governing permissions and limitations under the License. provides :arm_pl, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_arm_pl_common.rb' action_class do def armpl_platform - 'RHEL-8' + "RHEL-#{node['platform_version'].to_i}" + end + + def gcc_major_minor_version + case node['platform_version'].to_i + when 8 + '9.3' + else + '11.3' + end end end diff --git a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_rocky8.rb index 4f401b5551..65d18678dc 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_rocky8.rb @@ -13,13 +13,22 @@ # See the License for the specific language governing permissions and limitations under the License. provides :arm_pl, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_arm_pl_common.rb' action_class do def armpl_platform - 'RHEL-8' + "RHEL-#{node['platform_version'].to_i}" + end + + def gcc_major_minor_version + case node['platform_version'].to_i + when 8 + '9.3' + else + '11.3' + end end end diff --git a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_ubuntu20.rb b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_ubuntu20.rb index b3aab5d66a..743e871968 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_ubuntu20.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_ubuntu20.rb @@ -26,4 +26,8 @@ def armpl_platform def modulefile_dir "/usr/share/modules/modulefiles" end + + def gcc_major_minor_version + '9.3' + end end diff --git a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_ubuntu22.rb b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_ubuntu22.rb index 093cde2067..365343afc3 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_ubuntu22.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/arm_pl_ubuntu22.rb @@ -18,11 +18,6 @@ use 'partial/_arm_pl_common.rb' -property :armpl_major_minor_version, String, default: '23.04' -property :armpl_patch_version, String, default: '1' -property :gcc_major_minor_version, String, default: '11.3' -property :gcc_patch_version, String, default: '0' - action_class do def armpl_platform "Ubuntu-#{node['platform_version']}" @@ -31,4 +26,8 @@ def armpl_platform def modulefile_dir "/usr/share/modules/modulefiles" end + + def gcc_major_minor_version + '11.3' + end end diff --git a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/partial/_arm_pl_common.rb b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/partial/_arm_pl_common.rb index 34f8930444..284d86179f 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/arm_pl/partial/_arm_pl_common.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/arm_pl/partial/_arm_pl_common.rb @@ -27,9 +27,7 @@ # We upload ArmPL to a ParallelCluster bucket (account for it in scope of the upgrade) and download it from there # to install ArmPL on the AMI. # We download gcc directly from gnu.org repository to install correct gcc version on the AMI. -property :armpl_major_minor_version, String, default: '21.0' -property :armpl_patch_version, String, default: '0' -property :gcc_major_minor_version, String, default: '9.3' +property :armpl_major_minor_version, String, default: '23.10' property :gcc_patch_version, String, default: '0' action :arm_pl_prerequisite do @@ -53,8 +51,8 @@ action_arm_pl_prerequisite - armpl_version = "#{new_resource.armpl_major_minor_version}.#{new_resource.armpl_patch_version}" - armpl_tarball_name = "arm-performance-libraries_#{armpl_version}_#{armpl_platform}_gcc-#{new_resource.gcc_major_minor_version}.tar" + armpl_version = "#{new_resource.armpl_major_minor_version}" + armpl_tarball_name = "arm-performance-libraries_#{armpl_version}_#{armpl_platform}_gcc-#{gcc_major_minor_version}.tar" armpl_url = %W( https://#{new_resource.region}-aws-parallelcluster.s3.#{new_resource.region}.#{new_resource.aws_domain} @@ -92,9 +90,9 @@ directory "#{modulefile_dir}/armpl" armpl_license_dir = if new_resource.armpl_major_minor_version == "21.0" - "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{new_resource.armpl_major_minor_version}_gcc-#{new_resource.gcc_major_minor_version}/license_terms" + "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{new_resource.armpl_major_minor_version}_gcc-#{gcc_major_minor_version}/license_terms" else - "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{armpl_version}_gcc-#{new_resource.gcc_major_minor_version}/license_terms" + "/opt/arm/armpl/#{armpl_version}/arm-performance-libraries_#{armpl_version}_gcc-#{gcc_major_minor_version}/license_terms" end # arm performance library modulefile configuration @@ -108,11 +106,11 @@ armpl_version: armpl_version, armpl_major_minor_version: new_resource.armpl_major_minor_version, armpl_license_dir: armpl_license_dir, - gcc_major_minor_version: new_resource.gcc_major_minor_version + gcc_major_minor_version: gcc_major_minor_version ) end - gcc_version = "#{new_resource.gcc_major_minor_version}.#{new_resource.gcc_patch_version}" + gcc_version = "#{gcc_major_minor_version}.#{new_resource.gcc_patch_version}" gcc_url = "https://ftp.gnu.org/gnu/gcc/gcc-#{gcc_version}/gcc-#{gcc_version}.tar.gz" gcc_tarball = "#{new_resource.sources_dir}/gcc-#{gcc_version}.tar.gz" @@ -152,7 +150,7 @@ creates '/opt/arm/armpl/gcc' end - gcc_modulefile = "/opt/arm/armpl/#{armpl_version}/modulefiles/armpl/gcc-#{new_resource.gcc_major_minor_version}" + gcc_modulefile = "/opt/arm/armpl/#{armpl_version}/modulefiles/armpl/gcc-#{gcc_major_minor_version}" # gcc modulefile configuration template gcc_modulefile do @@ -170,9 +168,8 @@ # to dependencies (for instance, test code) # Complete versions are intentionally redundant. node.default['cluster']['armpl']['major_minor_version'] = new_resource.armpl_major_minor_version - node.default['cluster']['armpl']['patch_version'] = new_resource.armpl_patch_version node.default['cluster']['armpl']['version'] = armpl_version - node.default['cluster']['armpl']['gcc']['major_minor_version'] = new_resource.gcc_major_minor_version + node.default['cluster']['armpl']['gcc']['major_minor_version'] = gcc_major_minor_version node.default['cluster']['armpl']['gcc']['patch_version'] = new_resource.gcc_patch_version node.default['cluster']['armpl']['gcc']['version'] = gcc_version diff --git a/cookbooks/aws-parallelcluster-platform/resources/build_tools/build_tools_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/build_tools/build_tools_redhat8.rb index df9a5711f7..846d2ebd89 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/build_tools/build_tools_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/build_tools/build_tools_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :build_tools, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_build_tools_yum.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/build_tools/build_tools_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/build_tools/build_tools_rocky8.rb index ba077e3b06..537f8af9ac 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/build_tools/build_tools_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/build_tools/build_tools_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :build_tools, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_build_tools_yum.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/c_states/c_states_redhat9.rb b/cookbooks/aws-parallelcluster-platform/resources/c_states/c_states_redhat9.rb new file mode 100644 index 0000000000..f75e51f3e2 --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/c_states/c_states_redhat9.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# Copyright:: 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +provides :c_states, platform: 'redhat' do |node| + node['platform_version'].to_i == 9 +end + +use 'partial/_c_states_common' +use 'partial/_c_states_redhat_9_based' diff --git a/cookbooks/aws-parallelcluster-platform/resources/c_states/c_states_rocky9.rb b/cookbooks/aws-parallelcluster-platform/resources/c_states/c_states_rocky9.rb new file mode 100644 index 0000000000..d02e536dba --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/c_states/c_states_rocky9.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# Copyright:: 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +provides :c_states, platform: 'rocky' do |node| + node['platform_version'].to_i == 9 +end + +use 'partial/_c_states_common' +use 'partial/_c_states_redhat_9_based' diff --git a/cookbooks/aws-parallelcluster-platform/resources/c_states/partial/_c_states_redhat_9_based.rb b/cookbooks/aws-parallelcluster-platform/resources/c_states/partial/_c_states_redhat_9_based.rb new file mode 100644 index 0000000000..67b78d56ce --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/c_states/partial/_c_states_redhat_9_based.rb @@ -0,0 +1,4 @@ +action :setup do + return if !x86_instance? || on_docker? + shell_out!('grubby --update-kernel=ALL --args="intel_idle.max_cstate=1 processor.max_cstate=1"') +end diff --git a/cookbooks/aws-parallelcluster-platform/resources/chrony/chrony_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/chrony/chrony_redhat8.rb index fe6a606004..c12911975f 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/chrony/chrony_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/chrony/chrony_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :chrony, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_chrony_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/chrony/chrony_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/chrony/chrony_rocky8.rb index ba8dce65dd..cf9537efd1 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/chrony/chrony_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/chrony/chrony_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :chrony, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_chrony_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/dcv/dcv_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/dcv/dcv_redhat8.rb index 070a8d3ad2..5716d4e2c3 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/dcv/dcv_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/dcv/dcv_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :dcv, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_dcv_common' diff --git a/cookbooks/aws-parallelcluster-platform/resources/dcv/dcv_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/dcv/dcv_rocky8.rb index 794844604f..5dc56c74dc 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/dcv/dcv_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/dcv/dcv_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :dcv, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_dcv_common' diff --git a/cookbooks/aws-parallelcluster-platform/resources/dcv/partial/_rhel_common.rb b/cookbooks/aws-parallelcluster-platform/resources/dcv/partial/_rhel_common.rb index d3c631c582..2832ba273d 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/dcv/partial/_rhel_common.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/dcv/partial/_rhel_common.rb @@ -15,30 +15,56 @@ def dcv_sha256sum if arm_instance? - '37aaaabfe1b8dde83254c738aa562b7eb7b1663cea09a53cedf9dabe5ddbb184' + case el_string + when "el7" + '37aaaabfe1b8dde83254c738aa562b7eb7b1663cea09a53cedf9dabe5ddbb184' + when "el8" + 'c5366bde8fb49bb41f2bb04e3390e5bda9aa668c0122e4fbf663e9a18c049edf' + when "el9" + '07148327f30dae656b338e24102945830601e85df4e135029b624df1e9ee3c2e' + else + '' + end + else + case el_string + when "el7" + 'e82e434a3f0c5d1a48d7cda2b6100ce51ae250d93b6a17b2fb81ba0fda463b3b' + when "el8" + '28a15bb557f59580f3b58d9bf99430d27d98efbaa9007ab3b2eca46e1a51fa4f' + when "el9" + 'eb906f41b0a064005499ab6a1be42c8d218a3fbaf9444cb907990aa6171bdea4' + else + '' + end + end +end + +def el_string + if platform?('amazon') + "el7" else - 'e82e434a3f0c5d1a48d7cda2b6100ce51ae250d93b6a17b2fb81ba0fda463b3b' + "el#{node['platform_version'].to_i}" end end def dcv_package - "nice-dcv-#{node['cluster']['dcv']['version']}-el7-#{dcv_url_arch}" + "nice-dcv-#{node['cluster']['dcv']['version']}-#{el_string}-#{dcv_url_arch}" end def dcv_server - "nice-dcv-server-#{node['cluster']['dcv']['server']['version']}.el7.#{dcv_url_arch}.rpm" + "nice-dcv-server-#{node['cluster']['dcv']['server']['version']}.#{el_string}.#{dcv_url_arch}.rpm" end def xdcv - "nice-xdcv-#{node['cluster']['dcv']['xdcv']['version']}.el7.#{dcv_url_arch}.rpm" + "nice-xdcv-#{node['cluster']['dcv']['xdcv']['version']}.#{el_string}.#{dcv_url_arch}.rpm" end def dcv_web_viewer - "nice-dcv-web-viewer-#{node['cluster']['dcv']['web_viewer']['version']}.el7.#{dcv_url_arch}.rpm" + "nice-dcv-web-viewer-#{node['cluster']['dcv']['web_viewer']['version']}.#{el_string}.#{dcv_url_arch}.rpm" end def dcv_gl - "nice-dcv-gl-#{node['cluster']['dcv']['gl']['version']}.el7.#{dcv_url_arch}.rpm" + "nice-dcv-gl-#{node['cluster']['dcv']['gl']['version']}.#{el_string}.#{dcv_url_arch}.rpm" end action_class do diff --git a/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/fabric_manager_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/fabric_manager_redhat8.rb index 86cff170d6..1eb5216da0 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/fabric_manager_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/fabric_manager_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :fabric_manager, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_fabric_manager_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/fabric_manager_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/fabric_manager_rocky8.rb index 576c7b40c0..8d12f10331 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/fabric_manager_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/fabric_manager_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :fabric_manager, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_fabric_manager_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_redhat8.rb index 17e96cee8f..7a66fbdd9e 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :gdrcopy, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_gdrcopy_common.rb' @@ -24,7 +24,7 @@ def gdrcopy_enabled? end def gdrcopy_platform - 'el8' + "el#{node['platform_version'].to_i}" end def gdrcopy_arch diff --git a/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_rocky8.rb index a55e80849e..b63152df6e 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :gdrcopy, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_gdrcopy_common.rb' @@ -24,7 +24,7 @@ def gdrcopy_enabled? end def gdrcopy_platform - 'el8' + "el#{node['platform_version'].to_i}" end def gdrcopy_arch diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb index 2021888015..c30f17c0c6 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :install_packages, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_install_packages_common.rb' @@ -24,11 +24,11 @@ def default_packages # Removed libssh2-devel from base_packages since is not shipped by RedHat 8 and in conflict with package libssh-0.9.6-3.el8.x86_64 # iptables needed for IMDS setup %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel - libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf pyparted libtool - httpd boost-devel redhat-lsb mlocate R atlas-devel + libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf libtool + httpd boost-devel mlocate R atlas-devel blas-devel libffi-devel dkms libedit-devel jq libical-devel sendmail libxml2-devel libglvnd-devel - python2 python2-pip libgcrypt-devel libevent-devel glibc-static bind-utils + libgcrypt-devel libevent-devel glibc-static bind-utils iproute NetworkManager-config-routing-rules python3 python3-pip iptables libcurl-devel yum-plugin-versionlock coreutils moreutils curl environment-modules gcc gcc-c++ bzip2) end diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_rocky8.rb index a2cbf87a29..20539b861c 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :install_packages, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_install_packages_common.rb' @@ -48,11 +48,11 @@ def default_packages # environment-modules required by EFA, Intel MPI and ARM PL # iptables needed for IMDS setup packages = %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel - libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf pyparted libtool - httpd boost-devel redhat-lsb mlocate R atlas-devel + libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf libtool + httpd boost-devel mlocate R atlas-devel blas-devel libffi-devel dkms libedit-devel jq libical-devel sendmail libxml2-devel libglvnd-devel - python2 python2-pip libgcrypt-devel libevent-devel glibc-static bind-utils + libgcrypt-devel libevent-devel glibc-static bind-utils iproute NetworkManager-config-routing-rules python3 python3-pip iptables libcurl-devel yum-plugin-versionlock moreutils curl environment-modules gcc gcc-c++ bzip2) packages.append("coreutils") unless on_docker? # on docker image coreutils conflict with coreutils-single, already installed on it diff --git a/cookbooks/aws-parallelcluster-platform/resources/modules/modules_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/modules/modules_redhat8.rb index 66ea8817f5..5c0b12d540 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/modules/modules_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/modules/modules_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :modules, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_modules_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/modules/modules_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/modules/modules_rocky8.rb index 26e6232dff..5377f0cd82 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/modules/modules_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/modules/modules_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :modules, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_modules_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/nvidia_dcgm_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/nvidia_dcgm_redhat8.rb index 1712af81dd..88a2e98e71 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/nvidia_dcgm_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/nvidia_dcgm_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :nvidia_dcgm, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_nvidia_dcgm_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/nvidia_dcgm_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/nvidia_dcgm_rocky8.rb index 3f6afed070..b56aa2cf5b 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/nvidia_dcgm_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_dcgm/nvidia_dcgm_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :nvidia_dcgm, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_nvidia_dcgm_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_redhat8.rb index bfa58c063f..5866b8ebcf 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :nvidia_driver, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_nvidia_driver_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_rocky8.rb index 85ca4be912..6c91e05936 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :nvidia_driver, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_nvidia_driver_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_redhat8.rb index 5185ac7138..524e968fb4 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_redhat8.rb @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and limitations under the License. provides :nvidia_repo, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_nvidia_repo_common.rb' def platform - 'rhel8' + "rhel#{node['platform_version'].to_i}" end def repository_key diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_rocky8.rb index 335c43e50f..c23c4a7ef1 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_rocky8.rb @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and limitations under the License. provides :nvidia_repo, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_nvidia_repo_common.rb' def platform - 'rhel8' + "rhel#{node['platform_version'].to_i}" end def repository_key diff --git a/cookbooks/aws-parallelcluster-platform/resources/stunnel/stunnel_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/stunnel/stunnel_redhat8.rb index e6e837f3fc..92ab4b89a7 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/stunnel/stunnel_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/stunnel/stunnel_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :stunnel, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_common' diff --git a/cookbooks/aws-parallelcluster-platform/resources/stunnel/stunnel_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/stunnel/stunnel_rocky8.rb index ac2ba57675..9a9700974e 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/stunnel/stunnel_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/stunnel/stunnel_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :stunnel, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_common' diff --git a/cookbooks/aws-parallelcluster-platform/resources/sudo_access/sudo_access_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/sudo_access/sudo_access_redhat8.rb index 963fad3a62..83baee7ba1 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/sudo_access/sudo_access_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/sudo_access/sudo_access_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :sudo_access, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_sudo_access_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/sudo_access/sudo_access_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/sudo_access/sudo_access_rocky8.rb index fbc952e5e8..b9acf26c9a 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/sudo_access/sudo_access_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/sudo_access/sudo_access_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :sudo_access, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_sudo_access_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb index cb0df42643..5438d19b37 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb @@ -16,18 +16,7 @@ def self.setup(chef_run) cached(:aws_region) { 'test_region' } cached(:aws_domain) { 'test_domain' } cached(:armpl_major_minor_version) do - if platform == 'ubuntu' && version == '22.04' - '23.04' - else - '21.0' - end - end - cached(:armpl_patch_version) do - if platform == 'ubuntu' && version == '22.04' - '1' - else - '0' - end + '23.10' end cached(:armpl_platform) do @@ -36,13 +25,15 @@ def self.setup(chef_run) 'RHEL-7' when 'ubuntu' "Ubuntu-#{version}" + when 'amazon' + "AmazonLinux-2" else - 'RHEL-8' + "RHEL-#{version}" end end cached(:gcc_major_minor_version) do - if platform == 'ubuntu' && version == '22.04' + if platform == 'ubuntu' && version == '22.04' || version == '9' '11.3' else '9.3' @@ -52,7 +43,7 @@ def self.setup(chef_run) cached(:gcc_patch_version) { '0' } cached(:sources_dir) { 'sources_test_dir' } cached(:modulefile_dir) { platform == 'ubuntu' ? '/usr/share/modules/modulefiles' : '/usr/share/Modules/modulefiles' } - cached(:armpl_version) { "#{armpl_major_minor_version}.#{armpl_patch_version}" } + cached(:armpl_version) { "#{armpl_major_minor_version}" } cached(:armpl_tarball_name) { "arm-performance-libraries_#{armpl_version}_#{armpl_platform}_gcc-#{gcc_major_minor_version}.tar" } cached(:armpl_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.#{aws_domain}/archives/armpl/#{armpl_platform}/#{armpl_tarball_name}" } cached(:armpl_installer) { "#{sources_dir}/#{armpl_tarball_name}" } @@ -187,7 +178,6 @@ def self.setup(chef_run) it 'sets node attributes' do expect(node['cluster']['armpl']['major_minor_version']).to eq(armpl_major_minor_version) - expect(node['cluster']['armpl']['patch_version']).to eq(armpl_patch_version) expect(node['cluster']['armpl']['version']).to eq(armpl_version) expect(node['cluster']['armpl']['gcc']['major_minor_version']).to eq(gcc_major_minor_version) expect(node['cluster']['armpl']['gcc']['patch_version']).to eq(gcc_patch_version) diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/c_states_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/c_states_spec.rb index 26eb76d284..83365434f9 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/c_states_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/c_states_spec.rb @@ -11,6 +11,9 @@ def self.setup(chef_run) end describe 'c_states:setup' do + stubs_for_provider("c_states[setup]") do |provider| + allow(provider).to receive_shell_out("grubby --update-kernel=ALL --args=\"intel_idle.max_cstate=1 processor.max_cstate=1\"") + end before do stubs_for_resource('c_states') do |res| allow(res).to receive(:append_if_not_present_grub_cmdline) @@ -40,16 +43,18 @@ def self.setup(chef_run) is_expected.to setup_c_states('setup') end - it 'edits /etc/default/grub' do - stubs_for_resource('c_states[setup]') do |res| - expect(res).to receive(:append_if_not_present_grub_cmdline).with(grub_cmdline_attributes, grub_variable) + if version.to_i != 9 + it 'edits /etc/default/grub' do + stubs_for_resource('c_states[setup]') do |res| + expect(res).to receive(:append_if_not_present_grub_cmdline).with(grub_cmdline_attributes, grub_variable) + end + chef_run end - chef_run - end - it 'regenerate grub boot menus' do - is_expected.to run_execute('Regenerate grub boot menu') - .with(command: regenerate_grub_boot_menu_command) + it 'regenerate grub boot menus' do + is_expected.to run_execute('Regenerate grub boot menu') + .with(command: regenerate_grub_boot_menu_command) + end end end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/dcv_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/dcv_spec.rb index 4651f5ad3a..8ba950f3b5 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/dcv_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/dcv_spec.rb @@ -143,11 +143,12 @@ def self.nothing(chef_run) expect(resource.dcv_web_viewer).to eq("nice-dcv-web-viewer_#{dcv_webviewer_version}_#{dcv_pkg_arch}.#{base_os}.deb") expect(resource.dcv_gl).to eq("/nice-dcv-gl_#{dcv_gl_version}_#{dcv_pkg_arch}.#{base_os}.deb") else - expect(resource.dcv_package).to eq("nice-dcv-#{dcv_version}-el7-#{dcv_url_arch}") - expect(resource.dcv_server).to eq("nice-dcv-server-#{dcv_server_version}.el7.#{dcv_url_arch}.rpm") - expect(resource.xdcv).to eq("nice-xdcv-#{xdcv_version}.el7.#{dcv_url_arch}.rpm") - expect(resource.dcv_web_viewer).to eq("nice-dcv-web-viewer-#{dcv_webviewer_version}.el7.#{dcv_url_arch}.rpm") - expect(resource.dcv_gl).to eq("nice-dcv-gl-#{dcv_gl_version}.el7.#{dcv_url_arch}.rpm") + dcv_platform_version = platform == "amazon" ? "7" : version.to_i + expect(resource.dcv_package).to eq("nice-dcv-#{dcv_version}-el#{dcv_platform_version}-#{dcv_url_arch}") + expect(resource.dcv_server).to eq("nice-dcv-server-#{dcv_server_version}.el#{dcv_platform_version}.#{dcv_url_arch}.rpm") + expect(resource.xdcv).to eq("nice-xdcv-#{xdcv_version}.el#{dcv_platform_version}.#{dcv_url_arch}.rpm") + expect(resource.dcv_web_viewer).to eq("nice-dcv-web-viewer-#{dcv_webviewer_version}.el#{dcv_platform_version}.#{dcv_url_arch}.rpm") + expect(resource.dcv_gl).to eq("nice-dcv-gl-#{dcv_gl_version}.el#{dcv_platform_version}.#{dcv_url_arch}.rpm") end end end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb index f1fb3708ba..f96eb304c5 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb @@ -87,7 +87,7 @@ def self.remove(chef_run) when 'amazon', 'centos' 'rhel7' when 'redhat', 'rocky' - 'rhel8' + "rhel#{version.to_i}" when 'ubuntu' "ubuntu#{version.delete('.')}" end @@ -127,7 +127,7 @@ def self.remove(chef_run) when 'amazon', 'centos' 'rhel7' when 'redhat' - 'rhel8' + "rhel#{version}" when 'ubuntu' "ubuntu#{version.delete('.')}" end diff --git a/cookbooks/aws-parallelcluster-platform/test/controls/c_states_spec.rb b/cookbooks/aws-parallelcluster-platform/test/controls/c_states_spec.rb index 9f946ae976..519800f3ac 100644 --- a/cookbooks/aws-parallelcluster-platform/test/controls/c_states_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/test/controls/c_states_spec.rb @@ -9,7 +9,7 @@ its('content') { should match(/intel_idle.max_cstate=1/) } end - if os.redhat? # redhat includes amazon + if os.redhat8? || os.alinux2? || os.centos7? || os.rocky8? describe file('/boot/grub2/grub.cfg') do it { should exist } @@ -24,14 +24,6 @@ its('content') { should match(/processor.max_cstate=1/) } its('content') { should match(/intel_idle.max_cstate=1/) } end - - else - describe "unsupported OS" do - # this produces a skipped control (ignore-like) - # adding a new OS to kitchen platform list and running the tests, - # it would surface the fact this recipe does not support this OS. - pending "support for #{os.name}-#{os.release} needs to be implemented" - end end end diff --git a/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_gdrcopy_spec.rb b/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_gdrcopy_spec.rb index 9510f1682e..49aa9ac984 100644 --- a/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_gdrcopy_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_gdrcopy_spec.rb @@ -52,7 +52,7 @@ end describe 'gdrcopy service should be disabled' do - subject { command("systemctl is-enabled #{node['cluster']['nvidia']['gdrcopy']['service']} | grep disabled") } - its('exit_status') { should eq 0 } + subject { command("systemctl is-enabled #{node['cluster']['nvidia']['gdrcopy']['service']}") } + its('exit_status') { should eq 1 } end end diff --git a/cookbooks/aws-parallelcluster-shared/attributes/users_redhat8.rb b/cookbooks/aws-parallelcluster-shared/attributes/users_redhat8.rb index bdd4fc475e..66947d728b 100644 --- a/cookbooks/aws-parallelcluster-shared/attributes/users_redhat8.rb +++ b/cookbooks/aws-parallelcluster-shared/attributes/users_redhat8.rb @@ -1,4 +1,4 @@ -return unless platform?('redhat') && node['platform_version'].to_i == 8 +return unless platform?('redhat') && node['platform_version'].to_i >= 8 default['cluster']['cluster_user'] = 'ec2-user' default['cluster']['cluster_user_home'] = "/home/#{node['cluster']['cluster_user']}" diff --git a/cookbooks/aws-parallelcluster-shared/attributes/users_rocky8.rb b/cookbooks/aws-parallelcluster-shared/attributes/users_rocky8.rb index 5f4369b2ee..47eff41c2f 100644 --- a/cookbooks/aws-parallelcluster-shared/attributes/users_rocky8.rb +++ b/cookbooks/aws-parallelcluster-shared/attributes/users_rocky8.rb @@ -1,4 +1,4 @@ -return unless platform?('rocky') && node['platform_version'].to_i == 8 +return unless platform?('rocky') && node['platform_version'].to_i >= 8 default['cluster']['cluster_user'] = 'rocky' default['cluster']['cluster_user_home'] = "/home/#{node['cluster']['cluster_user']}" diff --git a/cookbooks/aws-parallelcluster-shared/resources/os_type/os_type_redhat8.rb b/cookbooks/aws-parallelcluster-shared/resources/os_type/os_type_redhat8.rb index c9dd56141b..90835f14fc 100644 --- a/cookbooks/aws-parallelcluster-shared/resources/os_type/os_type_redhat8.rb +++ b/cookbooks/aws-parallelcluster-shared/resources/os_type/os_type_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :os_type, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_os_type_common.rb' diff --git a/cookbooks/aws-parallelcluster-shared/resources/os_type/os_type_rocky8.rb b/cookbooks/aws-parallelcluster-shared/resources/os_type/os_type_rocky8.rb index 9a915707c8..82ec876c46 100644 --- a/cookbooks/aws-parallelcluster-shared/resources/os_type/os_type_rocky8.rb +++ b/cookbooks/aws-parallelcluster-shared/resources/os_type/os_type_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :os_type, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_os_type_common.rb' diff --git a/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_redhat8.rb b/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_redhat8.rb index e34ef2f887..9e5ffac37b 100644 --- a/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_redhat8.rb +++ b/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :package_repos, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true @@ -32,7 +32,7 @@ execute 'yum-config-manager-rhel' do # Needed by hwloc-devel blas-devel libedit-devel and glibc-static packages - command "yum-config-manager --enable codeready-builder-for-rhel-8-rhui-rpms" + command "yum-config-manager --enable codeready-builder-for-rhel-#{node['platform_version'].to_i}-rhui-rpms" end unless on_docker? execute 'yum-config-manager_skip_if_unavail' do diff --git a/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_rocky8.rb b/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_rocky8.rb index 132bc0b3a3..c816a3a7e7 100644 --- a/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_rocky8.rb +++ b/cookbooks/aws-parallelcluster-shared/resources/package_repos/package_repos_rocky8.rb @@ -13,10 +13,20 @@ # See the License for the specific language governing permissions and limitations under the License. provides :package_repos, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end unified_mode true +def powertool_name + # See details here: https://wiki.rockylinux.org/rocky/repo/#base-repositories + case node['platform_version'].to_i + when 8 + "powertools" + when 9 + "crb" + end +end + use 'partial/_package_repos_rpm.rb' default_action :setup @@ -32,7 +42,7 @@ execute 'yum-config-manager-powertools' do # Needed by hwloc-devel blas-devel libedit-devel and glibc-static packages - command "yum-config-manager --enable powertools" + command "yum-config-manager --enable #{powertool_name}" end execute 'yum-config-manager_skip_if_unavail' do diff --git a/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb b/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb index 97a092026a..e81110e3e1 100644 --- a/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb +++ b/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb @@ -44,6 +44,8 @@ def for_all_oses %w(ubuntu 22.04), %w(redhat 8), %w(rocky 8), + %w(redhat 9), + %w(rocky 9), ].each do |platform, version| yield(platform, version) end diff --git a/cookbooks/aws-parallelcluster-shared/spec/unit/resources/package_repos_spec.rb b/cookbooks/aws-parallelcluster-shared/spec/unit/resources/package_repos_spec.rb index adbcc84ee3..5a37d90926 100644 --- a/cookbooks/aws-parallelcluster-shared/spec/unit/resources/package_repos_spec.rb +++ b/cookbooks/aws-parallelcluster-shared/spec/unit/resources/package_repos_spec.rb @@ -60,7 +60,7 @@ def self.setup(chef_run) it 'enables rhui' do is_expected.to run_execute('yum-config-manager-rhel') - .with(command: 'yum-config-manager --enable codeready-builder-for-rhel-8-rhui-rpms') + .with(command: "yum-config-manager --enable codeready-builder-for-rhel-#{version.to_i}-rhui-rpms") end when 'ubuntu' @@ -82,8 +82,14 @@ def self.setup(chef_run) end it 'enables powertools' do + case version + when '8' + powertool_name = "powertools" + when '9' + powertool_name = "crb" + end is_expected.to run_execute('yum-config-manager-powertools') - .with(command: 'yum-config-manager --enable powertools') + .with(command: "yum-config-manager --enable #{powertool_name}") end it 'skips unavailable repos' do diff --git a/cookbooks/aws-parallelcluster-slurm/kitchen.slurm-install.yml b/cookbooks/aws-parallelcluster-slurm/kitchen.slurm-install.yml index 1368a9514e..8fb64afd7b 100644 --- a/cookbooks/aws-parallelcluster-slurm/kitchen.slurm-install.yml +++ b/cookbooks/aws-parallelcluster-slurm/kitchen.slurm-install.yml @@ -25,6 +25,8 @@ suites: - /tag:install_jwt_dependencies/ attributes: resource: jwt_dependencies + dependencies: + - resource:package_repos - name: mysql_client run_list: - recipe[aws-parallelcluster-tests::setup] diff --git a/cookbooks/aws-parallelcluster-slurm/libraries/helpers.rb b/cookbooks/aws-parallelcluster-slurm/libraries/helpers.rb index 1b384c5a3e..f52d13bc60 100644 --- a/cookbooks/aws-parallelcluster-slurm/libraries/helpers.rb +++ b/cookbooks/aws-parallelcluster-slurm/libraries/helpers.rb @@ -65,7 +65,7 @@ def enable_munge_service action %i(enable start) retries 5 retry_delay 10 - end + end unless on_docker? end def setup_munge_head_node diff --git a/cookbooks/aws-parallelcluster-slurm/resources/dns_domain/dns_domain_redhat8.rb b/cookbooks/aws-parallelcluster-slurm/resources/dns_domain/dns_domain_redhat8.rb index 2eaf69184d..3cb10a8d8f 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/dns_domain/dns_domain_redhat8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/dns_domain/dns_domain_redhat8.rb @@ -9,7 +9,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :dns_domain, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_dns_domain_common' diff --git a/cookbooks/aws-parallelcluster-slurm/resources/dns_domain/dns_domain_rocky8.rb b/cookbooks/aws-parallelcluster-slurm/resources/dns_domain/dns_domain_rocky8.rb index 98c1109ef6..4ad36e1856 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/dns_domain/dns_domain_rocky8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/dns_domain/dns_domain_rocky8.rb @@ -9,7 +9,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :dns_domain, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_dns_domain_common' diff --git a/cookbooks/aws-parallelcluster-slurm/resources/jwt_dependencies/jwt_dependencies_redhat8.rb b/cookbooks/aws-parallelcluster-slurm/resources/jwt_dependencies/jwt_dependencies_redhat8.rb index 7c824d876f..13fbee4c91 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/jwt_dependencies/jwt_dependencies_redhat8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/jwt_dependencies/jwt_dependencies_redhat8.rb @@ -9,7 +9,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :jwt_dependencies, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_jwt_dependencies_common' diff --git a/cookbooks/aws-parallelcluster-slurm/resources/jwt_dependencies/jwt_dependencies_rocky8.rb b/cookbooks/aws-parallelcluster-slurm/resources/jwt_dependencies/jwt_dependencies_rocky8.rb index 959e4736dc..45b17aac11 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/jwt_dependencies/jwt_dependencies_rocky8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/jwt_dependencies/jwt_dependencies_rocky8.rb @@ -9,7 +9,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :jwt_dependencies, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_jwt_dependencies_common' diff --git a/cookbooks/aws-parallelcluster-slurm/resources/munge/munge_redhat8.rb b/cookbooks/aws-parallelcluster-slurm/resources/munge/munge_redhat8.rb index 49dfe86665..d808f519cc 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/munge/munge_redhat8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/munge/munge_redhat8.rb @@ -9,7 +9,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :munge, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_munge_actions' diff --git a/cookbooks/aws-parallelcluster-slurm/resources/munge/munge_rocky8.rb b/cookbooks/aws-parallelcluster-slurm/resources/munge/munge_rocky8.rb index 6d8e10b91c..2d0f57245b 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/munge/munge_rocky8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/munge/munge_rocky8.rb @@ -9,7 +9,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :munge, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_munge_actions' diff --git a/cookbooks/aws-parallelcluster-slurm/resources/munge/partial/_munge_actions.rb b/cookbooks/aws-parallelcluster-slurm/resources/munge/partial/_munge_actions.rb index 4f98bb63d1..b97e037eaf 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/munge/partial/_munge_actions.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/munge/partial/_munge_actions.rb @@ -48,7 +48,6 @@ action_purge_packages action_download_source_code action_compile_and_install - action_update_init_script action_set_user_and_group action_create_required_directories } @@ -93,19 +92,6 @@ end end -action :update_init_script do - # Updated munge init script for Amazon Linux - template '/etc/init.d/munge' do - source 'munge/munge-init.erb' - cookbook 'aws-parallelcluster-slurm' - owner 'root' - group 'root' - variables(munge_user: munge_user, - munge_group: munge_group) - mode '0755' - end -end - action :set_user_and_group do # Setup munge group group munge_group do diff --git a/cookbooks/aws-parallelcluster-slurm/resources/munge_key_manager.rb b/cookbooks/aws-parallelcluster-slurm/resources/munge_key_manager.rb index d0c93b8469..b77cbaef15 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/munge_key_manager.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/munge_key_manager.rb @@ -29,7 +29,7 @@ def restart_munge_service action :restart retries 5 retry_delay 10 - end + end unless on_docker? end def enable_munge_service @@ -38,7 +38,7 @@ def enable_munge_service action :enable retries 5 retry_delay 10 - end + end unless on_docker? end def share_munge_key_to_dir(shared_dir) diff --git a/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/mysql_client_redhat8.rb b/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/mysql_client_redhat8.rb index de3d28c968..2469b0f7ee 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/mysql_client_redhat8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/mysql_client_redhat8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :mysql_client, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_common' diff --git a/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/mysql_client_rocky8.rb b/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/mysql_client_rocky8.rb index da09f09e1a..337858a44a 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/mysql_client_rocky8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/mysql_client_rocky8.rb @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and limitations under the License. provides :mysql_client, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_common' diff --git a/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/partial/_common.rb b/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/partial/_common.rb index 5452f94bb0..c1fe9e3955 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/partial/_common.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/partial/_common.rb @@ -35,11 +35,11 @@ action_class do def package_version - "8.0.31-1" + "8.0.36-1" end def package_source_version - "8.0.31" + "8.0.36" end def package_filename diff --git a/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/partial/_setup_rhel_based.rb b/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/partial/_setup_rhel_based.rb index 4cd5d3aa14..472f5bfc24 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/partial/_setup_rhel_based.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/mysql_client/partial/_setup_rhel_based.rb @@ -46,7 +46,11 @@ action_class do def package_platform - arm_instance? ? "el/7/aarch64" : "el/7/x86_64" + platform_version = node['platform_version'].to_i + if platform_version == 2 + platform_version = 7 + end + arm_instance? ? "el/#{platform_version}/aarch64" : "el/#{platform_version}/x86_64" end def repository_packages diff --git a/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_redhat8.rb b/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_redhat8.rb index aa6dd0abde..d448c24d6d 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_redhat8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_redhat8.rb @@ -9,11 +9,11 @@ # See the License for the specific language governing permissions and limitations under the License. provides :slurm_dependencies, platform: 'redhat' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_slurm_dependencies_common' def dependencies - %w(json-c-devel http-parser-devel perl-Switch lua-devel) + %w(json-c-devel http-parser-devel lua-devel perl dbus-devel) end diff --git a/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_rocky8.rb b/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_rocky8.rb index a4f91a6b0e..e30c36798f 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_rocky8.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_rocky8.rb @@ -9,11 +9,11 @@ # See the License for the specific language governing permissions and limitations under the License. provides :slurm_dependencies, platform: 'rocky' do |node| - node['platform_version'].to_i == 8 + node['platform_version'].to_i >= 8 end use 'partial/_slurm_dependencies_common' def dependencies - %w(json-c-devel http-parser-devel perl-Switch lua-devel) + %w(json-c-devel http-parser-devel lua-devel perl dbus-devel) end diff --git a/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/mysql_client_spec.rb b/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/mysql_client_spec.rb index dbf8b78902..4680cbf2d0 100644 --- a/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/mysql_client_spec.rb +++ b/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/mysql_client_spec.rb @@ -23,18 +23,23 @@ def self.validate(chef_run) %w(x86_64 aarch64).each do |architecture| context "on #{platform}#{version} #{architecture}" do cached(:source_dir) { 'SOURCE_DIR' } - cached(:package_source_version) { '8.0.31' } - cached(:package_version) { '8.0.31-1' } + cached(:package_source_version) { '8.0.36' } + cached(:package_version) { '8.0.36-1' } cached(:package_filename) { "mysql-community-client-#{package_version}.tar.gz" } cached(:s3_url) { 's3://url' } cached(:package_platform) do + platform_version = if version.to_i == 2 + 7 + else + version.to_i + end if architecture == 'aarch64' - 'el/7/aarch64' + "el/#{platform_version}/aarch64" elsif architecture == 'x86_64' if platform == 'ubuntu' "ubuntu/${version}/x86_64" else - 'el/7/x86_64' + "el/#{platform_version}/x86_64" end else pending "unsupported architecture #{architecture}" diff --git a/cookbooks/aws-parallelcluster-slurm/templates/default/munge/munge-init.erb b/cookbooks/aws-parallelcluster-slurm/templates/default/munge/munge-init.erb deleted file mode 100644 index b2264ec3d3..0000000000 --- a/cookbooks/aws-parallelcluster-slurm/templates/default/munge/munge-init.erb +++ /dev/null @@ -1,570 +0,0 @@ -#!/bin/sh -############################################################################### -# Written by Chris Dunlap . -# Copyright (C) 2007-2013 Lawrence Livermore National Security, LLC. -# Copyright (C) 2002-2007 The Regents of the University of California. -# UCRL-CODE-155910. -############################################################################### -# chkconfig: - 66 33 -# description: MUNGE Uid 'N' Gid Emporium authentication service -############################################################################### -### BEGIN INIT INFO -# Provides: munge -# Required-Start: $local_fs $remote_fs $network $time -# Required-Stop: $local_fs $remote_fs -# Should-Start: $named $syslog -# Should-Stop: $named $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: MUNGE Uid 'N' Gid Emporium authentication service -# Description: MUNGE (MUNGE Uid 'N' Gid Emporium) is a highly scalable -# authentication service for creating and validating -# credentials. -### END INIT INFO -############################################################################### - -unset SERVICE_NAME DAEMON_EXEC DAEMON_ARGS CONFIG PIDFILE NICE USER GROUP \ - SIGHUP_RELOAD VARRUNDIR - -prefix="/usr" -exec_prefix="${prefix}" -sbindir="${exec_prefix}/sbin" -sysconfdir="/etc" -localstatedir="/var" - -SERVICE_NAME="MUNGE" -DAEMON_EXEC="$sbindir/munged" -#DAEMON_ARGS= -#CONFIG=#_NOT_SUPPORTED_# -PIDFILE="$localstatedir/run/munge/munged.pid" -#NICE= -USER=<%= @munge_user %> -GROUP=<%= @munge_group %> -#SIGHUP_RELOAD=#_NOT_SUPPORTED_# -VARRUNDIR="$localstatedir/run/munge" - -############################################################################### - -service_init () -{ -# Determine the system type and initialize the environment. -# -# Note that the shell positional parameters must be preserved when calling -# this function in order for SuSE to initialize its environment properly. -## - PATH=/sbin:/usr/sbin:/bin:/usr/bin - INIT_NAME="`basename \"$0\" .init | sed 's/^[SK][0-9][0-9]*//'`" - DAEMON_NAME="`basename \"$DAEMON_EXEC\"`" - SIGTERM_TIMEOUT="3" - STATUS=0 - - # Read configuration defaults to override variables: - # $CONFIG, $DAEMON_ARGS, $PIDFILE, $USER, $NICE, $SIGHUP_RELOAD - ## - for dir in "$sysconfdir/default" "$sysconfdir/sysconfig"; do - [ -r "$dir/$INIT_NAME" ] && . "$dir/$INIT_NAME" - done - [ -z "$DAEMON_ARGS" -a -n "$OPTIONS" ] && DAEMON_ARGS="$OPTIONS" - [ "`id | sed 's/^uid=\([0-9]*\).*/\1/'`" -ne 0 ] && unset USER - expr -- "$NICE" : '[0-9]*$' >/dev/null 2>&1 && NICE="+$NICE" - [ -n "$SIGHUP_RELOAD" -a "$SIGHUP_RELOAD" != 0 ] \ - && RELOAD=1 || unset RELOAD - - if [ -f /etc/debian_version -a -x /sbin/start-stop-daemon ]; then - SYSTEM="DEBIAN" - [ -x "$DAEMON_EXEC" ] || exit 0 # pkg removed but not purged - [ -r /etc/default/rcS ] && . /etc/default/rcS - [ -r /lib/init/vars.sh ] && . /lib/init/vars.sh - [ -r /lib/lsb/init-functions ] && . /lib/lsb/init-functions - elif [ -f /etc/redhat-release -a -r /etc/rc.d/init.d/functions ]; then - SYSTEM="REDHAT" - . /etc/rc.d/init.d/functions - RH_LOCK="/var/lock/subsys/$INIT_NAME" - elif grep -q Amazon /etc/system-release; then - SYSTEM="REDHAT" - . /etc/rc.d/init.d/functions - RH_LOCK="/var/lock/subsys/$INIT_NAME" - elif [ -f /etc/SuSE-release -a -r /etc/rc.status ]; then - SYSTEM="SUSE" - . /etc/rc.status - rc_reset - elif [ -r /lib/lsb/init-functions ]; then - SYSTEM="LSB" - . /lib/lsb/init-functions - else - SYSTEM="OTHER" - fi - - # Exit if the package has been removed. - ## - [ -x "$DAEMON_EXEC" ] || exit 5 # LSB: program not installed - - # Exit if the configuration has been removed. - ## - [ -z "$CONFIG" -o -r "$CONFIG" ] || exit 6 # LSB: program not configured -} - -service_fini () -{ -# Return the exit status. -## - case $SYSTEM in - SUSE) - rc_exit - ;; - DEBIAN|REDHAT|LSB|*) - exit $STATUS - ;; - esac -} - -service_start () -{ -# Start the service. -# -# Required by LSB, where running "start" on a service already running should be -# considered successful. -## - log_init "Starting $SERVICE_NAME" "$DAEMON_NAME" - - if [ -n "$VARRUNDIR" -a ! -d "$VARRUNDIR" ]; then - mkdir -m 755 -p "$VARRUNDIR" - [ -n "$USER" ] && chown "$USER" "$VARRUNDIR" - [ -n "$GROUP" ] && chgrp "$GROUP" "$VARRUNDIR" - fi - - case $SYSTEM in - DEBIAN) - if $0 status >/dev/null 2>&1; then - STATUS=0 - else - ERRMSG=`start-stop-daemon --start --quiet \ - ${NICE:+"--nicelevel"} ${NICE:+"$NICE"} \ - ${USER:+"--chuid"} ${USER:+"$USER"} \ - ${PIDFILE:+"--pidfile"} ${PIDFILE:+"$PIDFILE"} \ - --exec "$DAEMON_EXEC" -- $DAEMON_ARGS 2>&1` - STATUS=$? - fi - ;; - REDHAT) - if $0 status >/dev/null 2>&1; then - STATUS=0 - else - daemon ${NICE:+"$NICE"} ${USER:+"--user"} ${USER:+"$USER"} \ - "$DAEMON_EXEC" $DAEMON_ARGS - STATUS=$? - fi - [ $STATUS -eq 0 ] && touch "$RH_LOCK" >/dev/null 2>&1 - ;; - SUSE) - ERRMSG=`startproc ${NICE:+"-n"} ${NICE:+"$NICE"} \ - ${USER:+"-u"} ${USER:+"$USER"} \ - ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} \ - "$DAEMON_EXEC" $DAEMON_ARGS 2>&1` - rc_status -v - STATUS=$? - ;; - LSB) - if [ -n "$USER" ]; then - ERRMSG=`su "$USER" -c "/sbin/start_daemon \ - ${NICE:+\"-n\"} ${NICE:+\"$NICE\"} \ - ${PIDFILE:+\"-p\"} ${PIDFILE:+\"$PIDFILE\"} \ - \"$DAEMON_EXEC\" $DAEMON_ARGS" 2>&1` - else - ERRMSG=`start_daemon ${NICE:+"-n"} ${NICE:+"$NICE"} \ - ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} \ - "$DAEMON_EXEC" $DAEMON_ARGS 2>&1` - fi - STATUS=$? - ;; - *) - if $0 status >/dev/null 2>&1; then - STATUS=0 - else - [ -n "$NICE" ] && nice="nice -n $NICE" - if [ -n "$USER" ]; then - ERRMSG=`su "$USER" -c "$nice \"$DAEMON_EXEC\" $DAEMON_ARGS" 2>&1` - else - ERRMSG=`$nice "$DAEMON_EXEC" $DAEMON_ARGS 2>&1` - fi - STATUS=$? - fi - ;; - esac - log_fini "$STATUS" "$ERRMSG" -} - -service_stop () -{ -# Stop the service. -# -# Required by LSB, where running "stop" on a service already stopped or not -# running should be considered successful. -## - log_init "Stopping $SERVICE_NAME" "$DAEMON_NAME" - case $SYSTEM in - DEBIAN) - if ! $0 status >/dev/null 2>&1; then - STATUS=0 - else - start-stop-daemon --stop --quiet \ - ${PIDFILE:+"--pidfile"} ${PIDFILE:+"$PIDFILE"} \ - --name "$DAEMON_NAME" ${SIGTERM_TIMEOUT:+"--retry"} \ - ${SIGTERM_TIMEOUT:+"$SIGTERM_TIMEOUT"} >/dev/null 2>&1 - STATUS=$? - fi - ;; - REDHAT) - if ! $0 status >/dev/null 2>&1; then - STATUS=0 - else - killproc ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} \ - ${SIGTERM_TIMEOUT:+"-d"} ${SIGTERM_TIMEOUT:+"$SIGTERM_TIMEOUT"} \ - "$DAEMON_EXEC" - STATUS=$? - fi - [ $STATUS -eq 0 ] && rm -f "$RH_LOCK" >/dev/null 2>&1 - ;; - SUSE) - killproc ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} \ - ${SIGTERM_TIMEOUT:+"-t"} ${SIGTERM_TIMEOUT:+"$SIGTERM_TIMEOUT"} \ - "$DAEMON_EXEC" - rc_status -v - ;; - LSB) - killproc ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} "$DAEMON_EXEC" - STATUS=$? - ;; - *) - signal_process "$DAEMON_EXEC" - rc=$? - [ $rc -eq 0 -o $rc -eq 2 ] && STATUS=0 || STATUS=1 - ;; - esac - log_fini "$STATUS" - [ -f "$PIDFILE" ] && rm -f "$PIDFILE" -} - -service_restart () -{ -# Stop and restart the service if it is already running; -# otherwise, start the service. -# -# Required by LSB, where running "restart" on a service already stopped or not -# running should be considered successful. -## - if $0 status >/dev/null 2>&1; then - $0 stop && $0 start - else - $0 start - fi - - case $SYSTEM in - SUSE) - rc_status - ;; - DEBIAN|REDHAT|LSB|*) - STATUS=$? - ;; - esac -} - -service_try_restart () -{ -# Restart the service if it is already running. -# -# Optional for LSB, where running "try-restart" on a service already stopped or -# not running should be considered successful. -# Also known as "condrestart" by RedHat. -## - case $SYSTEM in - REDHAT) - [ -f "$RH_LOCK" ] && $0 restart || : - STATUS=$? - ;; - SUSE) - $0 status >/dev/null 2>&1 && $0 restart || rc_reset - rc_status - ;; - DEBIAN|LSB|*) - $0 status >/dev/null 2>&1 && $0 restart || : - STATUS=$? - ;; - esac -} - -service_reload () -{ -# Reload the configuration without stopping and restarting the service. -# -# Optional for LSB. -## - [ -z "$RELOAD" ] && STATUS=3 # LSB: unimplemented feature - - log_init "Reloading $SERVICE_NAME" "$DAEMON_NAME" - case $SYSTEM in - DEBIAN) - if [ -n "$RELOAD" ]; then - start-stop-daemon --stop --quiet --signal HUP \ - ${PIDFILE:+"--pidfile"} ${PIDFILE:+"$PIDFILE"} \ - --name "$DAEMON_NAME" >/dev/null 2>&1 - STATUS=$? - fi - ;; - REDHAT) - if [ -n "$RELOAD" ]; then - killproc ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} "$DAEMON_EXEC" -HUP - STATUS=$? - else - echo_failure - fi - ;; - SUSE) - if [ -n "$RELOAD" ]; then - killproc -HUP ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} "$DAEMON_EXEC" - else - rc_failed $STATUS - fi - rc_status -v - ;; - LSB) - if [ -n "$RELOAD" ]; then - killproc ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} "$DAEMON_EXEC" -HUP - STATUS=$? - fi - ;; - *) - if [ -n "$RELOAD" ]; then - signal_process "$DAEMON_EXEC" "HUP" - STATUS=$? - fi - ;; - esac - log_fini "$STATUS" -} - -service_force_reload () -{ -# Reload the configuration if the service supports this; -# otherwise, restart the service if it is already running. -# -# Required by LSB, where running "force-reload" on a service already stopped or -# not running should be considered successful. -## - if [ -n "$RELOAD" ]; then - $0 reload - else - $0 try-restart - fi - - case $SYSTEM in - SUSE) - rc_status - ;; - DEBIAN|REDHAT|LSB|*) - STATUS=$? - ;; - esac -} - -service_status () -{ -# Print the current status of the service. -# -# Required by LSB. -## - case $SYSTEM in - REDHAT) - status ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} "$DAEMON_EXEC" - STATUS=$? - ;; - SUSE) - printf "Checking for service $SERVICE_NAME: " - checkproc ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} "$DAEMON_EXEC" - rc_status -v - ;; - LSB) - printf "Checking status of $SERVICE_NAME: " - pids=`pidofproc ${PIDFILE:+"-p"} ${PIDFILE:+"$PIDFILE"} \ - "$DAEMON_EXEC" 2>/dev/null` - STATUS=$? - if [ $STATUS -eq 0 -a -n "$pids" ]; then - echo "running." - elif [ $STATUS -ne 0 -a -s "$PIDFILE" ]; then - echo "dead." - else - echo "stopped." - fi - ;; - DEBIAN|*) - printf "Checking status of $SERVICE_NAME: " - pids=`query_pids "$DAEMON_EXEC" "$PIDFILE"` - rc=$? - if [ $rc -eq 0 -a -n "$pids" ]; then - echo "running." - STATUS=0 # LSB: program is running - elif [ $rc -ne 0 -a -s "$PIDFILE" ]; then - echo "dead." - STATUS=1 # LSB: program is dead & pidfile exists - elif [ $rc -ne 0 ]; then - echo "stopped." - STATUS=3 # LSB: program is not running - else - echo "unknown." - STATUS=4 # LSB: program status unknown - fi - ;; - esac -} - -query_pids () -{ -# Writes the matching PIDs to stdout. -# Returns 0 on success (ie, pids found). -## - PROCNAME="$1" - PIDFILE="$2" - - if type pgrep >/dev/null 2>&1; then - pids=`pgrep -d ' ' -x "\`basename \"$PROCNAME\"\`" 2>/dev/null` - rc=$? - elif type pidof >/dev/null 2>&1; then - pids=`pidof -o $$ -x "$PROCNAME" 2>/dev/null` - rc=$? - else - pids=`(ps awx -o pid -o command || ps -e -f -o pid -o args) 2>/dev/null \ - | tail +2 | egrep "( |/)$PROCNAME( |$)" | grep -v egrep \ - | sed 's/ *\([0-9]*\).*/\1/' | sort -n | tr '\012' ' '` - [ -n "$pids" ] && rc=0 || rc=1 - fi - - unset pids_running - if [ -n "$pids" -a -r "$PIDFILE" ]; then - read pid_line < "$PIDFILE" - for pid in $pid_line; do - expr -- "$pid" : '[0-9]*$' >/dev/null 2>&1 \ - && expr -- " $pids " : ".* $pid .*" >/dev/null 2>&1 \ - && pids_running="$pids_running $pid" - done - [ -n "$pids_running" ] && pids=$pids_running - fi - - echo $pids - return $rc -} - -signal_process () -{ -# Returns 0 on success, 1 if kill failed, 2 if PROCNAME is not running. -## - PROCNAME="$1" - SIGNUM="$2" - - pids=`query_pids "$DAEMON_EXEC" "$PIDFILE"` - [ $? -ne 0 -o -z "$pids" ] && return 2 - - kill ${SIGNUM:+"-$SIGNUM"} $pids >/dev/null 2>&1 - [ $? -ne 0 ] && return 1 - [ -n "$SIGNUM" ] && return 0 - - sleep 1 - pids=`query_pids "$DAEMON_EXEC" "$PIDFILE"` - [ $? -ne 0 -o -z "$pids" ] && return 0 - [ -z "$SIGTERM_TIMEOUT" ] && return 1 - - sleep "$SIGTERM_TIMEOUT" - kill -KILL $pids >/dev/null 2>&1 - pids=`query_pids "$DAEMON_EXEC" "$PIDFILE"` - [ $? -ne 0 -o -z "$pids" ] && return 0 - return 1 -} - -log_init () -{ -# Output informational message at beginning of action. -## - MESSAGE="$1" - PROCNAME="$2" - - case $SYSTEM in - DEBIAN) - if [ "$VERBOSE" != no ]; then - if type log_daemon_msg >/dev/null 2>&1; then - log_daemon_msg "$MESSAGE" "$PROCNAME" - else - printf "$MESSAGE: $PROCNAME" - fi - fi - ;; - REDHAT|SUSE|LSB|*) - printf "$MESSAGE: $PROCNAME" - ;; - esac -} - -log_fini () -{ -# Output informational/error message at end of action. -## - STATUS="$1" - ERRMSG="$2" - - case $SYSTEM in - DEBIAN) - if [ "$VERBOSE" != no ]; then - if ( type log_end_msg && type log_failure_msg ) >/dev/null 2>&1; then - log_end_msg "$STATUS" - [ $STATUS -eq 0 -o -z "$ERRMSG" ] || log_failure_msg "$ERRMSG" - else - [ $STATUS -eq 0 ] && echo "." || echo " (failed)." - [ $STATUS -eq 0 -o -z "$ERRMSG" ] || echo "$ERRMSG" >&2 - fi - fi - ;; - REDHAT) - echo - ;; - SUSE) - [ $STATUS -eq 0 -o -z "$ERRMSG" ] || echo "$ERRMSG" >&2 - ;; - LSB|*) - [ $STATUS -eq 0 ] && echo "." || echo " (failed)." - [ $STATUS -eq 0 -o -z "$ERRMSG" ] || echo "$ERRMSG" >&2 - ;; - esac -} - -############################################################################### - -service_init "$@" - -case "$1" in - start) - service_start - ;; - stop) - service_stop - ;; - restart) - service_restart - ;; - try-restart|condrestart) - service_try_restart - ;; - reload) - service_reload - ;; - force-reload) - service_force_reload - ;; - status) - service_status - ;; - *) - echo "Usage: `basename \"$0\"`" \ - "(start|stop|restart|try-restart|reload|force-reload|status)" >&2 - exit 2 # LSB: invalid or excess argument(s) - ;; -esac - -service_fini diff --git a/cookbooks/aws-parallelcluster-slurm/test/controls/munge_spec.rb b/cookbooks/aws-parallelcluster-slurm/test/controls/munge_spec.rb index ea07167696..07251acf29 100644 --- a/cookbooks/aws-parallelcluster-slurm/test/controls/munge_spec.rb +++ b/cookbooks/aws-parallelcluster-slurm/test/controls/munge_spec.rb @@ -35,21 +35,6 @@ end end unless os_properties.redhat_on_docker? -control 'tag:install_munge_init_script_configured' do - title 'Check munge init script is configured with the proper user and group' - - describe file("/etc/init.d/munge") do - it { should exist } - its('mode') { should cmp '0755' } - its('owner') { should eq 'root' } - its('group') { should eq 'root' } - its('content') do - should match("USER=#{munge_user}") - should match("GROUP=#{munge_group}") - end - end unless os_properties.redhat_on_docker? -end - control 'tag:install_munge_folders_created' do title 'Munge folder have been created' diff --git a/cookbooks/aws-parallelcluster-slurm/test/controls/mysql_client_spec.rb b/cookbooks/aws-parallelcluster-slurm/test/controls/mysql_client_spec.rb index 35d403e1b8..0a1f48de2a 100644 --- a/cookbooks/aws-parallelcluster-slurm/test/controls/mysql_client_spec.rb +++ b/cookbooks/aws-parallelcluster-slurm/test/controls/mysql_client_spec.rb @@ -15,7 +15,10 @@ mysql_packages = [] if os.redhat? mysql_packages.concat %w(mysql-community-client-plugins mysql-community-common - mysql-community-devel mysql-community-libs mysql-community-libs-compat) + mysql-community-devel mysql-community-libs) + if os_properties.alinux2? || os_properties.centos7? + mysql_packages.concat %w(mysql-community-libs-compat) + end elsif os_properties.ubuntu2004? || os_properties.ubuntu2204? mysql_packages.concat %w(libmysqlclient-dev libmysqlclient21) else @@ -29,7 +32,7 @@ mysql_packages.each do |pkg| describe package(pkg) do it { should be_installed } - its('version') { should match /^8.0.31-/ } unless ubuntu + its('version') { should match /^8.0.36-/ } unless ubuntu end end end @@ -45,7 +48,7 @@ its('content') do should eq %(You can get MySQL source code here: -https://#{node['cluster']['region']}-aws-parallelcluster.s3.#{node['cluster']['region']}.amazonaws.com/archives/source/mysql-8.0.31.tar.gz +https://#{node['cluster']['region']}-aws-parallelcluster.s3.#{node['cluster']['region']}.amazonaws.com/archives/source/mysql-8.0.36.tar.gz ) end end diff --git a/cookbooks/aws-parallelcluster-slurm/test/controls/slurm_dependencies_spec.rb b/cookbooks/aws-parallelcluster-slurm/test/controls/slurm_dependencies_spec.rb index e2fed2f198..e531ebcfe6 100644 --- a/cookbooks/aws-parallelcluster-slurm/test/controls/slurm_dependencies_spec.rb +++ b/cookbooks/aws-parallelcluster-slurm/test/controls/slurm_dependencies_spec.rb @@ -24,8 +24,12 @@ def lua_devel_package packages = [] if os.redhat? - # Skipping redhat on docker since ubi-appstream repo is not aligned with the main repo - packages.concat %w(json-c-devel http-parser-devel perl-Switch) unless os_properties.redhat_on_docker? + if os_properties.alinux2? || os_properties.centos7? + # Skipping redhat on docker since ubi-appstream repo is not aligned with the main repo + packages.concat %w(json-c-devel http-parser-devel perl-Switch) unless os_properties.redhat_on_docker? + else + packages.concat %w(json-c-devel http-parser-devel perl dbus-devel) unless os_properties.redhat_on_docker? + end elsif os.debian? packages.concat %w(libjson-c-dev libhttp-parser-dev libswitch-perl) else diff --git a/cookbooks/aws-parallelcluster-tests/recipes/docker_mock.rb b/cookbooks/aws-parallelcluster-tests/recipes/docker_mock.rb index 7d7085a0ec..ae7f8bc390 100644 --- a/cookbooks/aws-parallelcluster-tests/recipes/docker_mock.rb +++ b/cookbooks/aws-parallelcluster-tests/recipes/docker_mock.rb @@ -28,15 +28,10 @@ #!/bin/bash echo "service 0:off 1:off 2:on 3:on 4:on 5:on 6:off" ) + mode '0744' end %w( - /etc/init.d/rpc-statd - /etc/init.d/rpc-statd.service - /etc/init.d/nfs-idmapd - /etc/init.d/nfs-client.target - /etc/init.d/nfs-config.service - /etc/init.d/nfs-kernel-server.service /sbin/service /usr/local/bin/udevadm /usr/local/sbin/sysctl @@ -63,6 +58,8 @@ if redhat_on_docker? package 'openssh-clients' + package 'python3' + package 'python3-pip' # Mock python environment package 'python39' diff --git a/kitchen.docker.yml b/kitchen.docker.yml index a66d1feb9d..a9acab8233 100644 --- a/kitchen.docker.yml +++ b/kitchen.docker.yml @@ -82,3 +82,24 @@ platforms: # Since the kernel version of the docker images is not in the expected pattern, set a fake kernel value to permit to install Lustre on docker. # Specific kernel versions are not compatible with Lustre. kernel_release: '4.18.0-477.10.1.el8_8.fake-value' + - name: rhel9 + driver: + image: <% if ENV['KITCHEN_RHEL9_IMAGE'] %> <%= ENV['KITCHEN_RHEL9_IMAGE'] %> <% else %>registry.access.redhat.com/ubi9/ubi <% end %> + intermediate_instructions: + - RUN chmod +t /tmp + - RUN yum install libxcrypt-compat -y + attributes: + cluster: + base_os: rhel9 + # Since the kernel version of the docker images is not in the expected pattern, set a fake kernel value to permit to install Lustre on docker. + # Specific kernel versions are not compatible with Lustre. + kernel_release: '5.14.0-362.13.1.el9_3.fake-value' # Use 477 version to match 8.8 kernel version available on docker + - name: rocky9 + driver: + image: <% if ENV['KITCHEN_ROCKY9_IMAGE'] %> <%= ENV['KITCHEN_ROCKY9_IMAGE'] %> <% else %> dokken/rockylinux-9 <% end %> + attributes: + cluster: + base_os: rocky9 + # Since the kernel version of the docker images is not in the expected pattern, set a fake kernel value to permit to install Lustre on docker. + # Specific kernel versions are not compatible with Lustre. + kernel_release: '5.14.0-362.8.1.el9_3.fake-value' diff --git a/kitchen.ec2.yml b/kitchen.ec2.yml index 366dcf4456..e0fc9e481a 100644 --- a/kitchen.ec2.yml +++ b/kitchen.ec2.yml @@ -154,6 +154,58 @@ platforms: attributes: cluster: base_os: rocky8 + - name: rhel9 + driver_plugin: ec2 + driver: + <% if ENV['KITCHEN_RHEL9_AMI'] %> + # Use the RedHat 8 AMI most similar to the base AMI used to build the ParallelCluster image + image_id: <%= ENV['KITCHEN_RHEL9_AMI'] %> + <% else %> + image_search: + name: <% if ENV['KITCHEN_PHASE']=='install' %>RHEL-9.3.*_HVM*<% else %><%= pcluster_prefix %>-rhel9-hvm-*<% end %> + architecture: <%= ENV['KITCHEN_ARCHITECTURE'] %> + <% end %> + block_device_mappings: + - device_name: /dev/sda1 + ebs: + volume_size: <% if (ENV['KITCHEN_VOLUME_SIZE'] || '') == '' %> 40 <% else %> <%= ENV['KITCHEN_VOLUME_SIZE'] %> <% end %> + volume_type: gp2 + delete_on_termination: true + <% %w(a b c d e f g h i j k l m n o p q r s t u v w x).each_with_index do | c, i | %> + - device_name: /dev/xvdb<%= c %> + virtual_name: ephemeral<%= i %> + <% end %> + transport: + username: ec2-user + attributes: + cluster: + base_os: rhel9 + - name: rocky9 + driver_plugin: ec2 + driver: + <% if ENV['KITCHEN_ROCKY9_AMI'] %> + # Use the Rocky Linux 8 AMI most similar to the base AMI used to build the ParallelCluster image + image_id: <%= ENV['KITCHEN_ROCKY9_AMI'] %> + <% else %> + image_search: + name: <% if ENV['KITCHEN_PHASE']=='install' %>Rocky-9-EC2-Base-9.3*<% else %><%= pcluster_prefix %>-rocky9-hvm-*<% end %> + architecture: <%= ENV['KITCHEN_ARCHITECTURE'] %> + <% end %> + block_device_mappings: + - device_name: /dev/sda1 + ebs: + volume_size: <% if (ENV['KITCHEN_VOLUME_SIZE'] || '') == '' %> 40 <% else %> <%= ENV['KITCHEN_VOLUME_SIZE'] %> <% end %> + volume_type: gp2 + delete_on_termination: true + <% %w(a b c d e f g h i j k l m n o p q r s t u v w x).each_with_index do | c, i | %> + - device_name: /dev/xvdb<%= c %> + virtual_name: ephemeral<%= i %> + <% end %> + transport: + username: rocky + attributes: + cluster: + base_os: rocky9 - name: centos7 driver_plugin: ec2 driver: diff --git a/test/unit/ec2_udev_rules/test_ec2_dev_2_volid.py b/test/unit/ec2_udev_rules/test_ec2_dev_2_volid.py index b97c759468..3fc49500d4 100644 --- a/test/unit/ec2_udev_rules/test_ec2_dev_2_volid.py +++ b/test/unit/ec2_udev_rules/test_ec2_dev_2_volid.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and limitations under the License. import pytest from assertpy import assert_that -from ec2_dev_2_volid import adapt_device_name, get_device_volume_id, get_imdsv2_token, validate_device_name +from ec2_dev_2_volid import adapt_device_name, validate_device_name @pytest.mark.parametrize( @@ -33,18 +33,6 @@ def test_validate_device_name(device_name, raises): assert_that(validate_device_name(device_name)).is_true() -@pytest.mark.parametrize( - ("status_code", "content", "expected_value"), - [(200, {"key": "value"}, {"X-aws-ec2-metadata-token": {"key": "value"}}), (400, {"key": "value"}, {})], -) -def test_get_imdsv2_token(mocker, status_code, content, expected_value): - mock = mocker.Mock() - mocker.patch("requests.put", mock) - mock.return_value.status_code = status_code - mock.return_value.content = content - assert_that(get_imdsv2_token()).is_equal_to(expected_value) - - @pytest.mark.parametrize( ("dev", "expected_name", "raises"), [ @@ -65,82 +53,3 @@ def test_adapt_device_name(mocker, dev, expected_name, raises, capsys): assert_that(e.value.code).is_equal_to(0) else: assert_that(adapt_device_name(dev)).matches(expected_name) - - -@pytest.fixture(name="ec2_mock") -def fixture_ec2_mock(mocker): - mock = mocker.MagicMock() - mocker.patch("boto3.client", mock) - return mock - - -@pytest.mark.parametrize( - ("dev", "block", "output_value"), - [ - ( - "/dev/sda1", - { - "InstanceId": "i-1234567890abcdef0", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "Status": "attached", - "DeleteOnTermination": True, - "VolumeId": "vol-049df61146c4d7901", - "AttachTime": "2013-05-17T22:42:34.000Z", - }, - }, - { - "DeviceName": "/dev/sdf", - "Ebs": { - "Status": "attached", - "DeleteOnTermination": False, - "VolumeId": "vol-049df61146c4d7901", - "AttachTime": "2013-09-10T23:07:00.000Z", - }, - }, - ], - }, - "vol-049df61146c4d7901", - ), - ( - "/dev/sda2", - { - "InstanceId": "i-1234567890abcdef0", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "Status": "attached", - "DeleteOnTermination": True, - "VolumeId": "vol-049df61146c4d7901", - "AttachTime": "2013-05-17T22:42:34.000Z", - }, - }, - { - "DeviceName": "/dev/sdf", - "Ebs": { - "Status": "attached", - "DeleteOnTermination": False, - "VolumeId": "vol-049df61146c4d7901", - "AttachTime": "2013-09-10T23:07:00.000Z", - }, - }, - ], - }, - SystemExit, - ), - ], -) -def test_get_device_volume_id(mocker, ec2_mock, dev, block, output_value): - mocker.patch("time.sleep", return_value=None) - ec2_mock.describe_instance_attribute.return_value = block - if output_value == SystemExit: - with pytest.raises(SystemExit) as e: - get_device_volume_id(ec2_mock, dev, 1) - assert_that(e.type).is_equal_to(SystemExit) - assert_that(e.value.code).is_equal_to(1) - else: - volume_id = get_device_volume_id(ec2_mock, dev, 1) - assert_that(volume_id).is_equal_to(output_value) diff --git a/test/unit/ec2_udev_rules/test_manage_volume.py b/test/unit/ec2_udev_rules/test_manage_volume.py index 9d61830ec2..42e77fc154 100644 --- a/test/unit/ec2_udev_rules/test_manage_volume.py +++ b/test/unit/ec2_udev_rules/test_manage_volume.py @@ -117,6 +117,8 @@ def test_attach_volume(mocker, volume_response, state, message, ec2_mock, capsys mocker.patch("time.sleep", return_value=None) mocker.patch("os.popen", mocker.mock_open(read_data="sdf")) mocker.patch("subprocess.check_output", return_value="xvda") + mocker.patch("os.makedirs") + mocker.patch("builtins.open", mocker.mock_open()) volume_response["State"] = state ec2_mock.attach_volume.return_value = volume_response