From 8ef43528a3932e1e94a60777f1e94afd2489612b Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Wed, 12 Mar 2025 09:24:28 -0400 Subject: [PATCH 1/3] [Test] Add Amazon Linux 2023 to spec tests. --- cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb b/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb index ff6af2da76..78a6e77e09 100644 --- a/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb +++ b/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb @@ -40,6 +40,7 @@ def for_oses(os_list) def for_all_oses [ %w(amazon 2), + %w(amazon 2023), %w(ubuntu 20.04), %w(ubuntu 22.04), %w(redhat 8), From 0b41c38b4d4e5b448c1d33ffd8145e0bb34a07a2 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Wed, 12 Mar 2025 09:30:14 -0400 Subject: [PATCH 2/3] [Test] Add Ubuntu 24.04 to spec tests. --- cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb b/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb index 78a6e77e09..27993e0270 100644 --- a/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb +++ b/cookbooks/aws-parallelcluster-shared/spec/spec_helper.rb @@ -43,6 +43,7 @@ def for_all_oses %w(amazon 2023), %w(ubuntu 20.04), %w(ubuntu 22.04), + %w(ubuntu 24.04), %w(redhat 8), %w(rocky 8), %w(redhat 9), From 94693dd1f0d71ea831448040123c26719a04d2dc Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Wed, 12 Mar 2025 12:31:16 -0400 Subject: [PATCH 3/3] [Test] Fix spec tests failures for Ubuntu24.04 and Amazon Linux 2023. --- .../spec/unit/resources/efa_spec.rb | 4 +- .../unit/resources/network_service_spec.rb | 43 ++++++++++++------- .../unit/resources/raid_mount_unmount_spec.rb | 2 +- .../spec/unit/resources/arm_pl_spec.rb | 9 +++- .../spec/unit/resources/dcv_spec.rb | 30 +++++++------ .../spec/unit/resources/gdrcopy_spec.rb | 8 ++++ .../unit/resources/install_packages_spec.rb | 2 +- .../spec/unit/resources/modules_spec.rb | 2 + .../spec/unit/resources/nvidia_dcgm_spec.rb | 4 +- .../spec/unit/resources/nvidia_driver_spec.rb | 39 +++++++++++------ .../spec/unit/resources/nvidia_repo_spec.rb | 4 +- .../spec/unit/resources/stunnel_spec.rb | 2 +- .../spec/unit/resources/package_repos_spec.rb | 11 +++-- .../spec/unit/resources/dns_domain_spec.rb | 8 ++-- .../spec/unit/resources/mysql_client_spec.rb | 2 + 15 files changed, 112 insertions(+), 58 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efa_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efa_spec.rb index a11e606ca6..bd48b7a21a 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efa_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efa_spec.rb @@ -53,9 +53,9 @@ def self.configure(chef_run) for_all_oses do |platform, version| context "on #{platform}#{version}" do cached(:prerequisites) do - if %(redhat rocky).include?(platform) + if %(redhat rocky).include?(platform) || platform == 'amazon' && version == '2023' %w(environment-modules libibverbs-utils librdmacm-utils rdma-core-devel) - elsif platform == 'amazon' + elsif platform == 'amazon' && version == '2' %w(environment-modules libibverbs-utils librdmacm-utils) else "environment-modules" diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/network_service_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/network_service_spec.rb index d778cfbc85..a7bc29b3ab 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/network_service_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/network_service_spec.rb @@ -27,19 +27,28 @@ def self.reload(chef_run) end cached(:node) { chef_run.node } cached(:network_service_name) do - { - 'amazon' => 'network', - 'centos' => 'network', - 'redhat' => 'NetworkManager', - 'rocky' => 'NetworkManager', - 'ubuntu' => 'systemd-resolved', - }[platform] + if platform == 'amazon' && version == '2' || platform == 'centos' + 'network' + elsif platform == 'amazon' && version == '2023' + 'systemd-networkd' + elsif platform == 'ubuntu' + 'systemd-resolved' + elsif %(redhat rocky).include?(platform) + 'NetworkManager' + else + raise "Cannot determine network_service_name: unrecognized platform #{platform}" + end end it "restarts network service" do is_expected.to restart_network_service('restart') + network_services_to_restart = if platform == 'amazon' && version == '2023' + [network_service_name, 'systemd-resolved'] + else + [network_service_name] + end - is_expected.to write_log("Restarting '#{network_service_name}' service, platform #{platform} '#{node['platform_version']}'") + is_expected.to write_log("Restarting '#{network_services_to_restart.join(' ')}' service, platform #{platform} '#{node['platform_version']}'") is_expected.to restart_service(network_service_name) .with(ignore_failure: true) @@ -56,13 +65,17 @@ def self.reload(chef_run) ConvergeNetworkService.reload(runner) end cached(:network_service_name) do - { - 'amazon' => 'network', - 'centos' => 'network', - 'redhat' => 'NetworkManager', - 'rocky' => 'NetworkManager', - 'ubuntu' => 'systemd-resolved', - }[platform] + if platform == 'amazon' && version == '2' || platform == 'centos' + 'network' + elsif platform == 'amazon' && version == '2023' + 'systemd-networkd' + elsif platform == 'ubuntu' + 'systemd-resolved' + elsif %(redhat rocky).include?(platform) + 'NetworkManager' + else + raise "Cannot determine network_service_name: unrecognized platform #{platform}" + end end it 'reloads network_service' do diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/raid_mount_unmount_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/raid_mount_unmount_spec.rb index f0600d3893..ad4f16ff78 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/raid_mount_unmount_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/raid_mount_unmount_spec.rb @@ -11,7 +11,7 @@ def wait_for_block_dev(_path) context "on #{platform}#{version}" do cached(:venv_path) { 'venv' } cached(:raid_superblock_version) do - %(redhat rocky).include?(platform) || "#{platform}#{version}" == 'ubuntu20.04' || "#{platform}#{version}" == 'ubuntu22.04' ? '1.2' : '0.90' + %(redhat rocky ubuntu).include?(platform) || "#{platform}#{version}" == 'amazon2023' ? '1.2' : '0.90' end cached(:chef_run) do runner = runner( diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb index a5c956879d..e9f0c1c2f2 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/arm_pl_spec.rb @@ -26,14 +26,19 @@ def self.setup(chef_run) when 'ubuntu' "Ubuntu-#{version}" when 'amazon' - "AmazonLinux-2" + if version == '2' + "AmazonLinux-2" + elsif version == '2023' + 'RHEL-9' + end else "RHEL-#{version}" end end cached(:gcc_major_minor_version) do - if platform == 'ubuntu' && version == '22.04' || version == '9' + case "#{platform}#{version}" + when 'amazon2023', 'ubuntu24.04', 'ubuntu22.04', 'redhat9', 'rocky9' '11.3' else '9.3' diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/dcv_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/dcv_spec.rb index 08d07adcfa..882c03da6d 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/dcv_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/dcv_spec.rb @@ -42,7 +42,8 @@ def self.nothing(chef_run) allow_any_instance_of(Object).to receive(:arm_instance?).and_return(true) end - if platform == 'ubuntu' && version.to_i == 20 + case "#{platform}#{version}" + when "amazon2023", "ubuntu20.04" it "is false" do expect(resource.dcv_supported?).to eq(false) end @@ -58,9 +59,10 @@ def self.nothing(chef_run) end context 'when not on arm' do - it "is true" do + is_supported = !("#{platform}#{version}" == 'amazon2023') + it "is #{is_supported}" do allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) - expect(resource.dcv_supported?).to eq(true) + expect(resource.dcv_supported?).to eq(is_supported) end end end @@ -142,8 +144,8 @@ def self.nothing(chef_run) expect(resource.xdcv).to eq("nice-xdcv_#{xdcv_version}_#{dcv_pkg_arch}.#{base_os}.deb") expect(resource.dcv_web_viewer).to eq("nice-dcv-web-viewer_#{dcv_webviewer_version}_#{dcv_pkg_arch}.#{base_os}.deb") expect(resource.dcv_gl).to eq("/nice-dcv-gl_#{dcv_gl_version}_#{dcv_pkg_arch}.#{base_os}.deb") - else - dcv_platform_version = platform == "amazon" ? "7" : version.to_i + elsif "#{platform}#{version}" != 'amazon2023' + dcv_platform_version = "#{platform}#{version}" == "amazon2" ? "7" : version.to_i dcv_platform_version_pkg = platform == "amazon" ? "amzn2" : "el" + version expect(resource.dcv_package).to eq("nice-dcv-#{dcv_version}-#{dcv_platform_version_pkg}-#{dcv_url_arch}") expect(resource.dcv_server).to eq("nice-dcv-server-#{dcv_server_version}.el#{dcv_platform_version}.#{dcv_url_arch}.rpm") @@ -479,14 +481,16 @@ def self.nothing(chef_run) .with_code(/apt -y purge ifupdown/) .with_code(%r{wget https://d1uj6qtbmh3dt5.cloudfront.net/NICE-GPG-KEY}) when 'amazon' - is_expected.to install_package(alinux_prereq_packages).with_retries(10).with_retry_delay(5) - is_expected.to create_file('Setup Gnome standard').with( - content: "PREFERRED=/usr/bin/gnome-session", - owner: "root", - group: "root", - mode: "0755", - path: "/etc/sysconfig/desktop" - ) + if version == '2' + is_expected.to install_package(alinux_prereq_packages).with_retries(10).with_retry_delay(5) + is_expected.to create_file('Setup Gnome standard').with( + content: "PREFERRED=/usr/bin/gnome-session", + owner: "root", + group: "root", + mode: "0755", + path: "/etc/sysconfig/desktop" + ) + end else is_expected.to run_execute('Install gnome desktop').with_command('yum -y install @gnome').with_retries(3).with_retry_delay(5) is_expected.to install_package('xorg-x11-server-Xorg').with_retries(3).with_retry_delay(5) diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/gdrcopy_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/gdrcopy_spec.rb index 3a8a6a8f73..441e62fd95 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/gdrcopy_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/gdrcopy_spec.rb @@ -201,6 +201,12 @@ def self.configure(chef_run) case platform when 'ubuntu' %w(build-essential devscripts debhelper check libsubunit-dev fakeroot pkg-config dkms) + when 'amazon' + if version == '2023' + %w(dkms rpm-build make check check-devel) + else + %w(dkms rpm-build make check check-devel subunit subunit-devel) + end else %w(dkms rpm-build make check check-devel subunit subunit-devel) end @@ -209,6 +215,7 @@ def self.configure(chef_run) cached(:gdrcopy_platform) do platforms = { 'amazon2' => 'amzn-2', + 'amazon2023' => 'amzn-2023', 'centos7' => 'el7', 'rhel8' => 'el8', 'rocky8' => 'el8', @@ -216,6 +223,7 @@ def self.configure(chef_run) 'rocky9' => 'el9', 'ubuntu20.04' => 'Ubuntu20_04', 'ubuntu22.04' => 'Ubuntu22_04', + 'ubuntu24.04' => 'Ubuntu24_04', } platforms["#{platform}#{version}"] end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/install_packages_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/install_packages_spec.rb index 4db9fc9d7d..2e49318e20 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/install_packages_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/install_packages_spec.rb @@ -59,7 +59,7 @@ def self.setup(chef_run) end end - if platform == 'amazon' + if platform == 'amazon' && version == '2' it 'installs extra packages' do is_expected.to install_alinux_extras_topic('R3.4') end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/modules_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/modules_spec.rb index 0ac9173058..00ae24dc6d 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/modules_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/modules_spec.rb @@ -38,6 +38,8 @@ "/usr/share/modules/init/.modulespath" when 'redhat', 'rocky' '/etc/environment-modules/modulespath' + when 'amazon' + version == '2' ? '/usr/share/Modules/init/.modulespath' : '/etc/environment-modules/modulespath' else "/usr/share/Modules/init/.modulespath" end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_dcgm_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_dcgm_spec.rb index ca32faed4c..3cf2779901 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_dcgm_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_dcgm_spec.rb @@ -89,7 +89,7 @@ def self.setup(chef_run, nvidia_enabled: nil) chef_run.find_resource('nvidia_dcgm', 'setup') end - if %w(centos amazon).include?(platform) + if %w(centos7 amazon2).include?("#{platform}#{version}") it "is not enabled" do expect(resource._nvidia_dcgm_enabled).to eq(false) end @@ -164,7 +164,7 @@ def self.setup(chef_run, nvidia_enabled: nil) ConvergeNvidiaDcgm.setup(chef_run) end - if %w(centos amazon).include?(platform) + if %w(centos7 amazon2).include?("#{platform}#{version}") it 'does not install datacenter gpu manager' do is_expected.not_to run_bash('Install datacenter-gpu-manager') end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_driver_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_driver_spec.rb index a1d81426d7..4d0b8b57ca 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_driver_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_driver_spec.rb @@ -221,21 +221,34 @@ def self.setup(chef_run, nvidia_driver_version: nil) end if platform == 'amazon' - compiler_path = version == 2023 ? 'CC=/usr/bin/gcc' : 'CC=/usr/bin/gcc10-gcc' - it 'installs gcc10' do - is_expected.to install_package('gcc10').with_retries(10).with_retry_delay(5) + compiler_version = version == '2023' ? 'gcc' : 'gcc10' + compiler_path = version == '2023' ? 'CC=/usr/bin/gcc' : 'CC=/usr/bin/gcc10-gcc' + if version == '2' + it "installs #{compiler_version}" do + is_expected.to install_package(compiler_version).with_retries(10).with_retry_delay(5) + end + it 'creates dkms/nvidia.conf' do + is_expected.to create_template('/etc/dkms/nvidia.conf').with( + source: 'nvidia/amazon/dkms/nvidia.conf.erb', + cookbook: 'aws-parallelcluster-platform', + owner: 'root', + group: 'root', + mode: '0644', + variables: { compiler_path: compiler_path } + ) + end + else + # Amazon Linux 2023 is expected to install the compiler and create nvidia conf when kernel version is 6. + # Here we are testing with kernel version 5 + it "does not install #{compiler_version}" do + is_expected.not_to install_package(compiler_version).with_retries(10).with_retry_delay(5) + end + + it 'does not create dkms/nvidia.conf' do + is_expected.not_to create_template('/etc/dkms/nvidia.conf') + end end - it 'creates dkms/nvidia.conf' do - is_expected.to create_template('/etc/dkms/nvidia.conf').with( - source: 'nvidia/amazon/dkms/nvidia.conf.erb', - cookbook: 'aws-parallelcluster-platform', - owner: 'root', - group: 'root', - mode: '0644', - variables: { compiler_path: compiler_path } - ) - end it 'installs nvidia driver' do is_expected.to run_bash('nvidia.run advanced') .with( diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb index f96eb304c5..bb3a1b145f 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb @@ -84,7 +84,9 @@ def self.remove(chef_run) cached(:arch_suffix) { 'arch_suffix' } cached(:nvidia_platform) do case platform - when 'amazon', 'centos' + when 'amazon' + version == '2023' ? 'rhel9' : 'rhel7' + when 'centos' 'rhel7' when 'redhat', 'rocky' "rhel#{version.to_i}" diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/stunnel_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/stunnel_spec.rb index 02b345d463..68d19ec189 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/stunnel_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/stunnel_spec.rb @@ -45,7 +45,7 @@ def self.setup(chef_run, stunnel_version:, stunnel_checksum:) is_expected.to setup_stunnel('setup') end - if platform == 'amazon' + if platform == 'amazon' && version == '2' it "doesn't install stunnel" do is_expected.not_to run_bash('install stunnel') end diff --git a/cookbooks/aws-parallelcluster-shared/spec/unit/resources/package_repos_spec.rb b/cookbooks/aws-parallelcluster-shared/spec/unit/resources/package_repos_spec.rb index 9730f6ca5c..1758192042 100644 --- a/cookbooks/aws-parallelcluster-shared/spec/unit/resources/package_repos_spec.rb +++ b/cookbooks/aws-parallelcluster-shared/spec/unit/resources/package_repos_spec.rb @@ -28,10 +28,15 @@ def self.setup(chef_run) expect(chef_run).to include_recipe('yum') end - it 'installs epel' do - is_expected.to install_alinux_extras_topic('epel') + if version == '2' + it 'installs epel' do + is_expected.to install_alinux_extras_topic('epel') + end + else + it 'does not install epel' do + is_expected.not_to install_alinux_extras_topic('epel') + end end - when 'centos' it 'installs yum and epel' do expect(chef_run).to include_recipe('yum') diff --git a/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/dns_domain_spec.rb b/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/dns_domain_spec.rb index e5b6e84b39..6e5b72ffdd 100644 --- a/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/dns_domain_spec.rb +++ b/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/dns_domain_spec.rb @@ -41,9 +41,9 @@ def self.configure(chef_run) for_all_oses do |platform, version| context "on #{platform}#{version}" do cached(:dns_domain) { 'dns_domain' } - cached(:search_domain_config_path) { platform == 'ubuntu' ? '/etc/systemd/resolved.conf' : '/etc/dhcp/dhclient.conf' } - cached(:append_pattern) { platform == 'ubuntu' ? 'Domains=*' : 'append domain-name*' } - cached(:append_line) { platform == 'ubuntu' ? "Domains=#{dns_domain}" : "append domain-name \" #{dns_domain}\";" } + cached(:search_domain_config_path) { platform == 'ubuntu' || platform == 'amazon' && version == '2023' ? '/etc/systemd/resolved.conf' : '/etc/dhcp/dhclient.conf' } + cached(:append_pattern) { platform == 'ubuntu' || platform == 'amazon' && version == '2023' ? 'Domains=*' : 'append domain-name*' } + cached(:append_line) { platform == 'ubuntu' || platform == 'amazon' && version == '2023' ? "Domains=#{dns_domain}" : "append domain-name \" #{dns_domain}\";" } cached(:chef_run) do runner = runner(platform: platform, version: version, step_into: ['dns_domain']) do |node| @@ -56,7 +56,7 @@ def self.configure(chef_run) is_expected.to configure_dns_domain('configure') end - it 'updates search domaint' do + it 'updates search domain' do is_expected.to edit_replace_or_add("append Route53 search domain in #{search_domain_config_path}").with( path: search_domain_config_path, pattern: append_pattern, diff --git a/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/mysql_client_spec.rb b/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/mysql_client_spec.rb index 048ddc2972..5550e3896e 100644 --- a/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/mysql_client_spec.rb +++ b/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/mysql_client_spec.rb @@ -30,6 +30,8 @@ def self.validate(chef_run) cached(:package_platform) do platform_version = if version.to_i == 2 7 + elsif platform == 'amazon' && version == '2023' + 9 else version.to_i end