From e86ce64c6b5a41b93096b77b3ce55d9d3406a4c3 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Wed, 14 May 2025 11:35:04 -0400 Subject: [PATCH 1/3] Revert "Update the efs-utils to use a different Cargo.toml" This reverts commit 3eca621f0fcfdc47023d481b7be3ec13ca2955b6. --- CHANGELOG.md | 3 ++ .../files/efs/Cargo.toml | 40 ------------------- .../resources/efs/partial/_common.rb | 2 +- .../resources/efs/partial/_debian.rb | 3 +- .../efs/partial/_install_from_tar.rb | 18 +-------- .../resources/efs/partial/_redhat_based.rb | 3 +- .../spec/unit/resources/efs_spec.rb | 26 +----------- .../test/controls/efs_spec.rb | 2 +- 8 files changed, 12 insertions(+), 85 deletions(-) delete mode 100644 cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index d9dccc0984..58da175a89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ This file is used to list changes made in each version of the AWS ParallelCluste **CHANGES** - Upgrade Slurm to version 24.05.8. +**BUG FIXES** +- Use `v2.1.0-patched.tar.gz` which pins backtrace version to v3.0.74 + **BUG FIXES** - Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket. diff --git a/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml b/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml deleted file mode 100644 index 8a356a0fa2..0000000000 --- a/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml +++ /dev/null @@ -1,40 +0,0 @@ -[package] -name = "efs-proxy" -edition = "2021" -build = "build.rs" -# The version of efs-proxy is tied to efs-utils. -version = "2.1.0" -publish = false - -[dependencies] -anyhow = "1.0.72" -async-trait = "0.1" -bytes = { version = "1.4.0" } -chrono = "0.4" -clap = { version = "=4.0.0", features = ["derive"] } -fern = "0.6" -futures = "0.3" -log = "0.4" -log4rs = { version = "0", features = ["rolling_file_appender", "compound_policy", "size_trigger", "fixed_window_roller"]} -nix = { version = "0.26.2", features = ["signal"]} -onc-rpc = "0.2.3" -rand = "0.8.5" -s2n-tls = "0.0" -s2n-tls-tokio = "0.0" -s2n-tls-sys = "0.0" -serde = {version="1.0.175",features=["derive"]} -serde_ini = "0.2.0" -thiserror = "1.0.44" -tokio = { version = "1.29.0, <1.39", features = ["full"] } -tokio-util = "0.7.8" -uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"]} -xdr-codec = "0.4.4" -backtrace = "=0.3.74" - -[dev-dependencies] -test-case = "*" -tokio = { version = "1.29.0", features = ["test-util"] } -tempfile = "3.10.1" - -[build-dependencies] -xdrgen = "0.4.4" \ No newline at end of file diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb index 368cfb383d..3840cd430c 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb @@ -3,7 +3,7 @@ default_action :install_utils property :efs_utils_version, String, default: '2.1.0' -property :efs_utils_checksum, String, default: '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' +property :efs_utils_checksum, String, default: 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887' def already_installed?(package_name, expected_version) Gem::Version.new(get_package_version(package_name)) >= Gem::Version.new(expected_version) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb index 60e01fa973..4af4e52ddc 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb @@ -13,9 +13,10 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version) +def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) <<-EFSUTILSINSTALL set -e + tar xf #{efs_utils_tarball} cd efs-utils-#{efs_utils_version} ./build-deb.sh apt-get -y install ./build/#{efs_utils_package}*deb diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index 7178a482cc..c4aac641e1 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -32,7 +32,7 @@ package_name = "amazon-efs-utils" package_version = new_resource.efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" - efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" + efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}-patched.tar.gz" # Do not install efs-utils if a same or newer version is already installed. return if already_installed?(package_name, package_version) @@ -50,22 +50,6 @@ action :create_if_missing end - bash "Untar the efs-utils" do - cwd node['cluster']['sources_dir'] - code <<-EFSUTILSUNTAR - set -e - tar xf #{efs_utils_tarball} - EFSUTILSUNTAR - end - - cookbook_file "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}/src/proxy/Cargo.toml" do - source 'efs/Cargo.toml' - owner 'root' - group 'root' - mode '0755' - action :create - end - # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html bash "install efs utils" do cwd node['cluster']['sources_dir'] diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb index eddf72aeb2..bc7887a5a7 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb @@ -13,9 +13,10 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version) +def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) <<-EFSUTILSINSTALL set -e + tar xf #{efs_utils_tarball} cd efs-utils-#{efs_utils_version} make rpm yum -y install ./build/#{efs_utils_package}*rpm diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb index 2a61eba847..f1ef2048ef 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb @@ -88,15 +88,10 @@ def mock_already_installed(package, expected_version, installed) cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" } cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" } cached(:tarball_checksum) { 'TARBALL CHECKSUM' } - cached(:bash_untar_code) do - <<-EFSUTILSUNTAR - set -e - tar xf #{tarball_path} - EFSUTILSUNTAR - end cached(:bash_code) do <<-EFSUTILSINSTALL set -e + tar xf #{tarball_path} cd efs-utils-#{utils_version} ./build-deb.sh apt-get -y install ./build/amazon-efs-utils*deb @@ -132,12 +127,6 @@ def mock_already_installed(package, expected_version, installed) .with(checksum: tarball_checksum) end - it 'it untars the downloaded tarball' do - is_expected.to run_bash('Untar the efs-utils') - .with(cwd: source_dir) - .with(code: bash_untar_code) - end - it 'installs package from downloaded tarball' do is_expected.to run_bash('install efs utils') .with(cwd: source_dir) @@ -179,15 +168,10 @@ def mock_already_installed(package, expected_version, installed) cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" } cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" } cached(:tarball_checksum) { 'TARBALL CHECKSUM' } - cached(:bash_untar_code) do - <<-EFSUTILSUNTAR - set -e - tar xf #{tarball_path} - EFSUTILSUNTAR - end cached(:bash_code) do <<-EFSUTILSINSTALL set -e + tar xf #{tarball_path} cd efs-utils-#{utils_version} make rpm yum -y install ./build/amazon-efs-utils*rpm @@ -234,12 +218,6 @@ def mock_already_installed(package, expected_version, installed) .with(checksum: tarball_checksum) end - it 'it untars the downloaded tarball' do - is_expected.to run_bash('Untar the efs-utils') - .with(cwd: source_dir) - .with(code: bash_untar_code) - end - it 'installs package from downloaded tarball' do is_expected.to run_bash('install efs utils') .with(cwd: source_dir) diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb index b6732d0086..a821709833 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb @@ -6,7 +6,7 @@ describe file("#{node['cluster']['sources_dir']}/efs-utils-2.1.0.tar.gz") do it { should exist } - its('sha256sum') { should eq '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' } + its('sha256sum') { should eq 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887' } its('owner') { should eq 'root' } its('group') { should eq 'root' } its('mode') { should cmp '0644' } From 2e3fcd68b75152d21b13c644e336026a580bf7b1 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Wed, 14 May 2025 11:35:41 -0400 Subject: [PATCH 2/3] Revert "Update the efs-utils patched version checksum and URL" This reverts commit c8e76a8556cb46a0514c4c471f047c36930625ff. --- CHANGELOG.md | 4 ---- .../resources/efs/partial/_common.rb | 2 +- .../resources/efs/partial/_install_from_tar.rb | 2 +- .../aws-parallelcluster-environment/test/controls/efs_spec.rb | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58da175a89..dc74e8c84c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,9 @@ This file is used to list changes made in each version of the AWS ParallelCluste **CHANGES** - Upgrade Slurm to version 24.05.8. -**BUG FIXES** -- Use `v2.1.0-patched.tar.gz` which pins backtrace version to v3.0.74 - **BUG FIXES** - Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket. -- Use patched version of efs-utils v2.1.0 which pins backtrace version to v3.0.74 to resolve build image failure. 3.13.0 ------ diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb index 3840cd430c..368cfb383d 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb @@ -3,7 +3,7 @@ default_action :install_utils property :efs_utils_version, String, default: '2.1.0' -property :efs_utils_checksum, String, default: 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887' +property :efs_utils_checksum, String, default: '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' def already_installed?(package_name, expected_version) Gem::Version.new(get_package_version(package_name)) >= Gem::Version.new(expected_version) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index c4aac641e1..cf65440cf0 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -32,7 +32,7 @@ package_name = "amazon-efs-utils" package_version = new_resource.efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" - efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}-patched.tar.gz" + efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" # Do not install efs-utils if a same or newer version is already installed. return if already_installed?(package_name, package_version) diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb index a821709833..b6732d0086 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb @@ -6,7 +6,7 @@ describe file("#{node['cluster']['sources_dir']}/efs-utils-2.1.0.tar.gz") do it { should exist } - its('sha256sum') { should eq 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887' } + its('sha256sum') { should eq '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' } its('owner') { should eq 'root' } its('group') { should eq 'root' } its('mode') { should cmp '0644' } From d5ec1b0afbfd746a93608e798e5d3ca7a4a66fcf Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Wed, 14 May 2025 13:27:13 -0400 Subject: [PATCH 3/3] [Efs-utils] Update efs utils from v2.1.0 to v2.3.1 except for AL AMI's --- CHANGELOG.md | 1 + .../resources/efs/efs_alinux2.rb | 2 ++ .../resources/efs/efs_alinux2023.rb | 2 ++ .../resources/efs/partial/_common.rb | 4 ++-- .../aws-parallelcluster-environment/test/controls/efs_spec.rb | 4 ++-- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc74e8c84c..a1e2f67fd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste **BUG FIXES** - Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket. +- Upgrade amazon-efs-utils to version 2.3.1 (from v2.1.0) for non-Amazon Linux AMI's. 3.13.0 ------ diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb index d2508a6ebc..8b2fe49a23 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb @@ -18,6 +18,8 @@ use 'partial/_common' use 'partial/_mount_umount' +property :efs_utils_version, String, default: '2.1.0' + action :install_utils do package_name = "amazon-efs-utils" full_package = "#{package_name}-#{new_resource.efs_utils_version}" diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb index e09e7d0114..0ad7aa6c7a 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb @@ -20,6 +20,8 @@ use 'partial/_common' use 'partial/_mount_umount' +property :efs_utils_version, String, default: '2.1.0' + action :install_utils do package_name = "amazon-efs-utils-#{new_resource.efs_utils_version}" diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb index 368cfb383d..3378128679 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb @@ -2,8 +2,8 @@ default_action :install_utils -property :efs_utils_version, String, default: '2.1.0' -property :efs_utils_checksum, String, default: '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' +property :efs_utils_version, String, default: '2.3.1' +property :efs_utils_checksum, String, default: 'ced12f82e76f9740476b63f30c49bd76cc00b6375e12a9f5f7ba852635c49e15' def already_installed?(package_name, expected_version) Gem::Version.new(get_package_version(package_name)) >= Gem::Version.new(expected_version) diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb index b6732d0086..7d9e812496 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb @@ -4,9 +4,9 @@ only_if { !os_properties.redhat_on_docker? } - describe file("#{node['cluster']['sources_dir']}/efs-utils-2.1.0.tar.gz") do + describe file("#{node['cluster']['sources_dir']}/efs-utils-2.3.1.tar.gz") do it { should exist } - its('sha256sum') { should eq '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' } + its('sha256sum') { should eq 'ced12f82e76f9740476b63f30c49bd76cc00b6375e12a9f5f7ba852635c49e15' } its('owner') { should eq 'root' } its('group') { should eq 'root' } its('mode') { should cmp '0644' }