From 2731b8372392d4676c3d7decb3711e3bf542db66 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 8 May 2025 15:13:03 -0400 Subject: [PATCH 1/2] Update the efs-utils patched version checksum and URL --- CHANGELOG.md | 4 ++++ .../resources/efs/partial/_common.rb | 2 +- .../resources/efs/partial/_install_from_tar.rb | 2 +- .../aws-parallelcluster-environment/test/controls/efs_spec.rb | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc74e8c84c..58da175a89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,13 @@ This file is used to list changes made in each version of the AWS ParallelCluste **CHANGES** - Upgrade Slurm to version 24.05.8. +**BUG FIXES** +- Use `v2.1.0-patched.tar.gz` which pins backtrace version to v3.0.74 + **BUG FIXES** - Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket. +- Use patched version of efs-utils v2.1.0 which pins backtrace version to v3.0.74 to resolve build image failure. 3.13.0 ------ diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb index 368cfb383d..3840cd430c 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb @@ -3,7 +3,7 @@ default_action :install_utils property :efs_utils_version, String, default: '2.1.0' -property :efs_utils_checksum, String, default: '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' +property :efs_utils_checksum, String, default: 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887' def already_installed?(package_name, expected_version) Gem::Version.new(get_package_version(package_name)) >= Gem::Version.new(expected_version) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index cf65440cf0..c4aac641e1 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -32,7 +32,7 @@ package_name = "amazon-efs-utils" package_version = new_resource.efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" - efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" + efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}-patched.tar.gz" # Do not install efs-utils if a same or newer version is already installed. return if already_installed?(package_name, package_version) diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb index b6732d0086..a821709833 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb @@ -6,7 +6,7 @@ describe file("#{node['cluster']['sources_dir']}/efs-utils-2.1.0.tar.gz") do it { should exist } - its('sha256sum') { should eq '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' } + its('sha256sum') { should eq 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887' } its('owner') { should eq 'root' } its('group') { should eq 'root' } its('mode') { should cmp '0644' } From 85503640a9e089fee5ce08344d9f37d684026210 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Mon, 12 May 2025 11:32:12 -0400 Subject: [PATCH 2/2] Update the efs-utils to use a different Cargo.toml --- CHANGELOG.md | 3 -- .../files/efs/Cargo.toml | 40 +++++++++++++++++++ .../resources/efs/partial/_common.rb | 2 +- .../resources/efs/partial/_debian.rb | 3 +- .../efs/partial/_install_from_tar.rb | 18 ++++++++- .../resources/efs/partial/_redhat_based.rb | 3 +- .../spec/unit/resources/efs_spec.rb | 26 +++++++++++- .../test/controls/efs_spec.rb | 2 +- 8 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 58da175a89..d9dccc0984 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,6 @@ This file is used to list changes made in each version of the AWS ParallelCluste **CHANGES** - Upgrade Slurm to version 24.05.8. -**BUG FIXES** -- Use `v2.1.0-patched.tar.gz` which pins backtrace version to v3.0.74 - **BUG FIXES** - Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket. diff --git a/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml b/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml new file mode 100644 index 0000000000..8a356a0fa2 --- /dev/null +++ b/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "efs-proxy" +edition = "2021" +build = "build.rs" +# The version of efs-proxy is tied to efs-utils. +version = "2.1.0" +publish = false + +[dependencies] +anyhow = "1.0.72" +async-trait = "0.1" +bytes = { version = "1.4.0" } +chrono = "0.4" +clap = { version = "=4.0.0", features = ["derive"] } +fern = "0.6" +futures = "0.3" +log = "0.4" +log4rs = { version = "0", features = ["rolling_file_appender", "compound_policy", "size_trigger", "fixed_window_roller"]} +nix = { version = "0.26.2", features = ["signal"]} +onc-rpc = "0.2.3" +rand = "0.8.5" +s2n-tls = "0.0" +s2n-tls-tokio = "0.0" +s2n-tls-sys = "0.0" +serde = {version="1.0.175",features=["derive"]} +serde_ini = "0.2.0" +thiserror = "1.0.44" +tokio = { version = "1.29.0, <1.39", features = ["full"] } +tokio-util = "0.7.8" +uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"]} +xdr-codec = "0.4.4" +backtrace = "=0.3.74" + +[dev-dependencies] +test-case = "*" +tokio = { version = "1.29.0", features = ["test-util"] } +tempfile = "3.10.1" + +[build-dependencies] +xdrgen = "0.4.4" \ No newline at end of file diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb index 3840cd430c..368cfb383d 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb @@ -3,7 +3,7 @@ default_action :install_utils property :efs_utils_version, String, default: '2.1.0' -property :efs_utils_checksum, String, default: 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887' +property :efs_utils_checksum, String, default: '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' def already_installed?(package_name, expected_version) Gem::Version.new(get_package_version(package_name)) >= Gem::Version.new(expected_version) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb index 4af4e52ddc..60e01fa973 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb @@ -13,10 +13,9 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) +def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version) <<-EFSUTILSINSTALL set -e - tar xf #{efs_utils_tarball} cd efs-utils-#{efs_utils_version} ./build-deb.sh apt-get -y install ./build/#{efs_utils_package}*deb diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index c4aac641e1..7178a482cc 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -32,7 +32,7 @@ package_name = "amazon-efs-utils" package_version = new_resource.efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" - efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}-patched.tar.gz" + efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" # Do not install efs-utils if a same or newer version is already installed. return if already_installed?(package_name, package_version) @@ -50,6 +50,22 @@ action :create_if_missing end + bash "Untar the efs-utils" do + cwd node['cluster']['sources_dir'] + code <<-EFSUTILSUNTAR + set -e + tar xf #{efs_utils_tarball} + EFSUTILSUNTAR + end + + cookbook_file "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}/src/proxy/Cargo.toml" do + source 'efs/Cargo.toml' + owner 'root' + group 'root' + mode '0755' + action :create + end + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html bash "install efs utils" do cwd node['cluster']['sources_dir'] diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb index bc7887a5a7..eddf72aeb2 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb @@ -13,10 +13,9 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) +def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version) <<-EFSUTILSINSTALL set -e - tar xf #{efs_utils_tarball} cd efs-utils-#{efs_utils_version} make rpm yum -y install ./build/#{efs_utils_package}*rpm diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb index f1ef2048ef..2a61eba847 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb @@ -88,10 +88,15 @@ def mock_already_installed(package, expected_version, installed) cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" } cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" } cached(:tarball_checksum) { 'TARBALL CHECKSUM' } + cached(:bash_untar_code) do + <<-EFSUTILSUNTAR + set -e + tar xf #{tarball_path} + EFSUTILSUNTAR + end cached(:bash_code) do <<-EFSUTILSINSTALL set -e - tar xf #{tarball_path} cd efs-utils-#{utils_version} ./build-deb.sh apt-get -y install ./build/amazon-efs-utils*deb @@ -127,6 +132,12 @@ def mock_already_installed(package, expected_version, installed) .with(checksum: tarball_checksum) end + it 'it untars the downloaded tarball' do + is_expected.to run_bash('Untar the efs-utils') + .with(cwd: source_dir) + .with(code: bash_untar_code) + end + it 'installs package from downloaded tarball' do is_expected.to run_bash('install efs utils') .with(cwd: source_dir) @@ -168,10 +179,15 @@ def mock_already_installed(package, expected_version, installed) cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" } cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" } cached(:tarball_checksum) { 'TARBALL CHECKSUM' } + cached(:bash_untar_code) do + <<-EFSUTILSUNTAR + set -e + tar xf #{tarball_path} + EFSUTILSUNTAR + end cached(:bash_code) do <<-EFSUTILSINSTALL set -e - tar xf #{tarball_path} cd efs-utils-#{utils_version} make rpm yum -y install ./build/amazon-efs-utils*rpm @@ -218,6 +234,12 @@ def mock_already_installed(package, expected_version, installed) .with(checksum: tarball_checksum) end + it 'it untars the downloaded tarball' do + is_expected.to run_bash('Untar the efs-utils') + .with(cwd: source_dir) + .with(code: bash_untar_code) + end + it 'installs package from downloaded tarball' do is_expected.to run_bash('install efs utils') .with(cwd: source_dir) diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb index a821709833..b6732d0086 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb @@ -6,7 +6,7 @@ describe file("#{node['cluster']['sources_dir']}/efs-utils-2.1.0.tar.gz") do it { should exist } - its('sha256sum') { should eq 'f257a9859059710bcd8a4a870de2a8f11574b2615feeb7989b764820a9f0c887' } + its('sha256sum') { should eq '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' } its('owner') { should eq 'root' } its('group') { should eq 'root' } its('mode') { should cmp '0644' }