From 9079f1a0c736af7f743d96b5f94f828857cf00de Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Mon, 12 May 2025 11:32:12 -0400 Subject: [PATCH] Update the efs-utils to use a different Cargo.toml --- CHANGELOG.md | 5 +++ .../files/efs/Cargo.toml | 40 +++++++++++++++++++ .../resources/efs/partial/_debian.rb | 3 +- .../efs/partial/_install_from_tar.rb | 16 ++++++++ .../resources/efs/partial/_redhat_based.rb | 3 +- .../spec/unit/resources/efs_spec.rb | 26 +++++++++++- 6 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dd53ca24e..58197c287c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,11 @@ This file is used to list changes made in each version of the AWS ParallelCluste **CHANGES** - Upgrade Slurm to version 24.05.8. +**BUG FIXES** +- Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments +due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket. +- Use patched version of efs-utils v2.1.0 which pins backtrace version to v3.0.74 to resolve build image failure. + 3.13.0 ------ **ENHANCEMENTS** diff --git a/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml b/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml new file mode 100644 index 0000000000..8a356a0fa2 --- /dev/null +++ b/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "efs-proxy" +edition = "2021" +build = "build.rs" +# The version of efs-proxy is tied to efs-utils. +version = "2.1.0" +publish = false + +[dependencies] +anyhow = "1.0.72" +async-trait = "0.1" +bytes = { version = "1.4.0" } +chrono = "0.4" +clap = { version = "=4.0.0", features = ["derive"] } +fern = "0.6" +futures = "0.3" +log = "0.4" +log4rs = { version = "0", features = ["rolling_file_appender", "compound_policy", "size_trigger", "fixed_window_roller"]} +nix = { version = "0.26.2", features = ["signal"]} +onc-rpc = "0.2.3" +rand = "0.8.5" +s2n-tls = "0.0" +s2n-tls-tokio = "0.0" +s2n-tls-sys = "0.0" +serde = {version="1.0.175",features=["derive"]} +serde_ini = "0.2.0" +thiserror = "1.0.44" +tokio = { version = "1.29.0, <1.39", features = ["full"] } +tokio-util = "0.7.8" +uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"]} +xdr-codec = "0.4.4" +backtrace = "=0.3.74" + +[dev-dependencies] +test-case = "*" +tokio = { version = "1.29.0", features = ["test-util"] } +tempfile = "3.10.1" + +[build-dependencies] +xdrgen = "0.4.4" \ No newline at end of file diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb index 4af4e52ddc..60e01fa973 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb @@ -13,10 +13,9 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) +def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version) <<-EFSUTILSINSTALL set -e - tar xf #{efs_utils_tarball} cd efs-utils-#{efs_utils_version} ./build-deb.sh apt-get -y install ./build/#{efs_utils_package}*deb diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index cf65440cf0..7178a482cc 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -50,6 +50,22 @@ action :create_if_missing end + bash "Untar the efs-utils" do + cwd node['cluster']['sources_dir'] + code <<-EFSUTILSUNTAR + set -e + tar xf #{efs_utils_tarball} + EFSUTILSUNTAR + end + + cookbook_file "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}/src/proxy/Cargo.toml" do + source 'efs/Cargo.toml' + owner 'root' + group 'root' + mode '0755' + action :create + end + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html bash "install efs utils" do cwd node['cluster']['sources_dir'] diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb index bc7887a5a7..eddf72aeb2 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb @@ -13,10 +13,9 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) +def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version) <<-EFSUTILSINSTALL set -e - tar xf #{efs_utils_tarball} cd efs-utils-#{efs_utils_version} make rpm yum -y install ./build/#{efs_utils_package}*rpm diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb index 738999a3eb..c9b3136cc2 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb @@ -88,10 +88,15 @@ def mock_already_installed(package, expected_version, installed) cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" } cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" } cached(:tarball_checksum) { 'TARBALL CHECKSUM' } + cached(:bash_untar_code) do + <<-EFSUTILSUNTAR + set -e + tar xf #{tarball_path} + EFSUTILSUNTAR + end cached(:bash_code) do <<-EFSUTILSINSTALL set -e - tar xf #{tarball_path} cd efs-utils-#{utils_version} ./build-deb.sh apt-get -y install ./build/amazon-efs-utils*deb @@ -127,6 +132,12 @@ def mock_already_installed(package, expected_version, installed) .with(checksum: tarball_checksum) end + it 'it untars the downloaded tarball' do + is_expected.to run_bash('Untar the efs-utils') + .with(cwd: source_dir) + .with(code: bash_untar_code) + end + it 'installs package from downloaded tarball' do is_expected.to run_bash('install efs utils') .with(cwd: source_dir) @@ -168,10 +179,15 @@ def mock_already_installed(package, expected_version, installed) cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" } cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" } cached(:tarball_checksum) { 'TARBALL CHECKSUM' } + cached(:bash_untar_code) do + <<-EFSUTILSUNTAR + set -e + tar xf #{tarball_path} + EFSUTILSUNTAR + end cached(:bash_code) do <<-EFSUTILSINSTALL set -e - tar xf #{tarball_path} cd efs-utils-#{utils_version} make rpm yum -y install ./build/amazon-efs-utils*rpm @@ -218,6 +234,12 @@ def mock_already_installed(package, expected_version, installed) .with(checksum: tarball_checksum) end + it 'it untars the downloaded tarball' do + is_expected.to run_bash('Untar the efs-utils') + .with(cwd: source_dir) + .with(code: bash_untar_code) + end + it 'installs package from downloaded tarball' do is_expected.to run_bash('install efs utils') .with(cwd: source_dir)