diff --git a/CHANGELOG.md b/CHANGELOG.md index e1ebe4f0b0..840b59595b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste **BUG FIXES** - Fix a bug in the installation of ARM Performance Library that was causing the build image fail in isolated environments due to cookbook retrieving GCC dependencies from GCC website rather than ParallelCluster bucket. -- Use patched version of efs-utils v2.1.0 which pins backtrace version to v3.0.74 to resolve build image failure. +- Upgrade amazon-efs-utils to version 2.3.1 (from v2.1.0) for non-Amazon Linux AMI's. 3.13.0 ------ diff --git a/cookbooks/aws-parallelcluster-environment/attributes/environment.rb b/cookbooks/aws-parallelcluster-environment/attributes/environment.rb index 4beb4c951e..366356c30e 100644 --- a/cookbooks/aws-parallelcluster-environment/attributes/environment.rb +++ b/cookbooks/aws-parallelcluster-environment/attributes/environment.rb @@ -73,6 +73,11 @@ default['cluster']['efa']['version'] = '1.41.0' default['cluster']['efa']['sha256'] = '3506354cdfbe31ff552fe75f5d0d9bb7efd29cf79bd99457347d29c751c38f9f' +default['cluster']['efs']['version'] = '2.3.1' +default['cluster']['efs']['sha256'] = 'ced12f82e76f9740476b63f30c49bd76cc00b6375e12a9f5f7ba852635c49e15' +if platform?('amazon') + default['cluster']['efs']['version'] = '2.1.0' +end # TODO: Move to platform cookbook default['cluster']['spack_shared_dir'] = "#{node['cluster']['shared_dir']}/spack" default['cluster']['spack']['version'] = '0.20.2' diff --git a/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml b/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml deleted file mode 100644 index 8a356a0fa2..0000000000 --- a/cookbooks/aws-parallelcluster-environment/files/efs/Cargo.toml +++ /dev/null @@ -1,40 +0,0 @@ -[package] -name = "efs-proxy" -edition = "2021" -build = "build.rs" -# The version of efs-proxy is tied to efs-utils. -version = "2.1.0" -publish = false - -[dependencies] -anyhow = "1.0.72" -async-trait = "0.1" -bytes = { version = "1.4.0" } -chrono = "0.4" -clap = { version = "=4.0.0", features = ["derive"] } -fern = "0.6" -futures = "0.3" -log = "0.4" -log4rs = { version = "0", features = ["rolling_file_appender", "compound_policy", "size_trigger", "fixed_window_roller"]} -nix = { version = "0.26.2", features = ["signal"]} -onc-rpc = "0.2.3" -rand = "0.8.5" -s2n-tls = "0.0" -s2n-tls-tokio = "0.0" -s2n-tls-sys = "0.0" -serde = {version="1.0.175",features=["derive"]} -serde_ini = "0.2.0" -thiserror = "1.0.44" -tokio = { version = "1.29.0, <1.39", features = ["full"] } -tokio-util = "0.7.8" -uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"]} -xdr-codec = "0.4.4" -backtrace = "=0.3.74" - -[dev-dependencies] -test-case = "*" -tokio = { version = "1.29.0", features = ["test-util"] } -tempfile = "3.10.1" - -[build-dependencies] -xdrgen = "0.4.4" \ No newline at end of file diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb index d2508a6ebc..8cba286a8c 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2.rb @@ -20,9 +20,9 @@ action :install_utils do package_name = "amazon-efs-utils" - full_package = "#{package_name}-#{new_resource.efs_utils_version}" + full_package = "#{package_name}-#{_efs_utils_version}" # Do not install efs-utils if a same or newer version is already installed. - return if already_installed?(package_name, new_resource.efs_utils_version) + return if already_installed?(package_name, _efs_utils_version) # On Amazon Linux 2, amazon-efs-utils and stunnel are installed from OS repo. package full_package do diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb index e09e7d0114..5b0d809d88 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_alinux2023.rb @@ -21,10 +21,10 @@ use 'partial/_mount_umount' action :install_utils do - package_name = "amazon-efs-utils-#{new_resource.efs_utils_version}" + package_name = "amazon-efs-utils-#{_efs_utils_version}" # Do not install efs-utils if a same or newer version is already installed. - return if already_installed?(package_name, new_resource.efs_utils_version) + return if already_installed?(package_name, _efs_utils_version) # On Amazon Linux 2, amazon-efs-utils and stunnel are installed from OS repo. package package_name do diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb index 368cfb383d..618e91d3fb 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb @@ -2,8 +2,16 @@ default_action :install_utils -property :efs_utils_version, String, default: '2.1.0' -property :efs_utils_checksum, String, default: '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' +property :efs_utils_version, String +property :efs_utils_checksum, String + +def _efs_utils_version + efs_utils_version || node['cluster']['efs']['version'] +end + +def _efs_utils_checksum + efs_utils_checksum || node['cluster']['efs']['sha256'] +end def already_installed?(package_name, expected_version) Gem::Version.new(get_package_version(package_name)) >= Gem::Version.new(expected_version) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb index 60e01fa973..4af4e52ddc 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_debian.rb @@ -13,9 +13,10 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version) +def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) <<-EFSUTILSINSTALL set -e + tar xf #{efs_utils_tarball} cd efs-utils-#{efs_utils_version} ./build-deb.sh apt-get -y install ./build/#{efs_utils_package}*deb diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index 7178a482cc..dea1449ac1 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -30,7 +30,7 @@ return if redhat_on_docker? package_name = "amazon-efs-utils" - package_version = new_resource.efs_utils_version + package_version = _efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" @@ -46,26 +46,10 @@ mode '0644' retries 3 retry_delay 5 - checksum new_resource.efs_utils_checksum + checksum _efs_utils_checksum action :create_if_missing end - bash "Untar the efs-utils" do - cwd node['cluster']['sources_dir'] - code <<-EFSUTILSUNTAR - set -e - tar xf #{efs_utils_tarball} - EFSUTILSUNTAR - end - - cookbook_file "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}/src/proxy/Cargo.toml" do - source 'efs/Cargo.toml' - owner 'root' - group 'root' - mode '0755' - action :create - end - # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html bash "install efs utils" do cwd node['cluster']['sources_dir'] diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb index eddf72aeb2..bc7887a5a7 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_redhat_based.rb @@ -13,9 +13,10 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -def install_script_code(_efs_utils_tarball, efs_utils_package, efs_utils_version) +def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) <<-EFSUTILSINSTALL set -e + tar xf #{efs_utils_tarball} cd efs-utils-#{efs_utils_version} make rpm yum -y install ./build/#{efs_utils_package}*rpm diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb index c9b3136cc2..738999a3eb 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb @@ -88,15 +88,10 @@ def mock_already_installed(package, expected_version, installed) cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" } cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" } cached(:tarball_checksum) { 'TARBALL CHECKSUM' } - cached(:bash_untar_code) do - <<-EFSUTILSUNTAR - set -e - tar xf #{tarball_path} - EFSUTILSUNTAR - end cached(:bash_code) do <<-EFSUTILSINSTALL set -e + tar xf #{tarball_path} cd efs-utils-#{utils_version} ./build-deb.sh apt-get -y install ./build/amazon-efs-utils*deb @@ -132,12 +127,6 @@ def mock_already_installed(package, expected_version, installed) .with(checksum: tarball_checksum) end - it 'it untars the downloaded tarball' do - is_expected.to run_bash('Untar the efs-utils') - .with(cwd: source_dir) - .with(code: bash_untar_code) - end - it 'installs package from downloaded tarball' do is_expected.to run_bash('install efs utils') .with(cwd: source_dir) @@ -179,15 +168,10 @@ def mock_already_installed(package, expected_version, installed) cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" } cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" } cached(:tarball_checksum) { 'TARBALL CHECKSUM' } - cached(:bash_untar_code) do - <<-EFSUTILSUNTAR - set -e - tar xf #{tarball_path} - EFSUTILSUNTAR - end cached(:bash_code) do <<-EFSUTILSINSTALL set -e + tar xf #{tarball_path} cd efs-utils-#{utils_version} make rpm yum -y install ./build/amazon-efs-utils*rpm @@ -234,12 +218,6 @@ def mock_already_installed(package, expected_version, installed) .with(checksum: tarball_checksum) end - it 'it untars the downloaded tarball' do - is_expected.to run_bash('Untar the efs-utils') - .with(cwd: source_dir) - .with(code: bash_untar_code) - end - it 'installs package from downloaded tarball' do is_expected.to run_bash('install efs utils') .with(cwd: source_dir) diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb index b6732d0086..7d9e812496 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/efs_spec.rb @@ -4,9 +4,9 @@ only_if { !os_properties.redhat_on_docker? } - describe file("#{node['cluster']['sources_dir']}/efs-utils-2.1.0.tar.gz") do + describe file("#{node['cluster']['sources_dir']}/efs-utils-2.3.1.tar.gz") do it { should exist } - its('sha256sum') { should eq '2996bdd5387131d302310812fa1e07e1be00f80814a580f5dfeb27d68519fd24' } + its('sha256sum') { should eq 'ced12f82e76f9740476b63f30c49bd76cc00b6375e12a9f5f7ba852635c49e15' } its('owner') { should eq 'root' } its('group') { should eq 'root' } its('mode') { should cmp '0644' }