From bd28beaf13213ee8ff3dc6797b1661b1913104c4 Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Wed, 5 Feb 2025 07:24:11 -0500 Subject: [PATCH 1/2] Download http-parser from s3 --- .../isolated/patch-iso-instance.sh.erb | 59 ++++++++++++++----- .../files/isolated/iso-ca-bundle-config.sh | 2 + .../slurm_dependencies_alinux2023.rb | 32 +++++++--- 3 files changed, 71 insertions(+), 22 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/templates/isolated/patch-iso-instance.sh.erb b/cookbooks/aws-parallelcluster-environment/templates/isolated/patch-iso-instance.sh.erb index c509530320..9206632f86 100644 --- a/cookbooks/aws-parallelcluster-environment/templates/isolated/patch-iso-instance.sh.erb +++ b/cookbooks/aws-parallelcluster-environment/templates/isolated/patch-iso-instance.sh.erb @@ -19,15 +19,41 @@ REGION="$(get_instance_region)" source /etc/os-release OS="${ID}${VERSION_ID}" -[[ "${OS}" != "amzn2" ]] && echo "[ERROR] Unsupported OS '${OS}'. Configuration supported only on Amazon Linux 2." && exit 1 + +if [[ "${OS}" != "amzn2" && "${OS}" != "amzn2023" ]]; then + echo "[ERROR] Unsupported OS '${OS}'. Configuration supported only on Amazon Linux 2 and Amazon Linux 2023." + exit 1 +fi echo "[INFO] Starting: instance configuration for US isolated region" echo "[INFO] Starting: installation of packages from amazon Linux 2 repository for US isolated region" -REPOSITORY_DEFINITION_FILE="/etc/yum.repos.d/tmp-amzn2-iso.repo" +REPOSITORY_DEFINITION_FILE="/etc/yum.repos.d/tmp-${OS}-iso.repo" + +if [[ "${OS}" == "amzn2023" ]]; then + cat > ${REPOSITORY_DEFINITION_FILE} < ${REPOSITORY_DEFINITION_FILE} <> /etc/dnf/dnf.conf + echo -n "" | sudo tee /etc/dnf/vars/dualstack + if [[ ${REGION} == us-isob* ]]; then + dnf install -y amazon-linux-repo-iso ca-certificates-isob + else + dnf install -y amazon-linux-repo-iso ca-certificates-iso + fi + sed -i "s/sslverify=0//g" /etc/dnf/dnf.conf +else + cat > ${REPOSITORY_DEFINITION_FILE} <) CA_BUNDLE="/etc/pki/${REGION}/certs/ca-bundle.pem" -for user in "${USERS[@]}"; do - echo "[INFO] Setting CA bundle ${CA_BUNDLE} for user ${user}" - sudo mkhomedir_helper $user - sudo -u $user aws configure set ca_bundle "$CA_BUNDLE" -done +sudo aws configure set ca_bundle "$CA_BUNDLE" echo "[INFO] Complete: CA bundle configuration for AWS CLI in US isolated region" echo "[INFO] Starting: Setting system-wide environment variables for AWS CLI in US isolated region" +echo "export AWS_CA_BUNDLE=/etc/pki/${REGION}/certs/ca-bundle.pem" >> /etc/profile.d/aws-cli-default-config.sh + echo "export AWS_DEFAULT_REGION=${REGION}" >> /etc/profile.d/aws-cli-default-config.sh -echo "Defaults env_keep += \"AWS_DEFAULT_REGION AWS_CA_BUNDLE\"" > /etc/sudoers.d/pcluster-aws-cli-envkeep +echo "export REQUESTS_CA_BUNDLE=${AWS_CA_BUNDLE}" >> /etc/profile.d/aws-cli-default-config.sh -echo "[INFO] Complete: Setting system-wide environment variables for AWS CLI in US isolated region" +echo "export SSL_CERT_FILE=${AWS_CA_BUNDLE}" >> /etc/profile.d/aws-cli-default-config.sh + +echo "Defaults env_keep += \"AWS_DEFAULT_REGION AWS_CA_BUNDLE REQUESTS_CA_BUNDLE SSL_CERT_FILE\"" > /etc/sudoers.d/pcluster-aws-cli-envkeep -echo "[INFO] Complete: instance configuration for US isolated region" +source /etc/profile.d/aws-cli-default-config.sh + +sudo aws configure set ca_bundle "$CA_BUNDLE" + +echo "[INFO] Complete: Setting system-wide environment variables for AWS CLI in US isolated region" diff --git a/cookbooks/aws-parallelcluster-platform/files/isolated/iso-ca-bundle-config.sh b/cookbooks/aws-parallelcluster-platform/files/isolated/iso-ca-bundle-config.sh index 9891fa080a..21fb88f490 100644 --- a/cookbooks/aws-parallelcluster-platform/files/isolated/iso-ca-bundle-config.sh +++ b/cookbooks/aws-parallelcluster-platform/files/isolated/iso-ca-bundle-config.sh @@ -9,6 +9,8 @@ function get_instance_region { REGION="$(get_instance_region)" +CA_BUNDLE="/etc/pki/${REGION}/certs/ca-bundle.pem" + echo "export AWS_CA_BUNDLE=/etc/pki/${REGION}/certs/ca-bundle.pem" >> /etc/profile.d/aws-cli-default-config.sh echo "export AWS_DEFAULT_REGION=${REGION}" >> /etc/profile.d/aws-cli-default-config.sh diff --git a/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_alinux2023.rb b/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_alinux2023.rb index 1d5a679bbf..52fa7f9bc7 100644 --- a/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_alinux2023.rb +++ b/cookbooks/aws-parallelcluster-slurm/resources/slurm_dependencies/slurm_dependencies_alinux2023.rb @@ -14,6 +14,10 @@ use 'partial/_slurm_dependencies_common' +http_parser_version = "2.9.4" +http_parser_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/http_parser/v#{http_parser_version}.tar.gz" +http_parser_tarball = "#{node['cluster']['sources_dir']}/http-parser-#{http_parser_version}.tar.gz" + def dependencies %w(json-c-devel perl perl-Switch lua-devel dbus-devel) end @@ -22,13 +26,25 @@ def dependencies # http parser is no longer maintained, therefore Amazon Linux 2023 does have have the package in OS repos # https://docs.aws.amazon.com/linux/al2023/release-notes/removed-AL2023.4-AL2.html # Following https://slurm.schedmd.com/related_software.html#jwt for Installing Http-parser - bash 'Install http-parser' do - code <<-HTTP_PARSER - set -e - git clone --depth 1 --single-branch -b v2.9.4 https://github.com/nodejs/http-parser.git http_parser - cd http_parser - make - make install - HTTP_PARSER + + remote_file "#{http_parser_tarball}" do + source "#{http_parser_url}" + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing + end + + bash 'make install' do + user 'root' + group 'root' + cwd "#{node['cluster']['sources_dir']}" + code <<-HTTP + set -e + tar xf #{http_parser_tarball} + cd http-parser-#{http_parser_version} + make + make install + HTTP end end From 5db864ec7fa55bd956e1313e8434c434b9c72fb3 Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Wed, 5 Feb 2025 08:50:17 -0500 Subject: [PATCH 2/2] Fix isolated spec test --- .../spec/unit/recipes/isolated_install_spec.rb | 2 +- .../test/controls/isolated_spec.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/isolated_install_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/isolated_install_spec.rb index 377416891d..5ee8463225 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/isolated_install_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/isolated_install_spec.rb @@ -26,7 +26,7 @@ it 'has the correct content' do is_expected.to render_file("#{node['cluster']['scripts_dir']}/patch-iso-instance.sh") - .with_content("USERS=(root #{node['cluster']['cluster_admin_user']} #{node['cluster']['cluster_user']})") + .with_content("source /etc/profile.d/aws-cli-default-config.sh") end end end diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/isolated_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/isolated_spec.rb index b7bac8bec7..d8eb35c585 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/isolated_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/isolated_spec.rb @@ -17,6 +17,5 @@ its('owner') { should cmp 'root' } its('group') { should cmp 'root' } its('mode') { should cmp '0744' } - its('content') { should match /USERS=\(root #{node['cluster']['cluster_admin_user']} #{node['cluster']['cluster_user']}\)/ } end end