[ADC Build Image] Add support for build image in ADC

CHANGELOG.md

@@ -14,7 +14,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 
 **ENHANCEMENTS**
 - Add support for external Slurmdbd.
-- Allow build-image to be run in an isolated network.
+- Add support for build-image to be run in an isolated network and ADC regions
 - Add support for Amazon Linux 2023.
 
 **CHANGES**

cookbooks/aws-parallelcluster-awsbatch/recipes/install.rb

@@ -28,6 +28,9 @@
 
 # Check whether install a custom aws-parallelcluster-awsbatch-cli package or the standard one
 # Install awsbatch cli into awsbatch virtual env
+if aws_region.start_with?("us-iso") && !node['cluster']['custom_awsbatchcli_package'].empty?
+  node.default['cluster']['custom_awsbatchcli_package'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/awsbatch/aws-parallelcluster.tgz"
+end
 if !node['cluster']['custom_awsbatchcli_package'].nil? && !node['cluster']['custom_awsbatchcli_package'].empty?
   # Install custom aws-parallelcluster package
   bash "install aws-parallelcluster-awsbatch-cli" do

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -19,15 +19,26 @@
 
 # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
 # bash block to a recipe that uses the pyenv resource.
-
-remote_file "#{Chef::Config[:file_cache_path]}/node-dependencies.tgz" do
+remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
   source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
   mode '0644'
   retries 3
   retry_delay 5
   action :create_if_missing
 end
 
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf node-dependencies.tgz
+    cd node
+    #{node_virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
+
 bash "install custom aws-parallelcluster-node" do
   cwd Chef::Config[:file_cache_path]
   code <<-NODE

cookbooks/aws-parallelcluster-computefleet/recipes/install/parallelcluster_node.rb

@@ -33,6 +33,10 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
+if aws_region.start_with?("us-iso") && !is_custom_node?
+  node.default['cluster']['custom_node_package'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/node/aws-parallelcluster-node.tgz"
+end
+
 if is_custom_node?
   include_recipe 'aws-parallelcluster-computefleet::custom_parallelcluster_node'
 else

cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb

@@ -57,7 +57,13 @@
 cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz"
 
 region = node['cluster']['region']
-bucket = region.start_with?('cn-') ? 's3.cn-north-1.amazonaws.com.cn/cn-north-1-aws-parallelcluster' : "s3.amazonaws.com"
+bucket = "s3.amazonaws.com"
+
+if region.start_with?('cn-')
+  bucket = 's3.cn-north-1.amazonaws.com.cn/cn-north-1-aws-parallelcluster'
+elsif region.start_with?("us-iso")
+  bucket = "s3.#{aws_region}.#{aws_domain}"
+end
 
 remote_file "/tmp/#{cfnbootstrap_package}" do
   source "https://#{bucket}/cloudformation-examples/#{cfnbootstrap_package}"

cookbooks/aws-parallelcluster-environment/resources/cloudwatch/partial/_cloudwatch_common.rb

@@ -27,9 +27,16 @@
 
   action_cloudwatch_prerequisite
 
+  region = node['cluster']['region']
+  key_path = "amazoncloudwatch-agent/assets/amazon-cloudwatch-agent.gpg"
+  cloudwatch_key_url = "https://s3.amazonaws.com/#{key_path}"
+  if region.start_with?("us-iso")
+    cloudwatch_key_url = "https://s3.#{aws_region}.#{aws_domain}/#{key_path}"
+  end
+
   public_key_local_path = "#{node['cluster']['sources_dir']}/amazon-cloudwatch-agent.gpg"
   remote_file public_key_local_path do
-    source 'https://s3.amazonaws.com/amazoncloudwatch-agent/assets/amazon-cloudwatch-agent.gpg'
+    source cloudwatch_key_url
     retries 3
     retry_delay 5
     action :create_if_missing

cookbooks/aws-parallelcluster-environment/resources/efa/efa_alinux2.rb

@@ -24,6 +24,10 @@ def conflicting_packages
   end
 
   def prerequisites
-    %w(environment-modules libibverbs-utils librdmacm-utils)
+    if aws_region.start_with?("us-iso")
+      %w(libibverbs-utils librdmacm-utils)
+    else
+      %w(environment-modules libibverbs-utils librdmacm-utils)
+    end
   end
 end

cookbooks/aws-parallelcluster-environment/resources/efa/partial/_common.rb

@@ -55,7 +55,11 @@
 
 action :download_and_install do
   # Get EFA Installer
+  region = aws_region
   efa_installer_url = "https://efa-installer.amazonaws.com/aws-efa-installer-#{new_resource.efa_version}.tar.gz"
+  if region.start_with?("us-iso")
+    efa_installer_url = "https://aws-efa-installer.s3.#{aws_region}.#{aws_domain}/aws-efa-installer-#{new_resource.efa_version}.tar.gz"
+  end
   remote_file efa_tarball do
     source efa_installer_url
     mode '0644'

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb

@@ -32,7 +32,7 @@
   package_name = "amazon-efs-utils"
   package_version = new_resource.efs_utils_version
   efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
-  efs_utils_url = "https://github.com/aws/efs-utils/archive/v#{package_version}.tar.gz"
+  efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
 
   # Do not install efs-utils if a same or newer version is already installed.
   return if already_installed?(package_name, package_version)

cookbooks/aws-parallelcluster-environment/resources/lustre/lustre_redhat8.rb

@@ -44,10 +44,18 @@
 action_class do
   def base_url
     # https://docs.aws.amazon.com/fsx/latest/LustreGuide/install-lustre-client.html#lustre-client-rhel
-    "https://fsx-lustre-client-repo.s3.amazonaws.com/el/#{node['platform_version']}/$basearch"
+    if aws_region.start_with?("us-iso")
+      "https://fsx-lustre-client-repo.s3.#{aws_region}.#{aws_domain}/el/#{node['platform_version']}/$basearch"
+    else
+      "https://fsx-lustre-client-repo.s3.amazonaws.com/el/#{node['platform_version']}/$basearch"
+    end
   end
 
   def public_key
-    "https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-rpm-public-key.asc"
+    if aws_region.start_with?("us-iso")
+      "https://fsx-lustre-client-repo-public-keys.s3.#{aws_region}.#{aws_domain}/fsx-rpm-public-key.asc"
+    else
+      "https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-rpm-public-key.asc"
+    end
   end
 end

cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb

@@ -86,7 +86,7 @@ def mock_already_installed(package, expected_version, installed)
       cached(:source_dir) { 'SOURCE DIR' }
       cached(:utils_version) { '1.2.3' }
       cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" }
-      cached(:tarball_url) { "https://github.com/aws/efs-utils/archive/v#{utils_version}.tar.gz" }
+      cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" }
       cached(:tarball_checksum) { 'TARBALL CHECKSUM' }
       cached(:bash_code) do
         <<-EFSUTILSINSTALL
@@ -104,6 +104,7 @@ def mock_already_installed(package, expected_version, installed)
           runner = runner(platform: platform, version: version, step_into: ['efs']) do |node|
             node.override['cluster']['efs_utils']['tarball_path'] = tarball_path
             node.override['cluster']['sources_dir'] = source_dir
+            node.override['cluster']['region'] = aws_region
           end
           ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum)
         end
@@ -166,7 +167,7 @@ def mock_already_installed(package, expected_version, installed)
       cached(:source_dir) { 'SOURCE DIR' }
       cached(:utils_version) { '1.2.3' }
       cached(:tarball_path) { "#{source_dir}/efs-utils-#{utils_version}.tar.gz" }
-      cached(:tarball_url) { "https://github.com/aws/efs-utils/archive/v#{utils_version}.tar.gz" }
+      cached(:tarball_url) { "https://#{aws_region}-aws-parallelcluster.s3.#{aws_region}.test_aws_domain/archives/dependencies/efs/v#{utils_version}.tar.gz" }
       cached(:tarball_checksum) { 'TARBALL CHECKSUM' }
       cached(:bash_code) do
         <<-EFSUTILSINSTALL
@@ -191,6 +192,7 @@ def mock_already_installed(package, expected_version, installed)
           runner = runner(platform: platform, version: version, step_into: ['efs']) do |node|
             node.override['cluster']['efs_utils']['tarball_path'] = tarball_path
             node.override['cluster']['sources_dir'] = source_dir
+            node.override['cluster']['region'] = aws_region
           end
           ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum)
         end

cookbooks/aws-parallelcluster-platform/files/isolated/iso-ca-bundle-config.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ex
+
+function get_instance_region {
+  local _token=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 3600")
+  curl -H "X-aws-ec2-metadata-token: $_token" -v "http://169.254.169.254/latest/meta-data/placement/region" 2> /dev/null
+}
+
+REGION="$(get_instance_region)"
+
+echo "export AWS_CA_BUNDLE=/etc/pki/${REGION}/certs/ca-bundle.pem" >> /etc/profile.d/aws-cli-default-config.sh
+
+echo "export AWS_DEFAULT_REGION=${REGION}" >> /etc/profile.d/aws-cli-default-config.sh
+
+echo "Defaults env_keep += \"AWS_DEFAULT_REGION AWS_CA_BUNDLE\"" > /etc/sudoers.d/pcluster-aws-cli-envkeep

cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb

@@ -19,10 +19,18 @@
 return if ::File.exist?("/usr/local/bin/aws") || redhat_on_docker?
 
 file_cache_path = Chef::Config[:file_cache_path]
+region = aws_region
+awscli_url = "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip"
+
+if region.start_with?("us-iso-")
+  awscli_url = "https://aws-sdk-common-infra-dca-prod-deployment-bucket.s3.#{aws_region}.#{aws_domain}/aws-cli-v2/linux/x86_64/awscli-exe-linux-x86_64.zip"
+elsif region.start_with?("us-isob-")
+  awscli_url = "https://aws-sdk-common-infra-lck-prod-deployment-bucket.s3.#{aws_region}.#{aws_domain}/aws-cli-v2/linux/x86_64/awscli-exe-linux-x86_64.zip"
+end
 
 remote_file 'download awscli bundle from s3' do
   path "#{file_cache_path}/awscli-bundle.zip"
-  source 'https://s3.amazonaws.com/aws-cli/awscli-bundle.zip'
+  source awscli_url
   path
   retries 5
   retry_delay 5
@@ -34,6 +42,25 @@
   overwrite true
 end
 
-bash 'install awscli' do
-  code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws"
+if region.start_with?("us-iso")
+  bash 'install awscli' do
+    code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
+  end
+
+  cookbook_file "#{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh" do
+    source 'isolated/iso-ca-bundle-config.sh'
+    cookbook 'aws-parallelcluster-platform'
+    owner 'root'
+    group 'root'
+    mode '0755'
+    action :create_if_missing
+  end
+
+  execute "patch ca bundle" do
+    command "sh #{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh"
+  end
+else
+  bash 'install awscli' do
+    code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws"
+  end
 end

cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_amazon2.rb

@@ -25,6 +25,14 @@ def gdrcopy_platform
   'amzn-2'
 end
 
+def gdrcopy_build_dependencies
+  if aws_region.start_with?("us-iso")
+    %w(dkms rpm-build make check check-devel)
+  else
+    %w(dkms rpm-build make check check-devel subunit subunit-devel)
+  end
+end
+
 def gdrcopy_arch
   arm_instance? ? 'aarch64' : 'x86_64'
 end

cookbooks/aws-parallelcluster-platform/resources/gdrcopy/gdrcopy_redhat8.rb

@@ -23,6 +23,14 @@ def gdrcopy_enabled?
   nvidia_enabled?
 end
 
+def gdrcopy_build_dependencies
+  if aws_region.start_with?("us-iso")
+    %w(rpm-build make check check-devel)
+  else
+    %w(dkms rpm-build make check check-devel subunit subunit-devel)
+  end
+end
+
 def gdrcopy_platform
   "el#{node['platform_version'].to_i}"
 end

cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_amazon2.rb

@@ -22,7 +22,7 @@
 def default_packages
   # environment-modules required by EFA, Intel MPI and ARM PL
   # iptables needed for IMDS setup
-  %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
+  packages = %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
      libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf pyparted libtool
      httpd boost-devel system-lsb mlocate atlas-devel glibc-static iproute
      libffi-devel dkms libedit-devel sendmail cmake byacc libglvnd-devel libgcrypt-devel libevent-devel
@@ -31,7 +31,13 @@ def default_packages
      gcc-gfortran git indent intltool patchutils rcs subversion swig systemtap curl
      jq wget python-pip NetworkManager-config-routing-rules
      python3 python3-pip iptables libcurl-devel yum-plugin-versionlock
-     coreutils moreutils environment-modules bzip2)
+     coreutils environment-modules bzip2)
+
+  if aws_region.start_with?("us-iso")
+    packages -= %w(moreutils)
+  end
+
+  packages
 end
 
 action :install_extras do
@@ -40,4 +46,25 @@ def default_packages
   ['R3.4'].each do |topic|
     alinux_extras_topic topic
   end
+
+  if aws_region.start_with?("us-iso")
+    remote_file "epel_deps.tar.gz" do
+      source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel7/#{node['kernel']['machine']}/epel_deps.tar.gz"
+      mode '0644'
+      retries 3
+      retry_delay 5
+      action :create_if_missing
+    end
+
+    bash 'yum install missing deps' do
+      user 'root'
+      group 'root'
+      code <<-REQ
+      set -e
+      tar xzf epel_deps.tar.gz
+      cd epel
+      yum install -y * 2>/dev/null
+      REQ
+    end
+  end
 end

cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb

@@ -23,12 +23,42 @@ def default_packages
   # environment-modules required by EFA, Intel MPI and ARM PL
   # Removed libssh2-devel from base_packages since is not shipped by RedHat 8 and in conflict with package libssh-0.9.6-3.el8.x86_64
   # iptables needed for IMDS setup
-  %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
+  packages = %w(vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
      libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf libtool
      httpd boost-devel mlocate R atlas-devel
      blas-devel libffi-devel dkms libedit-devel jq
      libical-devel sendmail libxml2-devel libglvnd-devel
      libgcrypt-devel libevent-devel glibc-static bind-utils
      iproute NetworkManager-config-routing-rules python3 python3-pip iptables libcurl-devel yum-plugin-versionlock
      coreutils moreutils curl environment-modules gcc gcc-c++ bzip2)
+
+  if aws_region.start_with?("us-iso")
+    packages -= %w(openmotif-devel hwloc-devel R blas-devel dkms libedit-devel glibc-static
+       NetworkManager-config-routing-rules yum-plugin-versionlock moreutils)
+  end
+
+  packages
+end
+
+action :install_extras do
+  if aws_region.start_with?("us-iso")
+    remote_file "epel_deps.tar.gz" do
+      source "#{node['cluster']['artifacts_s3_url']}/dependencies/epel/rhel8/x86_64/epel_deps.tar.gz"
+      mode '0644'
+      retries 3
+      retry_delay 5
+      action :create_if_missing
+    end
+
+    bash 'yum install missing deps' do
+      user 'root'
+      group 'root'
+      code <<-REQ
+      set -e
+      tar xzf epel_deps.tar.gz
+      cd epel
+      yum install -y * 2>/dev/null
+      REQ
+    end
+  end
 end

cookbooks/aws-parallelcluster-platform/resources/install_packages/partial/_install_packages_common.rb

@@ -39,7 +39,7 @@
 end
 
 action :setup do
+  action_install_extras
   action_install_kernel_source
   action_install_base_packages
-  action_install_extras
 end

cookbooks/aws-parallelcluster-shared/attributes/environment.rb

@@ -5,5 +5,4 @@
 
 # URL for ParallelCluster Artifacts stored in public S3 buckets
 # ['cluster']['region'] will need to be defined by image_dna.json during AMI build.
-default['cluster']['artifacts_build_url'] = "s3://#{node['cluster']['region']}-aws-parallelcluster/archives/dependencies"
 default['cluster']['artifacts_s3_url'] = "https://#{node['cluster']['region']}-aws-parallelcluster.s3.#{node['cluster']['region']}.#{node['cluster']['aws_domain']}/archives"