[Isolated] Cherry picks from release-3.13

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -19,6 +19,40 @@
 
 # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
 # bash block to a recipe that uses the pyenv resource.
+command = if aws_region.start_with?("us-iso")
+            "pip install . --no-build-isolation"
+          else
+            "pip install ."
+          end
+
+if aws_region.start_with?("us-iso")
+  dependency_package_name = "pypi-node-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
+  dependency_folder_name = dependency_package_name
+  if platform?('amazon') && node['platform_version'] == "2"
+    dependency_package_name = "node-dependencies"
+    dependency_folder_name = "node"
+  end
+
+  remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
+    source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz"
+    mode '0644'
+    retries 3
+    retry_delay 5
+    action :create_if_missing
+  end
+
+  bash 'pip install' do
+    user 'root'
+    group 'root'
+    cwd "#{node['cluster']['base_dir']}"
+    code <<-REQ
+      set -e
+      tar xzf node-dependencies.tgz
+      cd #{dependency_folder_name}
+      #{node_virtualenv_path}/bin/pip install * -f ./ --no-index
+      REQ
+  end
+end
 
 bash "install custom aws-parallelcluster-node" do
   cwd Chef::Config[:file_cache_path]
@@ -38,7 +72,7 @@
     mkdir aws-parallelcluster-custom-node
     tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-custom-node
     cd aws-parallelcluster-custom-node/*aws-parallelcluster-node*
-    pip install .
+    #{command}
     deactivate
   NODE
 end

cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb

@@ -33,6 +33,34 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
+if aws_region.start_with?("us-iso")
+  dependency_package_name = "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
+  dependency_folder_name = dependency_package_name
+  if platform?('amazon') && node['platform_version'] == "2"
+    dependency_package_name = "cfn-dependencies"
+    dependency_folder_name = "cfn"
+  end
+  remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do
+    source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz"
+    mode '0644'
+    retries 3
+    retry_delay 5
+    action :create_if_missing
+  end
+
+  bash 'pip install' do
+    user 'root'
+    group 'root'
+    cwd "#{node['cluster']['base_dir']}"
+    code <<-REQ
+      set -e
+      tar xzf cfn-dependencies.tgz
+      cd #{dependency_folder_name}
+      #{virtualenv_path}/bin/pip install * -f ./ --no-index
+      REQ
+  end
+end
+
 cfnbootstrap_version = '2.0-33'
 cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz"
 
@@ -51,11 +79,16 @@
   retry_delay 5
 end
 
+command = if aws_region.start_with?("us-iso")
+            "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation"
+          else
+            "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}"
+          end
 bash "Install CloudFormation helpers from #{cfnbootstrap_package}" do
   user 'root'
   group 'root'
   cwd '/tmp'
-  code "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}"
+  code command
   creates "#{virtualenv_path}/bin/cfn-hup"
 end
 

cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb

@@ -21,3 +21,55 @@
 use 'partial/_redhat_based'
 use 'partial/_install_from_tar'
 use 'partial/_mount_umount'
+
+def adc_install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
+  <<-EFSUTILSINSTALL
+      set -e
+      tar xf #{efs_utils_tarball}
+      mv efs-proxy-dependencies-#{efs_utils_version}.tar.gz efs-utils-#{efs_utils_version}/src/proxy/
+      cd efs-utils-#{efs_utils_version}/src/proxy/
+      tar -xf efs-proxy-dependencies-#{efs_utils_version}.tar.gz
+      cargo build --offline
+      cd ../..
+      make rpm
+      yum -y install ./build/#{efs_utils_package}*rpm
+  EFSUTILSINSTALL
+end
+
+def prerequisites
+  %w(rpm-build make rust cargo openssl-devel)
+end
+
+action :install_efs_utils do
+  package_name = "amazon-efs-utils"
+  package_version = _efs_utils_version
+  efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
+
+  if aws_region.start_with?("us-iso")
+
+    efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz"
+    efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}"
+    efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}"
+    remote_file efs_proxy_deps_tarball do
+      source efs_proxy_deps_url
+      mode '0644'
+      retries 3
+      retry_delay 5
+      action :create_if_missing
+    end
+
+    bash "install efs utils" do
+      cwd node['cluster']['sources_dir']
+      code adc_install_script_code(efs_utils_tarball, package_name, package_version)
+    end
+
+  else
+    # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
+    bash "install efs utils" do
+      cwd node['cluster']['sources_dir']
+      code install_script_code(efs_utils_tarball, package_name, package_version)
+    end
+  end
+
+  action_increase_poll_interval
+end

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb

@@ -2,15 +2,12 @@
 
 default_action :install_utils
 
-property :efs_utils_version, String
-property :efs_utils_checksum, String
-
 def _efs_utils_version
-  efs_utils_version || node['cluster']['efs']['version']
+  node['cluster']['efs']['version']
 end
 
 def _efs_utils_checksum
-  efs_utils_checksum || node['cluster']['efs']['sha256']
+  node['cluster']['efs']['sha256']
 end
 
 def already_installed?(package_name, expected_version)

cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb

@@ -12,8 +12,13 @@
 # or in the "LICENSE.txt" file accompanying this file.
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+package_name = "amazon-efs-utils"
 
 action :install_utils do
+  package_version = _efs_utils_version
+  efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
+  efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
+
   package_repos 'update package repositories' do
     action :update
   end
@@ -29,11 +34,6 @@
 
   return if redhat_on_docker?
 
-  package_name = "amazon-efs-utils"
-  package_version = _efs_utils_version
-  efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
-  efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
-
   # Do not install efs-utils if a same or newer version is already installed.
   return if already_installed?(package_name, package_version)
 
@@ -50,10 +50,22 @@
     action :create_if_missing
   end
 
+  action_install_efs_utils
+
+  # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
+  # bash "install efs utils" do
+  #   cwd node['cluster']['sources_dir']
+  #   code install_script_code(efs_utils_tarball, package_name, package_version)
+  # end
+  action_increase_poll_interval
+end
+
+action :install_efs_utils do
+  package_version = _efs_utils_version
+  efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
   # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
   bash "install efs utils" do
     cwd node['cluster']['sources_dir']
     code install_script_code(efs_utils_tarball, package_name, package_version)
   end
-  action_increase_poll_interval
 end

cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb

@@ -1,11 +1,9 @@
 require 'spec_helper'
 
 class ConvergeEfs
-  def self.install_utils(chef_run, efs_utils_version:, tarball_checksum:)
+  def self.install_utils(chef_run)
     chef_run.converge_dsl('aws-parallelcluster-environment') do
       efs 'install_utils' do
-        efs_utils_checksum tarball_checksum
-        efs_utils_version efs_utils_version
         action :install_utils
       end
     end
@@ -28,14 +26,17 @@ def mock_already_installed(package, expected_version, installed)
   context "on amazon2" do
     cached(:efs_utils_version) { '1.2.3' }
     cached(:tarball_checksum) { 'tarball_checksum' }
-    let(:chef_run) do
-      runner(platform: 'amazon', version: '2', step_into: ['efs'])
-    end
 
     context "when same version of amazon-efs-utils already installed" do
+      cached(:chef_run) do
+          runner(platform: 'amazon', version: '2', step_into: ['efs'])
+        end
+      cached(:node) { chef_run.node }
       before do
+        node.override['cluster']['efs']['version'] = efs_utils_version
+        node.override['cluster']['efs']['sha256'] = tarball_checksum
         mock_get_package_version('amazon-efs-utils', efs_utils_version)
-        ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum)
+        ConvergeEfs.install_utils(chef_run)
       end
 
       it 'does not install amazon-efs-utils' do
@@ -44,9 +45,13 @@ def mock_already_installed(package, expected_version, installed)
     end
 
     context "when newer version of amazon-efs-utils already installed" do
+      cached(:chef_run) do
+        runner(platform: 'amazon', version: '2', step_into: ['efs'])
+      end
+      cached(:node) { chef_run.node }
       before do
         mock_get_package_version('amazon-efs-utils', '1.3.2')
-        ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum)
+        ConvergeEfs.install_utils(chef_run)
       end
 
       it 'does not install amazon-efs-utils' do
@@ -55,9 +60,15 @@ def mock_already_installed(package, expected_version, installed)
     end
 
     context "when amazon-efs-utils not installed" do
+      cached(:chef_run) do
+        runner(platform: 'amazon', version: '2', step_into: ['efs'])
+      end
+      cached(:node) { chef_run.node }
       before do
+        node.override['cluster']['efs']['version'] = efs_utils_version
+        node.override['cluster']['efs']['sha256'] = tarball_checksum
         mock_get_package_version('amazon-efs-utils', '')
-        ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum)
+        ConvergeEfs.install_utils(chef_run)
       end
 
       it 'installs amazon-efs-utils' do
@@ -67,9 +78,15 @@ def mock_already_installed(package, expected_version, installed)
     end
 
     context "when older version of amazon-efs-utils installed" do
+      cached(:chef_run) do
+        runner(platform: 'amazon', version: '2', step_into: ['efs'])
+      end
+      cached(:node) { chef_run.node }
       before do
+        node.override['cluster']['efs']['version'] = efs_utils_version
+        node.override['cluster']['efs']['sha256'] = tarball_checksum
         mock_get_package_version('amazon-efs-utils', '1.1.4')
-        ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum)
+        ConvergeEfs.install_utils(chef_run)
       end
 
       it 'installs amazon-efs-utils' do
@@ -105,8 +122,10 @@ def mock_already_installed(package, expected_version, installed)
             node.override['cluster']['efs_utils']['tarball_path'] = tarball_path
             node.override['cluster']['sources_dir'] = source_dir
             node.override['cluster']['region'] = aws_region
+            node.override['cluster']['efs']['version'] = utils_version
+            node.override['cluster']['efs']['sha256'] = tarball_checksum
           end
-          ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum)
+          ConvergeEfs.install_utils(runner)
         end
         cached(:node) { chef_run.node }
 
@@ -140,8 +159,10 @@ def mock_already_installed(package, expected_version, installed)
           runner = runner(platform: platform, version: version, step_into: ['efs']) do |node|
             node.override['cluster']['efs_utils']['tarball_path'] = tarball_path
             node.override['cluster']['sources_dir'] = source_dir
+            node.override['cluster']['efs']['version'] = utils_version
+            node.override['cluster']['efs']['sha256'] = tarball_checksum
           end
-          ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum)
+          ConvergeEfs.install_utils(runner)
         end
         cached(:node) { chef_run.node }
 
@@ -190,9 +211,11 @@ def mock_already_installed(package, expected_version, installed)
           runner = runner(platform: platform, version: version, step_into: ['efs']) do |node|
             node.override['cluster']['efs_utils']['tarball_path'] = tarball_path
             node.override['cluster']['sources_dir'] = source_dir
+            node.override['cluster']['efs']['version'] = utils_version
+            node.override['cluster']['efs']['sha256'] = tarball_checksum
             node.override['cluster']['region'] = aws_region
           end
-          ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum)
+          ConvergeEfs.install_utils(runner)
         end
 
         it 'creates sources dir' do
@@ -230,9 +253,11 @@ def mock_already_installed(package, expected_version, installed)
           mock_already_installed('amazon-efs-utils', utils_version, true)
           runner = runner(platform: platform, version: version, step_into: ['efs']) do |node|
             node.override['cluster']['efs_utils']['tarball_path'] = tarball_path
+            node.override['cluster']['efs']['version'] = utils_version
+            node.override['cluster']['efs']['sha256'] = tarball_checksum
             node.override['cluster']['sources_dir'] = source_dir
           end
-          ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum)
+          ConvergeEfs.install_utils(runner)
         end
 
         it 'does not download tarball' do

cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb

@@ -12,8 +12,10 @@
 # limitations under the License.
 
 virtualenv_path = cookbook_virtualenv_path
-pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz"
+dependency_package_name = "pypi-cookbook-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
+pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{dependency_package_name}.tgz"
 if platform?('amazon') && node['platform_version'] == "2"
+  dependency_package_name = "dependencies"
   pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
 end
 
@@ -44,9 +46,9 @@
   group 'root'
   cwd "#{node['cluster']['base_dir']}"
   code <<-REQ
-    set -e
-    tar xzf cookbook-dependencies.tgz
-    cd dependencies
-    #{virtualenv_path}/bin/pip install * -f ./ --no-index
-    REQ
+  set -e
+  tar xzf cookbook-dependencies.tgz
+  cd #{dependency_package_name}
+  #{virtualenv_path}/bin/pip install * -f ./ --no-index
+  REQ
 end