diff --git a/CHANGELOG.md b/CHANGELOG.md index d225ebb87d..b9baa618fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ CHANGELOG **CHANGES** - Ubuntu 20.04 is no longer supported. -- Support DCV on AL2023 +- Support DCV on Amazon Linux 2023. +- Remove `berkshelf`. All cookbooks are local and do not need `berkshelf` dependency management. **BUG FIXES** - Fix an issue where Security Group validation failed when a rule contained both IPv4 ranges (IpRanges) and security group references (UserIdGroupPairs). diff --git a/cli/src/pcluster/constants.py b/cli/src/pcluster/constants.py index 5b7f51b408..8a93f15527 100644 --- a/cli/src/pcluster/constants.py +++ b/cli/src/pcluster/constants.py @@ -165,7 +165,6 @@ "parallelcluster": "3.14.0", "cookbook": "aws-parallelcluster-cookbook-3.14.0", "chef": "18.4.12", - "berkshelf": "8.0.7", "ami": "dev", } diff --git a/cli/src/pcluster/resources/compute_node/user_data.sh b/cli/src/pcluster/resources/compute_node/user_data.sh index df24b01150..b57826d3a2 100644 --- a/cli/src/pcluster/resources/compute_node/user_data.sh +++ b/cli/src/pcluster/resources/compute_node/user_data.sh @@ -80,14 +80,7 @@ write_files: { mkdir /tmp/cookbooks cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - HOME_BAK="${!HOME}" - export HOME="/tmp" - for d in `ls /tmp/cookbooks`; do - cd /tmp/cookbooks/$d - LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.' - done; - export HOME="${!HOME_BAK}" + tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 } function publish_startup_time @@ -131,7 +124,6 @@ write_files: export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion} export cookbook_version=${CookbookVersion} export chef_version=${ChefVersion} - export berkshelf_version=${BerkshelfVersion} if [ -f /opt/parallelcluster/.bootstrapped ]; then installed_version=$(cat /opt/parallelcluster/.bootstrapped) if [ "${!cookbook_version}" != "${!installed_version}" ]; then diff --git a/cli/src/pcluster/resources/head_node/user_data.sh b/cli/src/pcluster/resources/head_node/user_data.sh index b93c2127ae..1dc1d129c1 100644 --- a/cli/src/pcluster/resources/head_node/user_data.sh +++ b/cli/src/pcluster/resources/head_node/user_data.sh @@ -77,14 +77,7 @@ function vendor_cookbook { mkdir /tmp/cookbooks cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - HOME_BAK="${!HOME}" - export HOME="/tmp" - for d in `ls /tmp/cookbooks`; do - cd /tmp/cookbooks/$d - LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.' - done; - export HOME="${!HOME_BAK}" + tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 } [ -f /etc/profile.d/proxy.sh ] && . /etc/profile.d/proxy.sh @@ -117,7 +110,6 @@ fi export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion} export cookbook_version=${CookbookVersion} export chef_version=${ChefVersion} -export berkshelf_version=${BerkshelfVersion} if [ -f /opt/parallelcluster/.bootstrapped ]; then installed_version=$(cat /opt/parallelcluster/.bootstrapped) if [ "${!cookbook_version}" != "${!installed_version}" ]; then diff --git a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml index faf36f820f..805b710d33 100644 --- a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml +++ b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml @@ -6,9 +6,6 @@ constants: - ChefVersion: type: string value: 18.4.12 - - BerkshelfVersion: - type: string - value: 8.0.7 - FailExitCode: type: string value: 1 @@ -313,13 +310,6 @@ phases: ln -sf ${!CA_CERTS_FILE} /opt/cinc/embedded/ssl/certs/cacert.pem fi - curl --retry 3 -L -o gems.tgz https://${AWS::Region}-aws-parallelcluster.s3.${AWS::Region}.${AWS::URLSuffix}/archives/dependencies/ruby/gems.tgz - tar -xf gems.tgz - - cd vendor/cache - /opt/cinc/embedded/bin/gem install --no-document minitar:0.9 - /opt/cinc/embedded/bin/gem install --local --no-document berkshelf:{{ BerkshelfVersion }} - # Download and vendor Cookbook - name: DownloadCookbook action: ExecuteBash @@ -331,15 +321,7 @@ phases: curl --retry 3 -L -o /etc/chef/aws-parallelcluster-cookbook.tgz "{{ build.CookbookUrl.outputs.stdout }}" - mkdir -p /tmp/cookbooks - cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - - export HOME="/tmp" - for dir in $(ls /tmp/cookbooks); do - cd /tmp/cookbooks/${!dir} - LANG=en_US.UTF-8 sudo /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || (echo 'Vendoring cookbook failed.' && exit {{ FailExitCode }}) - done; + cd /etc/chef && tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 && rm -f aws-parallelcluster-cookbook.tgz - name: CreatingChefClientFile action: CreateFile diff --git a/cli/src/pcluster/resources/login_node/user_data.sh b/cli/src/pcluster/resources/login_node/user_data.sh index 5e130bb50c..32d71567bc 100644 --- a/cli/src/pcluster/resources/login_node/user_data.sh +++ b/cli/src/pcluster/resources/login_node/user_data.sh @@ -67,14 +67,7 @@ write_files: { mkdir /tmp/cookbooks cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - HOME_BAK="${!HOME}" - export HOME="/tmp" - for d in `ls /tmp/cookbooks`; do - cd /tmp/cookbooks/$d - LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.' - done; - export HOME="${!HOME_BAK}" + tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 } export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin @@ -113,7 +106,7 @@ write_files: export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion} export cookbook_version=${CookbookVersion} export chef_version=${ChefVersion} - export berkshelf_version=${BerkshelfVersion} + if [ -f /opt/parallelcluster/.bootstrapped ]; then installed_version=$(cat /opt/parallelcluster/.bootstrapped) if [ "${!cookbook_version}" != "${!installed_version}" ]; then diff --git a/cli/src/pcluster/templates/cdk_builder_utils.py b/cli/src/pcluster/templates/cdk_builder_utils.py index a619ce38cd..ee99369161 100644 --- a/cli/src/pcluster/templates/cdk_builder_utils.py +++ b/cli/src/pcluster/templates/cdk_builder_utils.py @@ -94,7 +94,6 @@ def get_common_user_data_env(node: Union[HeadNode, SlurmQueue, LoginNodesPool], "ParallelClusterVersion": COOKBOOK_PACKAGES_VERSIONS["parallelcluster"], "CookbookVersion": COOKBOOK_PACKAGES_VERSIONS["cookbook"], "ChefVersion": COOKBOOK_PACKAGES_VERSIONS["chef"], - "BerkshelfVersion": COOKBOOK_PACKAGES_VERSIONS["berkshelf"], } diff --git a/cloudformation/external-slurmdbd/external-slurmdbd.json b/cloudformation/external-slurmdbd/external-slurmdbd.json index 2cba9b9b99..76c2a883ab 100644 --- a/cloudformation/external-slurmdbd/external-slurmdbd.json +++ b/cloudformation/external-slurmdbd/external-slurmdbd.json @@ -434,7 +434,7 @@ "UserData": { "Fn::Base64": { "Fn::Sub": [ - "Content-Type: multipart/mixed; boundary=\"==BOUNDARY==\"\nMIME-Version: 1.0\n\n--==BOUNDARY==\nContent-Type: text/cloud-config; charset=us-ascii\nMIME-Version: 1.0\n\npackage_update: false\npackage_upgrade: false\nrepo_upgrade: none\ndatasource_list: [ Ec2, None ]\n\n--==BOUNDARY==\nContent-Type: text/x-shellscript; charset=\"us-ascii\"\nMIME-Version: 1.0\n#!/bin/bash -x\n\nfunction vendor_cookbook\n{\n mkdir /tmp/cookbooks\n cd /tmp/cookbooks\n tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz\n HOME_BAK=\"${!HOME}\"\n export HOME=\"/tmp\"\n for d in /tmp/cookbooks/*; do\n cd \"$d\" || continue\n LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete\n done;\n export HOME=\"${!HOME_BAK}\"\n}\n\nfunction wait_for_private_ip_assignment\n{\n rc=1\n retries=10\n retry=1\n sleeptime=1\n while [ \\( $rc -eq 1 \\) -a \\( $retry -le $retries \\) ]; do\n number_of_ips=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/local-ipv4s | wc -l)\n ((number_of_ips>0))\n rc=$?\n retry=$((retry+1))\n sleep $sleeptime\n done\n return $rc\n}\n\nTOKEN=`curl -X PUT \"http://169.254.169.254/latest/api/token\" -H \"X-aws-ec2-metadata-token-ttl-seconds: 21600\"`\nMAC=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/mac)\nENI_ID=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/interface-id)\nDEVICE_NAME=$(ls /sys/class/net | grep e)\n\n# Configure AWS CLI using the expected overrides, if any.\n[ -f /etc/profile.d/aws-cli-default-config.sh ] && . /etc/profile.d/aws-cli-default-config.sh\n\naws ec2 assign-private-ip-addresses --region \"${AWS::Region}\" --network-interface-id \"${!ENI_ID}\" --private-ip-addresses ${PrivateIp} --allow-reassignment\n\nwait_for_private_ip_assignment || echo \"Assignment of private IP ${PrivateIp} was not successful.\"\n\nip addr add ${PrivateIp}/${SubnetPrefix} dev \"${!DEVICE_NAME}\"\n\nif [ \"${CustomCookbookUrl}\" != \"NONE\" ]; then\n curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${CustomCookbookUrl}\n vendor_cookbook\nfi\n\n# This is necessary to find the cfn-init application\nexport PATH=/opt/aws/bin:${!PATH}\n[ -f /etc/parallelcluster/pcluster_cookbook_environment.sh ] && . /etc/parallelcluster/pcluster_cookbook_environment.sh\n\n$CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -s ${AWS::StackName} -v -c default -r LaunchTemplate --region \"${AWS::Region}\"\n", + "Content-Type: multipart/mixed; boundary=\"==BOUNDARY==\"\nMIME-Version: 1.0\n\n--==BOUNDARY==\nContent-Type: text/cloud-config; charset=us-ascii\nMIME-Version: 1.0\n\npackage_update: false\npackage_upgrade: false\nrepo_upgrade: none\ndatasource_list: [ Ec2, None ]\n\n--==BOUNDARY==\nContent-Type: text/x-shellscript; charset=\"us-ascii\"\nMIME-Version: 1.0\n#!/bin/bash -x\n\nfunction vendor_cookbook\n{\n mkdir /tmp/cookbooks\n cd /tmp/cookbooks\n tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1\n}\n\nfunction wait_for_private_ip_assignment\n{\n rc=1\n retries=10\n retry=1\n sleeptime=1\n while [ \\( $rc -eq 1 \\) -a \\( $retry -le $retries \\) ]; do\n number_of_ips=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/local-ipv4s | wc -l)\n ((number_of_ips>0))\n rc=$?\n retry=$((retry+1))\n sleep $sleeptime\n done\n return $rc\n}\n\nTOKEN=`curl -X PUT \"http://169.254.169.254/latest/api/token\" -H \"X-aws-ec2-metadata-token-ttl-seconds: 21600\"`\nMAC=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/mac)\nENI_ID=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/interface-id)\nDEVICE_NAME=$(ls /sys/class/net | grep e)\n\n# Configure AWS CLI using the expected overrides, if any.\n[ -f /etc/profile.d/aws-cli-default-config.sh ] && . /etc/profile.d/aws-cli-default-config.sh\n\naws ec2 assign-private-ip-addresses --region \"${AWS::Region}\" --network-interface-id \"${!ENI_ID}\" --private-ip-addresses ${PrivateIp} --allow-reassignment\n\nwait_for_private_ip_assignment || echo \"Assignment of private IP ${PrivateIp} was not successful.\"\n\nip addr add ${PrivateIp}/${SubnetPrefix} dev \"${!DEVICE_NAME}\"\n\nif [ \"${CustomCookbookUrl}\" != \"NONE\" ]; then\n curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${CustomCookbookUrl}\n vendor_cookbook\nfi\n\n# This is necessary to find the cfn-init application\nexport PATH=/opt/aws/bin:${!PATH}\n[ -f /etc/parallelcluster/pcluster_cookbook_environment.sh ] && . /etc/parallelcluster/pcluster_cookbook_environment.sh\n\n$CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -s ${AWS::StackName} -v -c default -r LaunchTemplate --region \"${AWS::Region}\"\n", { "CustomCookbookUrl": { "Ref": "CustomCookbookUrl" diff --git a/cloudformation/external-slurmdbd/resources/user_data.sh b/cloudformation/external-slurmdbd/resources/user_data.sh index 211cbe2912..3d3d9b4e19 100644 --- a/cloudformation/external-slurmdbd/resources/user_data.sh +++ b/cloudformation/external-slurmdbd/resources/user_data.sh @@ -19,14 +19,7 @@ function vendor_cookbook { mkdir /tmp/cookbooks cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - HOME_BAK="${!HOME}" - export HOME="/tmp" - for d in /tmp/cookbooks/*; do - cd "$d" || continue - LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete - done; - export HOME="${!HOME_BAK}" + tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 } function wait_for_private_ip_assignment