From 91632b14f70f2c20da6a96b7cee4d1acd11c475c Mon Sep 17 00:00:00 2001 From: hanwenli Date: Mon, 7 Jul 2025 13:12:27 -0700 Subject: [PATCH] Remove berkshelf Berkshelf is a dependency manager for Chef cookbooks. With it, you can depend on community cookbooks and include them in your workflow. Berkshelf is not useful when all cookbooks are local. ParallelCluster cookbook has all third party cookbooks ready locally. Reducing software we depend on improves stability of the product. (e.g we have one known issue related to berkshelf https://github.com/aws/aws-parallelcluster/wiki/(3.8.0-%E2%80%90-3.9.3)-ParallelCluster-Build-Image-Failing-during-Installation-of-Minitar-Ruby-Gem-Dependency) --- CHANGELOG.md | 3 ++- cli/src/pcluster/constants.py | 1 - .../resources/compute_node/user_data.sh | 10 +--------- .../pcluster/resources/head_node/user_data.sh | 10 +--------- .../imagebuilder/parallelcluster.yaml | 20 +------------------ .../resources/login_node/user_data.sh | 11 ++-------- .../pcluster/templates/cdk_builder_utils.py | 1 - .../external-slurmdbd/external-slurmdbd.json | 2 +- .../external-slurmdbd/resources/user_data.sh | 9 +-------- 9 files changed, 9 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d225ebb87d..b9baa618fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ CHANGELOG **CHANGES** - Ubuntu 20.04 is no longer supported. -- Support DCV on AL2023 +- Support DCV on Amazon Linux 2023. +- Remove `berkshelf`. All cookbooks are local and do not need `berkshelf` dependency management. **BUG FIXES** - Fix an issue where Security Group validation failed when a rule contained both IPv4 ranges (IpRanges) and security group references (UserIdGroupPairs). diff --git a/cli/src/pcluster/constants.py b/cli/src/pcluster/constants.py index 5b7f51b408..8a93f15527 100644 --- a/cli/src/pcluster/constants.py +++ b/cli/src/pcluster/constants.py @@ -165,7 +165,6 @@ "parallelcluster": "3.14.0", "cookbook": "aws-parallelcluster-cookbook-3.14.0", "chef": "18.4.12", - "berkshelf": "8.0.7", "ami": "dev", } diff --git a/cli/src/pcluster/resources/compute_node/user_data.sh b/cli/src/pcluster/resources/compute_node/user_data.sh index df24b01150..b57826d3a2 100644 --- a/cli/src/pcluster/resources/compute_node/user_data.sh +++ b/cli/src/pcluster/resources/compute_node/user_data.sh @@ -80,14 +80,7 @@ write_files: { mkdir /tmp/cookbooks cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - HOME_BAK="${!HOME}" - export HOME="/tmp" - for d in `ls /tmp/cookbooks`; do - cd /tmp/cookbooks/$d - LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.' - done; - export HOME="${!HOME_BAK}" + tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 } function publish_startup_time @@ -131,7 +124,6 @@ write_files: export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion} export cookbook_version=${CookbookVersion} export chef_version=${ChefVersion} - export berkshelf_version=${BerkshelfVersion} if [ -f /opt/parallelcluster/.bootstrapped ]; then installed_version=$(cat /opt/parallelcluster/.bootstrapped) if [ "${!cookbook_version}" != "${!installed_version}" ]; then diff --git a/cli/src/pcluster/resources/head_node/user_data.sh b/cli/src/pcluster/resources/head_node/user_data.sh index b93c2127ae..1dc1d129c1 100644 --- a/cli/src/pcluster/resources/head_node/user_data.sh +++ b/cli/src/pcluster/resources/head_node/user_data.sh @@ -77,14 +77,7 @@ function vendor_cookbook { mkdir /tmp/cookbooks cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - HOME_BAK="${!HOME}" - export HOME="/tmp" - for d in `ls /tmp/cookbooks`; do - cd /tmp/cookbooks/$d - LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.' - done; - export HOME="${!HOME_BAK}" + tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 } [ -f /etc/profile.d/proxy.sh ] && . /etc/profile.d/proxy.sh @@ -117,7 +110,6 @@ fi export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion} export cookbook_version=${CookbookVersion} export chef_version=${ChefVersion} -export berkshelf_version=${BerkshelfVersion} if [ -f /opt/parallelcluster/.bootstrapped ]; then installed_version=$(cat /opt/parallelcluster/.bootstrapped) if [ "${!cookbook_version}" != "${!installed_version}" ]; then diff --git a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml index faf36f820f..805b710d33 100644 --- a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml +++ b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml @@ -6,9 +6,6 @@ constants: - ChefVersion: type: string value: 18.4.12 - - BerkshelfVersion: - type: string - value: 8.0.7 - FailExitCode: type: string value: 1 @@ -313,13 +310,6 @@ phases: ln -sf ${!CA_CERTS_FILE} /opt/cinc/embedded/ssl/certs/cacert.pem fi - curl --retry 3 -L -o gems.tgz https://${AWS::Region}-aws-parallelcluster.s3.${AWS::Region}.${AWS::URLSuffix}/archives/dependencies/ruby/gems.tgz - tar -xf gems.tgz - - cd vendor/cache - /opt/cinc/embedded/bin/gem install --no-document minitar:0.9 - /opt/cinc/embedded/bin/gem install --local --no-document berkshelf:{{ BerkshelfVersion }} - # Download and vendor Cookbook - name: DownloadCookbook action: ExecuteBash @@ -331,15 +321,7 @@ phases: curl --retry 3 -L -o /etc/chef/aws-parallelcluster-cookbook.tgz "{{ build.CookbookUrl.outputs.stdout }}" - mkdir -p /tmp/cookbooks - cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - - export HOME="/tmp" - for dir in $(ls /tmp/cookbooks); do - cd /tmp/cookbooks/${!dir} - LANG=en_US.UTF-8 sudo /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || (echo 'Vendoring cookbook failed.' && exit {{ FailExitCode }}) - done; + cd /etc/chef && tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 && rm -f aws-parallelcluster-cookbook.tgz - name: CreatingChefClientFile action: CreateFile diff --git a/cli/src/pcluster/resources/login_node/user_data.sh b/cli/src/pcluster/resources/login_node/user_data.sh index 5e130bb50c..32d71567bc 100644 --- a/cli/src/pcluster/resources/login_node/user_data.sh +++ b/cli/src/pcluster/resources/login_node/user_data.sh @@ -67,14 +67,7 @@ write_files: { mkdir /tmp/cookbooks cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - HOME_BAK="${!HOME}" - export HOME="/tmp" - for d in `ls /tmp/cookbooks`; do - cd /tmp/cookbooks/$d - LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.' - done; - export HOME="${!HOME_BAK}" + tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 } export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin @@ -113,7 +106,7 @@ write_files: export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion} export cookbook_version=${CookbookVersion} export chef_version=${ChefVersion} - export berkshelf_version=${BerkshelfVersion} + if [ -f /opt/parallelcluster/.bootstrapped ]; then installed_version=$(cat /opt/parallelcluster/.bootstrapped) if [ "${!cookbook_version}" != "${!installed_version}" ]; then diff --git a/cli/src/pcluster/templates/cdk_builder_utils.py b/cli/src/pcluster/templates/cdk_builder_utils.py index a619ce38cd..ee99369161 100644 --- a/cli/src/pcluster/templates/cdk_builder_utils.py +++ b/cli/src/pcluster/templates/cdk_builder_utils.py @@ -94,7 +94,6 @@ def get_common_user_data_env(node: Union[HeadNode, SlurmQueue, LoginNodesPool], "ParallelClusterVersion": COOKBOOK_PACKAGES_VERSIONS["parallelcluster"], "CookbookVersion": COOKBOOK_PACKAGES_VERSIONS["cookbook"], "ChefVersion": COOKBOOK_PACKAGES_VERSIONS["chef"], - "BerkshelfVersion": COOKBOOK_PACKAGES_VERSIONS["berkshelf"], } diff --git a/cloudformation/external-slurmdbd/external-slurmdbd.json b/cloudformation/external-slurmdbd/external-slurmdbd.json index 2cba9b9b99..76c2a883ab 100644 --- a/cloudformation/external-slurmdbd/external-slurmdbd.json +++ b/cloudformation/external-slurmdbd/external-slurmdbd.json @@ -434,7 +434,7 @@ "UserData": { "Fn::Base64": { "Fn::Sub": [ - "Content-Type: multipart/mixed; boundary=\"==BOUNDARY==\"\nMIME-Version: 1.0\n\n--==BOUNDARY==\nContent-Type: text/cloud-config; charset=us-ascii\nMIME-Version: 1.0\n\npackage_update: false\npackage_upgrade: false\nrepo_upgrade: none\ndatasource_list: [ Ec2, None ]\n\n--==BOUNDARY==\nContent-Type: text/x-shellscript; charset=\"us-ascii\"\nMIME-Version: 1.0\n#!/bin/bash -x\n\nfunction vendor_cookbook\n{\n mkdir /tmp/cookbooks\n cd /tmp/cookbooks\n tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz\n HOME_BAK=\"${!HOME}\"\n export HOME=\"/tmp\"\n for d in /tmp/cookbooks/*; do\n cd \"$d\" || continue\n LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete\n done;\n export HOME=\"${!HOME_BAK}\"\n}\n\nfunction wait_for_private_ip_assignment\n{\n rc=1\n retries=10\n retry=1\n sleeptime=1\n while [ \\( $rc -eq 1 \\) -a \\( $retry -le $retries \\) ]; do\n number_of_ips=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/local-ipv4s | wc -l)\n ((number_of_ips>0))\n rc=$?\n retry=$((retry+1))\n sleep $sleeptime\n done\n return $rc\n}\n\nTOKEN=`curl -X PUT \"http://169.254.169.254/latest/api/token\" -H \"X-aws-ec2-metadata-token-ttl-seconds: 21600\"`\nMAC=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/mac)\nENI_ID=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/interface-id)\nDEVICE_NAME=$(ls /sys/class/net | grep e)\n\n# Configure AWS CLI using the expected overrides, if any.\n[ -f /etc/profile.d/aws-cli-default-config.sh ] && . /etc/profile.d/aws-cli-default-config.sh\n\naws ec2 assign-private-ip-addresses --region \"${AWS::Region}\" --network-interface-id \"${!ENI_ID}\" --private-ip-addresses ${PrivateIp} --allow-reassignment\n\nwait_for_private_ip_assignment || echo \"Assignment of private IP ${PrivateIp} was not successful.\"\n\nip addr add ${PrivateIp}/${SubnetPrefix} dev \"${!DEVICE_NAME}\"\n\nif [ \"${CustomCookbookUrl}\" != \"NONE\" ]; then\n curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${CustomCookbookUrl}\n vendor_cookbook\nfi\n\n# This is necessary to find the cfn-init application\nexport PATH=/opt/aws/bin:${!PATH}\n[ -f /etc/parallelcluster/pcluster_cookbook_environment.sh ] && . /etc/parallelcluster/pcluster_cookbook_environment.sh\n\n$CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -s ${AWS::StackName} -v -c default -r LaunchTemplate --region \"${AWS::Region}\"\n", + "Content-Type: multipart/mixed; boundary=\"==BOUNDARY==\"\nMIME-Version: 1.0\n\n--==BOUNDARY==\nContent-Type: text/cloud-config; charset=us-ascii\nMIME-Version: 1.0\n\npackage_update: false\npackage_upgrade: false\nrepo_upgrade: none\ndatasource_list: [ Ec2, None ]\n\n--==BOUNDARY==\nContent-Type: text/x-shellscript; charset=\"us-ascii\"\nMIME-Version: 1.0\n#!/bin/bash -x\n\nfunction vendor_cookbook\n{\n mkdir /tmp/cookbooks\n cd /tmp/cookbooks\n tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1\n}\n\nfunction wait_for_private_ip_assignment\n{\n rc=1\n retries=10\n retry=1\n sleeptime=1\n while [ \\( $rc -eq 1 \\) -a \\( $retry -le $retries \\) ]; do\n number_of_ips=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/local-ipv4s | wc -l)\n ((number_of_ips>0))\n rc=$?\n retry=$((retry+1))\n sleep $sleeptime\n done\n return $rc\n}\n\nTOKEN=`curl -X PUT \"http://169.254.169.254/latest/api/token\" -H \"X-aws-ec2-metadata-token-ttl-seconds: 21600\"`\nMAC=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/mac)\nENI_ID=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/interface-id)\nDEVICE_NAME=$(ls /sys/class/net | grep e)\n\n# Configure AWS CLI using the expected overrides, if any.\n[ -f /etc/profile.d/aws-cli-default-config.sh ] && . /etc/profile.d/aws-cli-default-config.sh\n\naws ec2 assign-private-ip-addresses --region \"${AWS::Region}\" --network-interface-id \"${!ENI_ID}\" --private-ip-addresses ${PrivateIp} --allow-reassignment\n\nwait_for_private_ip_assignment || echo \"Assignment of private IP ${PrivateIp} was not successful.\"\n\nip addr add ${PrivateIp}/${SubnetPrefix} dev \"${!DEVICE_NAME}\"\n\nif [ \"${CustomCookbookUrl}\" != \"NONE\" ]; then\n curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${CustomCookbookUrl}\n vendor_cookbook\nfi\n\n# This is necessary to find the cfn-init application\nexport PATH=/opt/aws/bin:${!PATH}\n[ -f /etc/parallelcluster/pcluster_cookbook_environment.sh ] && . /etc/parallelcluster/pcluster_cookbook_environment.sh\n\n$CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -s ${AWS::StackName} -v -c default -r LaunchTemplate --region \"${AWS::Region}\"\n", { "CustomCookbookUrl": { "Ref": "CustomCookbookUrl" diff --git a/cloudformation/external-slurmdbd/resources/user_data.sh b/cloudformation/external-slurmdbd/resources/user_data.sh index 211cbe2912..3d3d9b4e19 100644 --- a/cloudformation/external-slurmdbd/resources/user_data.sh +++ b/cloudformation/external-slurmdbd/resources/user_data.sh @@ -19,14 +19,7 @@ function vendor_cookbook { mkdir /tmp/cookbooks cd /tmp/cookbooks - tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz - HOME_BAK="${!HOME}" - export HOME="/tmp" - for d in /tmp/cookbooks/*; do - cd "$d" || continue - LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete - done; - export HOME="${!HOME_BAK}" + tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 } function wait_for_private_ip_assignment