Remove berkshelf

CHANGELOG.md

@@ -6,7 +6,8 @@ CHANGELOG
 
 **CHANGES**
 - Ubuntu 20.04 is no longer supported.
-- Support DCV on AL2023
+- Support DCV on Amazon Linux 2023.
+- Remove `berkshelf`. All cookbooks are local and do not need `berkshelf` dependency management.
 
 **BUG FIXES**
 - Fix an issue where Security Group validation failed when a rule contained both IPv4 ranges (IpRanges) and security group references (UserIdGroupPairs).

cli/src/pcluster/constants.py

@@ -165,7 +165,6 @@
     "parallelcluster": "3.14.0",
     "cookbook": "aws-parallelcluster-cookbook-3.14.0",
     "chef": "18.4.12",
-    "berkshelf": "8.0.7",
     "ami": "dev",
 }
 

cli/src/pcluster/resources/compute_node/user_data.sh

@@ -80,14 +80,7 @@ write_files:
       {
         mkdir /tmp/cookbooks
         cd /tmp/cookbooks
-        tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz
-        HOME_BAK="${!HOME}"
-        export HOME="/tmp"
-        for d in `ls /tmp/cookbooks`; do
-          cd /tmp/cookbooks/$d
-          LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.'
-        done;
-        export HOME="${!HOME_BAK}"
+        tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1
       }
 
       function publish_startup_time
@@ -131,7 +124,6 @@ write_files:
       export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion}
       export cookbook_version=${CookbookVersion}
       export chef_version=${ChefVersion}
-      export berkshelf_version=${BerkshelfVersion}
       if [ -f /opt/parallelcluster/.bootstrapped ]; then
         installed_version=$(cat /opt/parallelcluster/.bootstrapped)
         if [ "${!cookbook_version}" != "${!installed_version}" ]; then

cli/src/pcluster/resources/head_node/user_data.sh

@@ -77,14 +77,7 @@ function vendor_cookbook
 {
   mkdir /tmp/cookbooks
   cd /tmp/cookbooks
-  tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz
-  HOME_BAK="${!HOME}"
-  export HOME="/tmp"
-  for d in `ls /tmp/cookbooks`; do
-    cd /tmp/cookbooks/$d
-    LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.'
-  done;
-  export HOME="${!HOME_BAK}"
+  tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1
 }
 [ -f /etc/profile.d/proxy.sh ] && . /etc/profile.d/proxy.sh
 
@@ -117,7 +110,6 @@ fi
 export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion}
 export cookbook_version=${CookbookVersion}
 export chef_version=${ChefVersion}
-export berkshelf_version=${BerkshelfVersion}
 if [ -f /opt/parallelcluster/.bootstrapped ]; then
   installed_version=$(cat /opt/parallelcluster/.bootstrapped)
   if [ "${!cookbook_version}" != "${!installed_version}" ]; then

cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml

@@ -6,9 +6,6 @@ constants:
   - ChefVersion:
       type: string
       value: 18.4.12
-  - BerkshelfVersion:
-      type: string
-      value: 8.0.7
   - FailExitCode:
       type: string
       value: 1
@@ -313,13 +310,6 @@ phases:
                 ln -sf ${!CA_CERTS_FILE} /opt/cinc/embedded/ssl/certs/cacert.pem
               fi
 
-              curl --retry 3 -L -o gems.tgz https://${AWS::Region}-aws-parallelcluster.s3.${AWS::Region}.${AWS::URLSuffix}/archives/dependencies/ruby/gems.tgz
-              tar -xf gems.tgz
-
-              cd vendor/cache
-              /opt/cinc/embedded/bin/gem install --no-document minitar:0.9
-              /opt/cinc/embedded/bin/gem install --local --no-document berkshelf:{{ BerkshelfVersion }}
-
       # Download and vendor Cookbook
       - name: DownloadCookbook
         action: ExecuteBash
@@ -331,15 +321,7 @@ phases:
 
               curl --retry 3 -L -o /etc/chef/aws-parallelcluster-cookbook.tgz "{{ build.CookbookUrl.outputs.stdout }}"
 
-              mkdir -p /tmp/cookbooks
-              cd /tmp/cookbooks
-              tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz
-
-              export HOME="/tmp"
-              for dir in $(ls /tmp/cookbooks); do
-                cd /tmp/cookbooks/${!dir}
-                LANG=en_US.UTF-8 sudo /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || (echo 'Vendoring cookbook failed.' && exit {{ FailExitCode }})
-              done;
+              cd /etc/chef && tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1 && rm -f aws-parallelcluster-cookbook.tgz
 
       - name: CreatingChefClientFile
         action: CreateFile

cli/src/pcluster/resources/login_node/user_data.sh

@@ -67,14 +67,7 @@ write_files:
       {
         mkdir /tmp/cookbooks
         cd /tmp/cookbooks
-        tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz
-        HOME_BAK="${!HOME}"
-        export HOME="/tmp"
-        for d in `ls /tmp/cookbooks`; do
-          cd /tmp/cookbooks/$d
-          LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete || error_exit 'Vendoring cookbook failed.'
-        done;
-        export HOME="${!HOME_BAK}"
+        tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1
       }
 
       export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin
@@ -113,7 +106,7 @@ write_files:
       export parallelcluster_version=aws-parallelcluster-${ParallelClusterVersion}
       export cookbook_version=${CookbookVersion}
       export chef_version=${ChefVersion}
-      export berkshelf_version=${BerkshelfVersion}
+
       if [ -f /opt/parallelcluster/.bootstrapped ]; then
         installed_version=$(cat /opt/parallelcluster/.bootstrapped)
         if [ "${!cookbook_version}" != "${!installed_version}" ]; then

cli/src/pcluster/templates/cdk_builder_utils.py

@@ -94,7 +94,6 @@ def get_common_user_data_env(node: Union[HeadNode, SlurmQueue, LoginNodesPool],
         "ParallelClusterVersion": COOKBOOK_PACKAGES_VERSIONS["parallelcluster"],
         "CookbookVersion": COOKBOOK_PACKAGES_VERSIONS["cookbook"],
         "ChefVersion": COOKBOOK_PACKAGES_VERSIONS["chef"],
-        "BerkshelfVersion": COOKBOOK_PACKAGES_VERSIONS["berkshelf"],
     }
 
 

cloudformation/external-slurmdbd/external-slurmdbd.json

@@ -434,7 +434,7 @@
      "UserData": {
       "Fn::Base64": {
        "Fn::Sub": [
-        "Content-Type: multipart/mixed; boundary=\"==BOUNDARY==\"\nMIME-Version: 1.0\n\n--==BOUNDARY==\nContent-Type: text/cloud-config; charset=us-ascii\nMIME-Version: 1.0\n\npackage_update: false\npackage_upgrade: false\nrepo_upgrade: none\ndatasource_list: [ Ec2, None ]\n\n--==BOUNDARY==\nContent-Type: text/x-shellscript; charset=\"us-ascii\"\nMIME-Version: 1.0\n#!/bin/bash -x\n\nfunction vendor_cookbook\n{\n  mkdir /tmp/cookbooks\n  cd /tmp/cookbooks\n  tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz\n  HOME_BAK=\"${!HOME}\"\n  export HOME=\"/tmp\"\n  for d in /tmp/cookbooks/*; do\n    cd \"$d\" || continue\n    LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete\n  done;\n  export HOME=\"${!HOME_BAK}\"\n}\n\nfunction wait_for_private_ip_assignment\n{\n  rc=1\n  retries=10\n  retry=1\n  sleeptime=1\n  while [ \\( $rc -eq 1 \\) -a \\( $retry -le $retries \\) ]; do\n    number_of_ips=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/local-ipv4s | wc -l)\n    ((number_of_ips>0))\n    rc=$?\n    retry=$((retry+1))\n    sleep $sleeptime\n  done\n  return $rc\n}\n\nTOKEN=`curl -X PUT \"http://169.254.169.254/latest/api/token\" -H \"X-aws-ec2-metadata-token-ttl-seconds: 21600\"`\nMAC=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/mac)\nENI_ID=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/interface-id)\nDEVICE_NAME=$(ls /sys/class/net | grep e)\n\n# Configure AWS CLI using the expected overrides, if any.\n[ -f /etc/profile.d/aws-cli-default-config.sh ] && . /etc/profile.d/aws-cli-default-config.sh\n\naws ec2 assign-private-ip-addresses --region \"${AWS::Region}\" --network-interface-id \"${!ENI_ID}\" --private-ip-addresses ${PrivateIp} --allow-reassignment\n\nwait_for_private_ip_assignment || echo \"Assignment of private IP ${PrivateIp} was not successful.\"\n\nip addr add ${PrivateIp}/${SubnetPrefix} dev \"${!DEVICE_NAME}\"\n\nif [ \"${CustomCookbookUrl}\" != \"NONE\" ]; then\n  curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${CustomCookbookUrl}\n  vendor_cookbook\nfi\n\n# This is necessary to find the cfn-init application\nexport PATH=/opt/aws/bin:${!PATH}\n[ -f /etc/parallelcluster/pcluster_cookbook_environment.sh ] && . /etc/parallelcluster/pcluster_cookbook_environment.sh\n\n$CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -s ${AWS::StackName} -v -c default -r LaunchTemplate --region \"${AWS::Region}\"\n",
+        "Content-Type: multipart/mixed; boundary=\"==BOUNDARY==\"\nMIME-Version: 1.0\n\n--==BOUNDARY==\nContent-Type: text/cloud-config; charset=us-ascii\nMIME-Version: 1.0\n\npackage_update: false\npackage_upgrade: false\nrepo_upgrade: none\ndatasource_list: [ Ec2, None ]\n\n--==BOUNDARY==\nContent-Type: text/x-shellscript; charset=\"us-ascii\"\nMIME-Version: 1.0\n#!/bin/bash -x\n\nfunction vendor_cookbook\n{\n  mkdir /tmp/cookbooks\n  cd /tmp/cookbooks\n  tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1\n}\n\nfunction wait_for_private_ip_assignment\n{\n  rc=1\n  retries=10\n  retry=1\n  sleeptime=1\n  while [ \\( $rc -eq 1 \\) -a \\( $retry -le $retries \\) ]; do\n    number_of_ips=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/local-ipv4s | wc -l)\n    ((number_of_ips>0))\n    rc=$?\n    retry=$((retry+1))\n    sleep $sleeptime\n  done\n  return $rc\n}\n\nTOKEN=`curl -X PUT \"http://169.254.169.254/latest/api/token\" -H \"X-aws-ec2-metadata-token-ttl-seconds: 21600\"`\nMAC=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/mac)\nENI_ID=$(curl -H \"X-aws-ec2-metadata-token: $TOKEN\" -v http://169.254.169.254/latest/meta-data/network/interfaces/macs/\"$MAC\"/interface-id)\nDEVICE_NAME=$(ls /sys/class/net | grep e)\n\n# Configure AWS CLI using the expected overrides, if any.\n[ -f /etc/profile.d/aws-cli-default-config.sh ] && . /etc/profile.d/aws-cli-default-config.sh\n\naws ec2 assign-private-ip-addresses --region \"${AWS::Region}\" --network-interface-id \"${!ENI_ID}\" --private-ip-addresses ${PrivateIp} --allow-reassignment\n\nwait_for_private_ip_assignment || echo \"Assignment of private IP ${PrivateIp} was not successful.\"\n\nip addr add ${PrivateIp}/${SubnetPrefix} dev \"${!DEVICE_NAME}\"\n\nif [ \"${CustomCookbookUrl}\" != \"NONE\" ]; then\n  curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${CustomCookbookUrl}\n  vendor_cookbook\nfi\n\n# This is necessary to find the cfn-init application\nexport PATH=/opt/aws/bin:${!PATH}\n[ -f /etc/parallelcluster/pcluster_cookbook_environment.sh ] && . /etc/parallelcluster/pcluster_cookbook_environment.sh\n\n$CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -s ${AWS::StackName} -v -c default -r LaunchTemplate --region \"${AWS::Region}\"\n",
         {
          "CustomCookbookUrl": {
           "Ref": "CustomCookbookUrl"

cloudformation/external-slurmdbd/resources/user_data.sh

@@ -19,14 +19,7 @@ function vendor_cookbook
 {
   mkdir /tmp/cookbooks
   cd /tmp/cookbooks
-  tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz
-  HOME_BAK="${!HOME}"
-  export HOME="/tmp"
-  for d in /tmp/cookbooks/*; do
-    cd "$d" || continue
-    LANG=en_US.UTF-8 /opt/cinc/embedded/bin/berks vendor /etc/chef/cookbooks --delete
-  done;
-  export HOME="${!HOME_BAK}"
+  tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz --strip-components 1
 }
 
 function wait_for_private_ip_assignment