vSphere vCenter: 500 maximum session per user fix

vCenter only supports 500 sessions per user.
This is causing REST calls to fail (and jobs)

ci-operator/step-registry/ipi/conf/vsphere/check/ipi-conf-vsphere-check-commands.sh

@@ -17,23 +17,30 @@ cloud_where_run="VMC"
 dns_server="10.0.0.2"
 vsphere_resource_pool=""
 vsphere_url="vcenter.sddc-44-236-21-251.vmwarevmc.com"
-TFVARS_PATH=/var/run/vault/vsphere/secret.auto.tfvars
+
+VCENTER_AUTH_PATH=/var/run/vault/vsphere/secrets.sh
 
 # For leases >= than 88, run on the IBM Cloud
-if [ $((${LEASED_RESOURCE//[!0-9]/})) -ge 88 ]; then     
+if [ $((${LEASED_RESOURCE//[!0-9]/})) -ge 88 ]; then
   echo Scheduling job on IBM Cloud instance
-  TFVARS_PATH=/var/run/vault/ibmcloud/secret.auto.tfvars
+  VCENTER_AUTH_PATH=/var/run/vault/ibmcloud/secrets.sh
   vsphere_url="ibmvcenter.vmc-ci.devcluster.openshift.com"
-  vsphere_datacenter="IBMCloud"  
+  vsphere_datacenter="IBMCloud"
   cloud_where_run="IBM"
   dns_server="10.38.76.172"
   vsphere_resource_pool="/IBMCloud/host/vcs-ci-workload/Resources"
   vsphere_cluster="vcs-ci-workload"
   vsphere_datastore="vsanDatastore"
 fi
 
-vsphere_user=$(grep -oP 'vsphere_user\s*=\s*"\K[^"]+' ${TFVARS_PATH})
-vsphere_password=$(grep -oP 'vsphere_password\s*=\s*"\K[^"]+' ${TFVARS_PATH})
+declare vcenter_usernames
+declare vcenter_passwords
+# shellcheck source=/dev/null
+source "${VCENTER_AUTH_PATH}"
+
+account_loc=$(($RANDOM % 4))
+vsphere_user="${vcenter_usernames[$account_loc]}"
+vsphere_password="${vcenter_passwords[$account_loc]}"
 
 echo "$(date -u --rfc-3339=seconds) - Creating govc.sh file..."
 cat >> "${SHARED_DIR}/govc.sh" << EOF
@@ -43,17 +50,17 @@ export GOVC_PASSWORD="${vsphere_password}"
 export GOVC_INSECURE=1
 export GOVC_DATACENTER="${vsphere_datacenter}"
 export GOVC_DATASTORE="${vsphere_datastore}"
+export GOVC_RESOURCE_POOL=${vsphere_resource_pool}
 EOF
 
 echo "$(date -u --rfc-3339=seconds) - Creating vsphere_context.sh file..."
 cat >> "${SHARED_DIR}/vsphere_context.sh" << EOF
-export TFVARS_PATH="${TFVARS_PATH}"
 export vsphere_url="${vsphere_url}"
 export vsphere_cluster="${vsphere_cluster}"
 export vsphere_resource_pool="${vsphere_resource_pool}"
 export dns_server="${dns_server}"
 export cloud_where_run="${cloud_where_run}"
-export vsphere_datacenter="${vsphere_datacenter}"  
+export vsphere_datacenter="${vsphere_datacenter}"
 export vsphere_datastore="${vsphere_datastore}"
 EOF
 

ci-operator/step-registry/ipi/conf/vsphere/ipi-conf-vsphere-commands.sh

@@ -16,15 +16,14 @@ declare vsphere_datacenter
 declare vsphere_datastore
 declare vsphere_url
 declare vsphere_cluster
-declare TFVARS_PATH
 source "${SHARED_DIR}/vsphere_context.sh"
+# shellcheck source=/dev/null
+source "${SHARED_DIR}/govc.sh"
 
 declare -a vips
 mapfile -t vips < "${SHARED_DIR}/vips.txt"
 
 CONFIG="${SHARED_DIR}/install-config.yaml"
-vsphere_user=$(grep -oP 'vsphere_user\s*=\s*"\K[^"]+' ${TFVARS_PATH})
-vsphere_password=$(grep -oP 'vsphere_password\s*=\s*"\K[^"]+' ${TFVARS_PATH})
 base_domain=$(<"${SHARED_DIR}"/basedomain.txt)
 machine_cidr=$(<"${SHARED_DIR}"/machinecidr.txt)
 
@@ -54,8 +53,8 @@ platform:
     defaultDatastore: "${vsphere_datastore}"
     cluster: "${vsphere_cluster}"
     network: "${LEASED_RESOURCE}"
-    password: "${vsphere_password}"
-    username: "${vsphere_user}"
+    password: "${GOVC_PASSWORD}"
+    username: "${GOVC_USERNAME}"
     apiVIP: "${vips[0]}"
     ingressVIP: "${vips[1]}"
 networking:

ci-operator/step-registry/ipi/deprovision/vsphere/diags/ipi-deprovision-vsphere-diags-commands.sh

@@ -11,43 +11,27 @@ echo "$(date -u --rfc-3339=seconds) - Collecting vCenter performance data and al
 
 echo "$(date -u --rfc-3339=seconds) - sourcing context from vsphere_context.sh..."
 # shellcheck source=/dev/null
-declare vsphere_datacenter
-declare vsphere_datastore
 declare cloud_where_run
-declare vsphere_url
 source "${SHARED_DIR}/vsphere_context.sh"
 
-vsphere_user=$(grep -oP 'vsphere_user\s*=\s*"\K[^"]+' ${TFVARS_PATH})
-vsphere_password=$(grep -oP 'vsphere_password\s*=\s*"\K[^"]+' ${TFVARS_PATH})
-
-echo "$(date -u --rfc-3339=seconds) - Creating govc.sh file..."
-cat >> "${SHARED_DIR}/govc.sh" << EOF
-export GOVC_URL="${vsphere_url}"
-export GOVC_USERNAME="${vsphere_user}"
-export GOVC_PASSWORD="${vsphere_password}"
-export GOVC_INSECURE=1
-export GOVC_DATACENTER="${vsphere_datacenter}"
-export GOVC_DATASTORE="${vsphere_datastore}"
-EOF
-
 function collect_diagnostic_data {
-  set +e  
+  set +e
   source "${SHARED_DIR}/govc.sh"
   vm_path="/${GOVC_DATACENTER}/vm/${cluster_name}"
   vcenter_state=${ARTIFACT_DIR}/vcenter_state
   mkdir ${vcenter_state}
 
-  
-  govc object.collect "/${GOVC_DATACENTER}/host" triggeredAlarmState &> ${vcenter_state}/host_alarms.log  
+
+  govc object.collect "/${GOVC_DATACENTER}/host" triggeredAlarmState &> ${vcenter_state}/host_alarms.log
   clustervms=$(govc ls "${vm_path}-*")
-  for vm in $clustervms; do    
+  for vm in $clustervms; do
     vmname=$(echo $vm | rev | cut -d'/' -f 1 | rev)
     echo Collecting alarms from $vm
     govc object.collect $vm triggeredAlarmState &> ${vcenter_state}/${vmname}_alarms.log
     echo Collecting metrics from $vm
     METRICS=$(govc metric.ls $vm)
-    govc metric.sample -json -n 60 $vm $METRICS &> ${vcenter_state}/${vmname}_metrics.json    
-    echo "$(date -u --rfc-3339=seconds) - capture console image from $vm"    
+    govc metric.sample -json -n 60 $vm $METRICS &> ${vcenter_state}/${vmname}_metrics.json
+    echo "$(date -u --rfc-3339=seconds) - capture console image from $vm"
     govc vm.console -vm.ipath="$vm" -capture "${vcenter_state}/${vmname}.png"
   done
   first_vm=$(echo ${clustervms} | cut -d" " -f1)

ci-operator/step-registry/upi/conf/vsphere/upi-conf-vsphere-commands.sh

@@ -84,25 +84,12 @@ echo "export target_hw_version=${target_hw_version}" >> ${SHARED_DIR}/vsphere_co
 declare vsphere_datacenter
 declare vsphere_datastore
 declare vsphere_cluster
-declare vsphere_resource_pool
 declare dns_server
 declare vsphere_url
-declare TFVARS_PATH
 source "${SHARED_DIR}/vsphere_context.sh"
 
-vsphere_user=$(grep -oP 'vsphere_user\s*=\s*"\K[^"]+' ${TFVARS_PATH})
-vsphere_password=$(grep -oP 'vsphere_password\s*=\s*"\K[^"]+' ${TFVARS_PATH})
-
-echo "$(date -u --rfc-3339=seconds) - Creating govc.sh file..."
-cat >> "${SHARED_DIR}/govc.sh" << EOF
-export GOVC_URL="${vsphere_url}"
-export GOVC_USERNAME="${vsphere_user}"
-export GOVC_PASSWORD="${vsphere_password}"
-export GOVC_INSECURE=1
-export GOVC_RESOURCE_POOL=${vsphere_resource_pool}
-export GOVC_DATACENTER="${vsphere_datacenter}"
-export GOVC_DATASTORE="${vsphere_datastore}"
-EOF
+# shellcheck source=/dev/null
+source "${SHARED_DIR}/govc.sh"
 
 echo "$(date -u --rfc-3339=seconds) - Extend install-config.yaml ..."
 
@@ -121,8 +108,8 @@ platform:
     defaultDatastore: "${vsphere_datastore}"
     cluster: "${vsphere_cluster}"
     network: "${LEASED_RESOURCE}"
-    password: "${vsphere_password}"
-    username: "${vsphere_user}"
+    password: "${GOVC_PASSWORD}"
+    username: "${GOVC_USERNAME}"
     folder: "/${vsphere_datacenter}/vm/${cluster_name}"
 EOF
 
@@ -149,6 +136,13 @@ compute_ip_addresses = ["192.168.${third_octet}.7","192.168.${third_octet}.8","1
 control_plane_ip_addresses = ["192.168.${third_octet}.4","192.168.${third_octet}.5","192.168.${third_octet}.6"]
 EOF
 
+echo "$(date -u --rfc-3339=seconds) - Create secrets.auto.tfvars..."
+cat > "${SHARED_DIR}/secrets.auto.tfvars" <<-EOF
+vsphere_password="${GOVC_PASSWORD}"
+vsphere_user="${GOVC_USERNAME}"
+ipam_token=""
+EOF
+
 dir=/tmp/installer
 mkdir "${dir}/"
 pushd ${dir}

ci-operator/step-registry/upi/deprovision/vsphere/upi-deprovision-vsphere-commands.sh

@@ -55,6 +55,7 @@ cp -t "${installer_dir}" \
     "${SHARED_DIR}/install-config.yaml" \
     "${SHARED_DIR}/metadata.json" \
     "${SHARED_DIR}/terraform.tfvars" \
+    "${SHARED_DIR}/secrets.auto.tfvars" \
     "${SHARED_DIR}/bootstrap.ign" \
     "${SHARED_DIR}/worker.ign" \
     "${SHARED_DIR}/master.ign"
@@ -67,10 +68,6 @@ cp -t "${installer_dir}/auth" \
 cp -rt "${installer_dir}" \
     /var/lib/openshift-install/upi/"${CLUSTER_TYPE}"/*
 
-# Copy secrets to terraform path
-cp -t "${installer_dir}" \
-    ${TFVARS_PATH}
-
 tar -xf "${SHARED_DIR}/terraform_state.tar.xz"
 
 rm -rf .terraform || true

ci-operator/step-registry/upi/install/vsphere/upi-install-vsphere-commands.sh

@@ -28,6 +28,7 @@ cp -t "${installer_dir}" \
     "${SHARED_DIR}/install-config.yaml" \
     "${SHARED_DIR}/metadata.json" \
     "${SHARED_DIR}/terraform.tfvars" \
+    "${SHARED_DIR}/secrets.auto.tfvars" \
     "${SHARED_DIR}/bootstrap.ign" \
     "${SHARED_DIR}/worker.ign" \
     "${SHARED_DIR}/master.ign"
@@ -40,10 +41,6 @@ cp -t "${installer_dir}/auth" \
 cp -rt "${installer_dir}" \
     /var/lib/openshift-install/upi/"${CLUSTER_TYPE}"/*
 
-# Copy secrets to terraform path
-cp -t "${installer_dir}" \
-    ${TFVARS_PATH}
-
 export KUBECONFIG="${installer_dir}/auth/kubeconfig"
 
 function gather_console_and_bootstrap() {
@@ -110,7 +107,7 @@ function update_image_registry() {
 }
 
 function setE2eMirror() {
-  
+
   oc create -f - <<EOF
 apiVersion: machineconfiguration.openshift.io/v1
 kind: MachineConfig