add vsphere ELB config

openshift · Mar 27, 2024 · fbf38ad · fbf38ad
1 parent 441ab8d
commit fbf38ad
Show file tree

Hide file tree

Showing 4 changed files with 142 additions and 2 deletions.
diff --git a/...penshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml b/...penshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml
@@ -2658,6 +2658,19 @@ tests:
     test:
     - chain: openshift-e2e-test-qe-destructive
     workflow: cucushift-installer-rehearse-vsphere-ipi-zones-multisubnets-external-lb
+- as: vsphere-ipi-zones-multisubnets-external-lb-usermanaged-f28
+  cron: 8 23 15 * *
+  steps:
+    cluster_profile: vsphere-multizone-2
+    env:
+      LB_TYPE: UserManaged
+    leases:
+    - count: 2
+      env: VSPHERE_EXTRA_LEASED_RESOURCE
+      resource_type: vsphere-multizone-2-quota-slice
+    test:
+    - chain: openshift-e2e-test-qe
+    workflow: cucushift-installer-rehearse-vsphere-ipi-zones-multisubnets-external-lb
 - as: vsphere-ipi-zones-tp-f28
   cron: 35 3 3 * *
   steps:

diff --git a/...ift/openshift-tests-private/openshift-openshift-tests-private-release-4.16-periodics.yaml b/...ift/openshift-tests-private/openshift-openshift-tests-private-release-4.16-periodics.yaml
@@ -31697,6 +31697,94 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: vsphere02
+  cron: 8 23 15 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.16
+    org: openshift
+    repo: openshift-tests-private
+  labels:
+    ci-operator.openshift.io/cloud: vsphere
+    ci-operator.openshift.io/cloud-cluster-profile: vsphere-multizone-2
+    ci-operator.openshift.io/variant: amd64-nightly
+    ci.openshift.io/generator: prowgen
+    job-release: "4.16"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-openshift-tests-private-release-4.16-amd64-nightly-vsphere-ipi-zones-multisubnets-external-lb-usermanaged-f28
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --secret-dir=/usr/local/vsphere-ipi-zones-multisubnets-external-lb-usermanaged-f28-cluster-profile
+      - --target=vsphere-ipi-zones-multisubnets-external-lb-usermanaged-f28
+      - --variant=amd64-nightly
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /usr/local/vsphere-ipi-zones-multisubnets-external-lb-usermanaged-f28-cluster-profile
+        name: cluster-profile
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: cluster-profile
+      secret:
+        secretName: cluster-secrets-vsphere
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: vsphere02
   cron: 35 3 3 * *

diff --git a/...istry/ipi/conf/vsphere/zones/multisubnets/ipi-conf-vsphere-zones-multisubnets-commands.sh b/...istry/ipi/conf/vsphere/zones/multisubnets/ipi-conf-vsphere-zones-multisubnets-commands.sh
@@ -20,6 +20,26 @@ declare vsphere_extra_portgroup_2
 source "${SHARED_DIR}/vsphere_context.sh"
 # shellcheck source=/dev/null
 source "${SHARED_DIR}/govc.sh"
+declare -a vips
+mapfile -t vips < "${SHARED_DIR}"/vips.txt
+#APIVIP=${vips[0]}
+#INGRESSVIP=${vips[1]}
+
+if [[ ${LB_TYPE} == "UserManaged" ]]; then
+    APIVIPS_DEF="apiVIPs:
+      - ${vips[0]}"
+    INGRESSVIPS_DEF="ingressVIPs:
+      - ${vips[1]}"
+    LB_TYPE_DEF="loadBalancer:
+      type: UserManaged"
+else
+    APIVIPS_DEF=""
+    INGRESSVIPS_DEF=""
+    LB_TYPE_DEF=""
+fi
+
+#export APIVIP
+#export INGRESSVIP
 
 CONFIG="${SHARED_DIR}/install-config.yaml"
 base_domain=$(<"${SHARED_DIR}"/basedomain.txt)
@@ -51,8 +71,9 @@ compute:
        - "us-east-3"
 platform:
   vsphere:
-    apiVIP:
-    ingressVIP:
+    ${LB_TYPE_DEF}
+    ${APIVIPS_DEF}
+    ${INGRESSVIPS_DEF}
     vCenter: "${vsphere_url}"
     username: "${GOVC_USERNAME}"
     password: ${GOVC_PASSWORD}
@@ -99,6 +120,19 @@ networking:
   machineNetwork:
   - cidr: "${machine_cidr}"
 EOF
+
+# if loadbalancer is UserManaged, it's mean using external LB,
+# then keepalived and haproxy will not deployed, but coredns still keep
+
+<<com
+if [ ${LB_TYPE} = "UserManaged" ]; then
+  if [ ! -f /tmp/yq ]; then
+    curl -L https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq
+  fi
+  /tmp/yq e --inplace '.platform.vsphere.apiVIPs += [strenv(APIVIP)],.platform.vsphere.ingressVIPs += [strenv(INGRESSVIP)]' ${CONFIG}
+  /tmp/yq e --inplace '.platform.vsphere.loadBalancer.type += "UserManaged"' ${CONFIG}
+fi
+com
 # TODO: Add this back in once we have an vsphere
 # environment that will support topology storage
 

diff --git a/...registry/ipi/conf/vsphere/zones/multisubnets/ipi-conf-vsphere-zones-multisubnets-ref.yaml b/...registry/ipi/conf/vsphere/zones/multisubnets/ipi-conf-vsphere-zones-multisubnets-ref.yaml
@@ -2,6 +2,11 @@ ref:
   as: ipi-conf-vsphere-zones-multisubnets
   from: upi-installer
   commands: ipi-conf-vsphere-zones-multisubnets-commands.sh
+  env:
+  - name: LB_TYPE
+    default: OpenShiftManagedDefault
+    documentation: |-
+       There are two values: OpenShiftManagedDefault and UserManaged, if UserManaged will use external LB.
   resources:
     requests:
       cpu: 10m