Remote libvirt UPI installation multi-arch compute s390x and x86

This commit adds a remote libvirt UPI installation that installas a multi-architecture compute cluster on s390x and x86 and runs e2e tests against the resulting cluster. This is needed for https://issues.redhat.com/browse/MIXEDARCH-357. The commit also changes openshift-e2e-libvirt-test to run the tests also against the new cluster profile libvirt-s390x-amd64. The current implementation uses a static setup for exactly one cluster.
openshift · Feb 27, 2024 · b323043 · b323043
1 parent 0f1416b
commit b323043
Show file tree

Hide file tree

Showing 34 changed files with 1,200 additions and 1 deletion.
diff --git a/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.15.yaml b/ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.15.yaml
@@ -416,6 +416,19 @@ tests:
       NODE_TUNING: "true"
       TEST_TYPE: conformance-parallel
     workflow: openshift-e2e-libvirt
+- as: ocp-e2e-ovn-remote-libvirt-s390x-heterogeneous
+  cluster: build01
+  cron: 0 4,16 * * 1-5
+  steps:
+    cluster_profile: libvirt-s390x-amd64
+    dependencies:
+      OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:multi-latest
+    env:
+      ARCH: s390x
+      BRANCH: "4.15"
+      TEST_TYPE: conformance-parallel
+      USE_RAMFS: "true"
+    workflow: openshift-e2e-libvirt-upi-heterogeneous
 - as: ocp-heavy-build-ovn-remote-libvirt-s390x
   cluster: build01
   cron: 0 12 * * 1-5

diff --git a/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml b/ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml
@@ -15778,6 +15778,88 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build01
+  cron: 0 4,16 * * 1-5
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: master
+    org: openshift
+    repo: multiarch
+  labels:
+    ci-operator.openshift.io/cloud: libvirt-s390x-amd64
+    ci-operator.openshift.io/cloud-cluster-profile: libvirt-s390x-amd64
+    ci-operator.openshift.io/cluster: build01
+    ci-operator.openshift.io/variant: nightly-4.15
+    ci.openshift.io/generator: prowgen
+    job-release: "4.15"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-multiarch-master-nightly-4.15-ocp-e2e-ovn-remote-libvirt-s390x-heterogeneous
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --secret-dir=/usr/local/ocp-e2e-ovn-remote-libvirt-s390x-heterogeneous-cluster-profile
+      - --target=ocp-e2e-ovn-remote-libvirt-s390x-heterogeneous
+      - --variant=nightly-4.15
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /usr/local/ocp-e2e-ovn-remote-libvirt-s390x-heterogeneous-cluster-profile
+        name: cluster-profile
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: cluster-profile
+      secret:
+        secretName: cluster-secrets-libvirt-s390x-amd64
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build04
   decorate: true

diff --git a/ci-operator/step-registry/openshift/e2e/libvirt/test/openshift-e2e-libvirt-test-commands.sh b/ci-operator/step-registry/openshift/e2e/libvirt/test/openshift-e2e-libvirt-test-commands.sh
@@ -8,7 +8,7 @@ export PATH=/usr/libexec/origin:$PATH
 
 # Initial check
 case "${CLUSTER_TYPE}" in
-libvirt-ppc64le|libvirt-s390x|powervs*)
+libvirt-ppc64le|libvirt-s390x*|powervs*)
     ;;
 *)
     >&2 echo "Unsupported cluster type '${CLUSTER_TYPE}'"

diff --git a/ci-operator/step-registry/openshift/e2e/libvirt/upi/OWNERS b/ci-operator/step-registry/openshift/e2e/libvirt/upi/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- multiarch-approvers
+reviewers:
+- multiarch-reviewers
diff --git a/ci-operator/step-registry/openshift/e2e/libvirt/upi/heterogeneous/OWNERS b/ci-operator/step-registry/openshift/e2e/libvirt/upi/heterogeneous/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- multiarch-approvers
+reviewers:
+- multiarch-reviewers
diff --git a/.../libvirt/upi/heterogeneous/openshift-e2e-libvirt-upi-heterogeneous-workflow.metadata.json b/.../libvirt/upi/heterogeneous/openshift-e2e-libvirt-upi-heterogeneous-workflow.metadata.json
@@ -0,0 +1,11 @@
+{
+	"path": "openshift/e2e/libvirt/upi/heterogeneous/openshift-e2e-libvirt-upi-heterogeneous-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"multiarch-approvers"
+		],
+		"reviewers": [
+			"multiarch-reviewers"
+		]
+	}
+}
diff --git a/...shift/e2e/libvirt/upi/heterogeneous/openshift-e2e-libvirt-upi-heterogeneous-workflow.yaml b/...shift/e2e/libvirt/upi/heterogeneous/openshift-e2e-libvirt-upi-heterogeneous-workflow.yaml
@@ -0,0 +1,13 @@
+workflow:
+  as: openshift-e2e-libvirt-upi-heterogeneous
+  steps:
+    pre:
+      - chain: upi-libvirt-pre
+      - ref: upi-libvirt-install-heterogeneous
+    test:
+      - chain: openshift-e2e-libvirt-test
+    post:
+      - chain: upi-libvirt-post
+  documentation: |-
+    The UPI workflow provides pre- and post- steps that provision and
+    deprovision a multi-architecture compute OpenShift cluster on libvirt.
diff --git a/ci-operator/step-registry/upi/libvirt/OWNERS b/ci-operator/step-registry/upi/libvirt/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- multiarch-approvers
+reviewers:
+- multiarch-reviewers
diff --git a/ci-operator/step-registry/upi/libvirt/conf/OWNERS b/ci-operator/step-registry/upi/libvirt/conf/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- multiarch-approvers
+reviewers:
+- multiarch-reviewers
diff --git a/ci-operator/step-registry/upi/libvirt/conf/upi-libvirt-conf-commands.sh b/ci-operator/step-registry/upi/libvirt/conf/upi-libvirt-conf-commands.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+
+echo "Installing from initial release ${RELEASE_IMAGE_LATEST}"
+
+openshift-install version
+
+CONFIG="${SHARED_DIR}/install-config.yaml"
+
+CLUSTER_NAME="libvirt-s390x-amd64-0-0"
+BASE_DOMAIN="ci"
+
+cat >> "${CONFIG}" << EOF
+apiVersion: v1
+baseDomain: ${BASE_DOMAIN}
+metadata:
+  name: ${CLUSTER_NAME}
+controlPlane:
+  architecture: ${ARCH}
+  hyperthreading: Enabled
+  name: master
+  replicas: ${MASTER_REPLICAS}
+compute:
+- architecture: ${ARCH}
+  hyperthreading: Enabled
+  name: worker
+  replicas: 0
+networking:
+  clusterNetwork:
+  - cidr: 10.128.0.0/14
+    hostPrefix: 23
+  networkType: OVNKubernetes
+  serviceNetwork:
+  - 172.30.0.0/16
+platform:
+  none: {}
+fips: false
+pullSecret: >
+  $(<"${CLUSTER_PROFILE_DIR}/pull-secret")
+sshKey: |
+  $(<"${CLUSTER_PROFILE_DIR}/ssh-publickey")
+EOF
diff --git a/ci-operator/step-registry/upi/libvirt/conf/upi-libvirt-conf-ref.metadata.json b/ci-operator/step-registry/upi/libvirt/conf/upi-libvirt-conf-ref.metadata.json
@@ -0,0 +1,11 @@
+{
+	"path": "upi/libvirt/conf/upi-libvirt-conf-ref.yaml",
+	"owners": {
+		"approvers": [
+			"multiarch-approvers"
+		],
+		"reviewers": [
+			"multiarch-reviewers"
+		]
+	}
+}
diff --git a/ci-operator/step-registry/upi/libvirt/conf/upi-libvirt-conf-ref.yaml b/ci-operator/step-registry/upi/libvirt/conf/upi-libvirt-conf-ref.yaml
@@ -0,0 +1,27 @@
+ref:
+  as: upi-libvirt-conf
+  from: libvirt-installer
+  commands: upi-libvirt-conf-commands.sh
+  env:
+  - name: ARCH
+  - name: BRANCH
+  - name: MASTER_REPLICAS
+    default: "3"
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  dependencies:
+    - name: "release:latest"
+      env: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE
+    - name: "release:latest"
+      env: RELEASE_IMAGE_LATEST
+  dnsConfig:
+    nameservers:
+    - 172.30.38.188
+    searches:
+    - "bastion-z.svc.cluster.local"
+    - "bastion-ppc64le-libvirt.svc.cluster.local"
+  documentation: >-
+    Generates the install-config.yaml file for a UPI installation with platform: none
+    and compute replicas: 0.
diff --git a/ci-operator/step-registry/upi/libvirt/deprovision/OWNERS b/ci-operator/step-registry/upi/libvirt/deprovision/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- multiarch-approvers
+reviewers:
+- multiarch-reviewers
diff --git a/ci-operator/step-registry/upi/libvirt/deprovision/upi-libvirt-deprovision-commands.sh b/ci-operator/step-registry/upi/libvirt/deprovision/upi-libvirt-deprovision-commands.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+LIBVIRT_DOMAIN_NAME_SUFFIX="libvirt-s390x-amd64-0-0-ci"
+
+export LIBVIRT_DEFAULT_URI="qemu+tcp://lnxocp10:16509/system"
+
+DOMAINS_TO_DESTROY=$(virsh list --all --name | grep ${LIBVIRT_DOMAIN_NAME_SUFFIX})
+for domain in $DOMAINS_TO_DESTROY; do
+    virsh undefine "${domain}"
+    virsh destroy "${domain}"
+    virsh vol-delete --pool images "${domain}".qcow2
+done
+
+export LIBVIRT_DEFAULT_URI="qemu+tcp://xkvmocp04:16510/system"
+
+DOMAINS_TO_DESTROY=$(virsh list --all --name | grep ${LIBVIRT_DOMAIN_NAME_SUFFIX})
+for domain in $DOMAINS_TO_DESTROY; do
+    virsh undefine "${domain}"
+    virsh destroy "${domain}"
+    virsh vol-delete --pool default "${domain}".qcow2
+done
diff --git a/ci-operator/step-registry/upi/libvirt/deprovision/upi-libvirt-deprovision-ref.metadata.json b/ci-operator/step-registry/upi/libvirt/deprovision/upi-libvirt-deprovision-ref.metadata.json
@@ -0,0 +1,11 @@
+{
+	"path": "upi/libvirt/deprovision/upi-libvirt-deprovision-ref.yaml",
+	"owners": {
+		"approvers": [
+			"multiarch-approvers"
+		],
+		"reviewers": [
+			"multiarch-reviewers"
+		]
+	}
+}
diff --git a/ci-operator/step-registry/upi/libvirt/deprovision/upi-libvirt-deprovision-ref.yaml b/ci-operator/step-registry/upi/libvirt/deprovision/upi-libvirt-deprovision-ref.yaml
@@ -0,0 +1,18 @@
+ref:
+  as: upi-libvirt-deprovision
+  from: libvirt-installer
+  grace_period: 10m
+  commands: upi-libvirt-deprovision-commands.sh
+  resources:
+    requests:
+      cpu: 1000m
+      memory: 300Mi
+  dnsConfig:
+    nameservers:
+    - 172.30.38.188
+    searches:
+    - "bastion-z.svc.cluster.local"
+    - "bastion-ppc64le-libvirt.svc.cluster.local"
+  documentation: |-
+    Deprovisions the libvirt guests created by upi-libvirt-install and
+    upi-libvirt-install-heterogeneous.
diff --git a/ci-operator/step-registry/upi/libvirt/install/OWNERS b/ci-operator/step-registry/upi/libvirt/install/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- multiarch-approvers
+reviewers:
+- multiarch-reviewers
diff --git a/ci-operator/step-registry/upi/libvirt/install/heterogeneous/OWNERS b/ci-operator/step-registry/upi/libvirt/install/heterogeneous/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- multiarch-approvers
+reviewers:
+- multiarch-reviewers