MULTIARCH-3744 Add CI job of SNO with assisted installer for Power

Signed-off-by: CS Zhang <zhangcho@us.ibm.com>

ci-operator/config/openshift/multiarch/openshift-multiarch-master__nightly-4.15.yaml

@@ -682,6 +682,13 @@ tests:
       OCP_ARCH: amd64
       TEST_SUITE: upgrade-conformance
     workflow: openshift-upgrade-gcp-heterogeneous
+- as: ocp-e2e-assisted-sno-power
+  cron: 0 8 * * *
+  steps:
+    env:
+      INSTALL_TYPE: assisted
+      OCP_VERSION: "4.15"
+    workflow: upi-powervs-sno
 zz_generated_metadata:
   branch: master
   org: openshift

ci-operator/jobs/openshift/multiarch/openshift-multiarch-master-periodics.yaml

@@ -14233,6 +14233,69 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build01
+  cron: 0 8 * * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: master
+    org: openshift
+    repo: multiarch
+  labels:
+    ci-operator.openshift.io/variant: nightly-4.15
+    ci.openshift.io/generator: prowgen
+    job-release: "4.15"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-multiarch-master-nightly-4.15-ocp-e2e-assisted-sno-power
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=ocp-e2e-assisted-sno-power
+      - --variant=nightly-4.15
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build01
   decorate: true

ci-operator/step-registry/upi/install/powervs/sno/upi-install-powervs-sno-commands.sh

@@ -91,8 +91,15 @@ SSH_OPTIONS=(-o 'PreferredAuthentications=publickey' -o 'StrictHostKeyChecking=n
 # https://codeload.github.com/ppc64le-cloud/ocp-sno-hacks/zip/refs/heads/main contains the scripts to setup the SNO cluster
 ssh "${SSH_OPTIONS[@]}" root@${BASTION} "mkdir ${BASTION_CI_SCRIPTS_DIR} && cd ${BASTION_CI_SCRIPTS_DIR} && curl https://codeload.github.com/ppc64le-cloud/ocp-ci-hacks/zip/refs/heads/main -o ocp-ci-hacks.zip && unzip ocp-ci-hacks.zip && mv ocp-ci-hacks-main/sno/* ${BASTION_CI_SCRIPTS_DIR} && rm -rf ocp-ci-hacks.zip ocp-ci-hacks-main"
 
+# Save private-key, pull-secret and offline-token to bastion
+ssh "${SSH_OPTIONS[@]}" root@${BASTION} "mkdir -p ~/.sno"
+scp "${SSH_OPTIONS[@]}" /etc/sno-power-credentials/{ssh-publickey,pull-secret,offline-token} root@${BASTION}:~/.sno/.
+
+# set the default INSTALL_TYPE to sno
+INSTALL_TYPE=${INSTALL_TYPE:-sno}
+
 # Setting up the SNO config, generating the ignition and network boot on the bastion
-ssh "${SSH_OPTIONS[@]}" root@${BASTION} "cd ${BASTION_CI_SCRIPTS_DIR} && ./setup-sno.sh ${CLUSTER_NAME} ${BASE_DOMAIN} ${POWERVS_MACHINE_NETWORK_CIDR} ${INSTALLATION_DISK} $(eval "echo ${LIVE_ROOTFS_URL}") $(eval "echo ${LIVE_KERNEL_URL}") $(eval "echo ${LIVE_INITRAMFS_URL}") $(printf "http://%s" "${BASTION}") ${MAC_ADDRESS} ${IP_ADDRESS}"
+ssh "${SSH_OPTIONS[@]}" root@${BASTION} "cd ${BASTION_CI_SCRIPTS_DIR} && ./setup-sno.sh ${CLUSTER_NAME} ${BASE_DOMAIN} ${POWERVS_MACHINE_NETWORK_CIDR} ${INSTALLATION_DISK} $(eval "echo ${LIVE_ROOTFS_URL}") $(eval "echo ${LIVE_KERNEL_URL}") $(eval "echo ${LIVE_INITRAMFS_URL}") $(printf "http://%s" "${BASTION}") ${MAC_ADDRESS} ${IP_ADDRESS} ${INSTALL_TYPE} ${OCP_VERSION}"
 
 # Creating dns records in ibmcloud cis service for SNO node to reach hosted cluster and for ingress purpose
 ibmcloud cis dns-record-create ${CIS_DOMAIN_ID} --type A --name "api.${CLUSTER_NAME}" --content "${IP_ADDRESS}"
@@ -104,16 +111,11 @@ sleep 180
 # Rebooting the node to boot from net
 ibmcloud pi instance-soft-reboot $instance_id
 
-# Updating the boot disk of SNO node to volume attached to VSI
-ssh "${SSH_OPTIONS[@]}" root@${BASTION} "cd ${BASTION_CI_SCRIPTS_DIR} && ./update-boot-disk.sh ${IP_ADDRESS} ${INSTALLATION_DISK}" &
-
-# Run bootstrap-complete
-# Sometimes bootstrap-complete may timeout, but still install-complete can be tried so set +e
-set +e
-ssh "${SSH_OPTIONS[@]}" root@${BASTION} "openshift-install --dir=${BASTION_CI_SCRIPTS_DIR} wait-for bootstrap-complete --log-level debug"
+# Updating the boot disk of SNO node to volume attached to VSI, it required only for normal SNO installastion.
+if [[ ${INSTALL_TYPE} == "sno" ]]; then
+    ssh "${SSH_OPTIONS[@]}" root@${BASTION} "cd ${BASTION_CI_SCRIPTS_DIR} && ./update-boot-disk.sh ${IP_ADDRESS} ${INSTALLATION_DISK}" &
+fi
 
-# set -e to capture on install-complete command
-set -e
+# Run wait-sno-complete
+ssh "${SSH_OPTIONS[@]}" root@${BASTION} "cd ${BASTION_CI_SCRIPTS_DIR} && ./wait-sno-complete.sh ${CLUSTER_NAME} ${INSTALL_TYPE}"
 
-# Run install-complete
-ssh "${SSH_OPTIONS[@]}" root@${BASTION} "openshift-install --dir=${BASTION_CI_SCRIPTS_DIR} wait-for install-complete --log-level debug"

ci-operator/step-registry/upi/install/powervs/sno/upi-install-powervs-sno-ref.yaml

@@ -10,6 +10,9 @@ ref:
   env:
     - name: OCP_VERSION
       documentation: OCP Version to use to create the SNO cluster
+    - name: INSTALL_TYPE
+      default: "sno"
+      documentation: The installation type of SNO, supported value is sno, agent, assisted
     - name: BASE_DOMAIN
       documentation: The cluster's FQDN will be a subdomain of the base domain
     - name: BASTION