add e2e-kubevirt-mce-baremetalds-disconnected-conformance

add e2e-kubevirt-mce-baremetalds-disconnected-conformance
based on the work for #44946
and #46584

Signed-off-by: Simone Tiraboschi <stirabos@redhat.com>

ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.14__periodics.yaml

@@ -219,6 +219,15 @@ tests:
       MCE_VERSION: "2.4"
       ODF_OPERATOR_SUB_CHANNEL: stable-4.14
     workflow: hypershift-mce-kubevirt-baremetalds-conformance
+- as: e2e-kubevirt-mce-baremetalds-disconnected-conformance
+  cron: 0 8 * * *
+  steps:
+    cluster_profile: equinix-ocp-metal
+    env:
+      HYPERSHIFT_NODE_COUNT: "2"
+      MCE_VERSION: "2.4"
+      ODF_OPERATOR_SUB_CHANNEL: stable-4.14
+    workflow: hypershift-mce-kubevirt-baremetalds-disconnected-conformance
 - as: e2e-mce-power-conformance
   cron: 0 4 * * *
   steps:

ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-4.14-periodics.yaml

@@ -951,6 +951,87 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build03
+  cron: 0 8 * * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.14
+    org: openshift
+    repo: hypershift
+  labels:
+    ci-operator.openshift.io/cloud: equinix-ocp-metal
+    ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal
+    ci-operator.openshift.io/variant: periodics
+    ci.openshift.io/generator: prowgen
+    job-release: "4.14"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-hypershift-release-4.14-periodics-e2e-kubevirt-mce-baremetalds-disconnected-conformance
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --secret-dir=/usr/local/e2e-kubevirt-mce-baremetalds-disconnected-conformance-cluster-profile
+      - --target=e2e-kubevirt-mce-baremetalds-disconnected-conformance
+      - --variant=periodics
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /usr/local/e2e-kubevirt-mce-baremetalds-disconnected-conformance-cluster-profile
+        name: cluster-profile
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: cluster-profile
+      secret:
+        secretName: cluster-secrets-equinix-ocp-metal
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build05
   cron: 0 4 * * *

ci-operator/step-registry/hypershift/conformance/hypershift-conformance-chain.yaml

@@ -84,7 +84,13 @@ chain:
           export IC_API_KEY
           ;;
       kubevirt)
-          export TEST_PROVIDER='{"type":"kubevirt"}'
+          if [[ "${DISCONNECTED}" == "true" ]];
+          then
+            mirror_test_images
+            export TEST_PROVIDER='{"type":"kubevirt","disconnected":true}'
+          else
+            export TEST_PROVIDER='{"type":"kubevirt"}'
+          fi
 
           # This looks counterintuitive. Here's an explanation.
           #
@@ -97,7 +103,7 @@ chain:
           #
           # This might not be the best long term solution, but these older KubeVirt platform
           # tests won't be maintained for long and are not a part of any GA.
-          
+
           PIPE="\|"
           # if $TEST_SKIPS is empty then $PIPE should be empty too
           if [[ -z "$TEST_SKIPS" ]]; then

ci-operator/step-registry/hypershift/kubevirt/baremetalds/redhat-operators/hypershift-kubevirt-baremetalds-redhat-operators-commands.sh

@@ -5,60 +5,183 @@ set -o errexit
 set -o pipefail
 set -x
 
-name="redhat-operators-$(echo $REDHAT_OPERATORS_INDEX_TAG| sed "s/[.]/-/g")"
-
-oc apply -f - <<EOF
-apiVersion: operators.coreos.com/v1alpha1
-kind: CatalogSource
-metadata:
-  annotations:
-    operatorframework.io/managed-by: marketplace-operator
-    target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}'
-  generation: 5
-  name: $name
-  namespace: openshift-marketplace
-spec:
-  displayName: Red Hat Operators
-  grpcPodConfig:
-    nodeSelector:
-      kubernetes.io/os: linux
-      node-role.kubernetes.io/master: ""
-    priorityClassName: system-cluster-critical
-    securityContextConfig: restricted
-    tolerations:
-    - effect: NoSchedule
-      key: node-role.kubernetes.io/master
-      operator: Exists
-    - effect: NoExecute
-      key: node.kubernetes.io/unreachable
-      operator: Exists
-      tolerationSeconds: 120
-    - effect: NoExecute
-      key: node.kubernetes.io/not-ready
-      operator: Exists
-      tolerationSeconds: 120
-  icon:
-    base64data: ""
-    mediatype: ""
-  image: registry.redhat.io/redhat/redhat-operator-index:${REDHAT_OPERATORS_INDEX_TAG}
-  priority: -100
-  publisher: Red Hat
-  sourceType: grpc
-  updateStrategy:
-    registryPoll:
-      interval: 10m
-EOF
+function mirror_odf() {
+    echo "### Mirroring ODF images"
+    source "${SHARED_DIR}/packet-conf.sh"
+
+    echo "registry.redhat.io/redhat/redhat-operator-index:${REDHAT_OPERATORS_INDEX_TAG}" > /tmp/odf-catalog-image
+    scp "${SSHOPTS[@]}" "/tmp/odf-catalog-image" "root@${IP}:/home/odf-catalog-image"
+    echo "${REDHAT_OPERATORS_INDEX_TAG}" > /tmp/odf-version
+    scp "${SSHOPTS[@]}" "/tmp/odf-version" "root@${IP}:/home/odf-version"
+    echo "${ODF_OPERATOR_SUB_PACKAGE}" > /tmp/odf-package
+    scp "${SSHOPTS[@]}" "/tmp/odf-package" "root@${IP}:/home/odf-package"
+    echo "${ODF_OPERATOR_SUB_CHANNEL}" > /tmp/odf-channel
+    scp "${SSHOPTS[@]}" "/tmp/odf-channel" "root@${IP}:/home/odf-channel"
+
+    # shellcheck disable=SC2087
+    ssh "${SSHOPTS[@]}" "root@${IP}" bash - << EOF
+    set -xeo pipefail
+
+    ODF_CATALOG_IMAGE=\$(cat /home/odf-catalog-image)
+    ODF_VERSION=\$(cat /home/odf-version)
+    ODF_OPERATOR_SUB_PACKAGE=\$(cat /home/odf-package)
+    ODF_OPERATOR_SUB_CHANNEL=\$(cat /home/odf-channel)
+
+    echo "1. Get mirror registry"
+    mirror_registry=\$(oc get imagecontentsourcepolicy -o json | jq -r '.items[].spec.repositoryDigestMirrors[0].mirrors[0]')
+    mirror_registry=\${mirror_registry%%/*}
+    if [[ \$mirror_registry == "" ]] ; then
+        echo "Warning: Can not find the mirror registry, abort !!!"
+        exit 1
+    fi
+    echo "mirror registry is \${mirror_registry}"
 
-for i in $(seq 1 120); do
-    state=$(oc get catalogsources/$name -n openshift-marketplace -o jsonpath='{.status.connectionState.lastObservedState}')
-    echo $state
-    if [ "$state" == "READY" ] ; then
-        echo "Catalogsource created successfully after waiting $((5*i)) seconds"
-        echo "current state of catalogsource is \"$state\""
-        created=true
-        break
+    echo "2: get oc-mirror from candidate clients"
+    if [[ ! -f /home/oc-mirror ]]; then
+        MIRROR2URL="https://mirror2.openshift.com/pub/openshift-v4"
+        CLIENTURL="\${MIRROR2URL}"/x86_64/clients/ocp/candidate
+        curl -s -k -L "\${CLIENTURL}/oc-mirror.tar.gz" -o om.tar.gz && tar -C /home -xzvf om.tar.gz && rm -f om.tar.gz
+        if ls /home/oc-mirror > /dev/null ; then
+            chmod +x /home/oc-mirror
+        else
+            echo "Warning, can not find oc-mirror abort !!!"
+            exit 1
+        fi
     fi
-    sleep 5
-done
-[ "$created" = "true" ]
+    /home/oc-mirror version
+
+    echo "3: skopeo copy docker://\${ODF_CATALOG_IMAGE} oci:///home/odf-local-catalog --remove-signatures"
+    skopeo copy "docker://\${ODF_CATALOG_IMAGE}" "oci:///home/odf-local-catalog" --remove-signatures
+
+    echo "4: oc-mirror"
+    catalog_image="odf-local-catalog/odf-local-catalog"
+
+    cat <<END |tee "/home/registry.conf"
+    [[registry]]
+     location = "registry.stage.redhat.io"
+     insecure = true
+     blocked = false
+     mirror-by-digest-only = false
+     [[registry.mirror]]
+        location = "brew.registry.redhat.io"
+        insecure = true
+    [[registry]]
+     location = "registry-proxy.engineering.redhat.com/rh-osbs"
+     insecure = true
+     blocked = false
+     mirror-by-digest-only = false
+     [[registry.mirror]]
+        location = "brew.registry.redhat.io/rh-osbs"
+        insecure = true
+END
+
+    cat <<END |tee "/home/imageset-config.yaml"
+    kind: ImageSetConfiguration
+    apiVersion: mirror.openshift.io/v1alpha2
+    storageConfig:
+      local:
+        path: mirror
+    mirror:
+      operators:
+      - catalog: "oci:///home/odf-local-catalog"
+        targetCatalog: \${catalog_image}
+        targetTag: "\${ODF_VERSION}"
+        packages:
+        - name: \${ODF_OPERATOR_SUB_PACKAGE}
+          channels:
+          - name: \${ODF_OPERATOR_SUB_CHANNEL}
+        - name: odf-operator
+          channels:
+          - name: \${ODF_OPERATOR_SUB_CHANNEL}
+        - name: ocs-operator
+          channels:
+          - name: \${ODF_OPERATOR_SUB_CHANNEL}
+        - name: mcg-operator
+          channels:
+          - name: \${ODF_OPERATOR_SUB_CHANNEL}
+        - name: odf-csi-addons-operator
+          channels:
+          - name: \${ODF_OPERATOR_SUB_CHANNEL}
+END
+
+    pushd /home
+    # try at least 3 times to be sure to get all the images...
+    /home/oc-mirror --config "/home/imageset-config.yaml" docker://\${mirror_registry} --oci-registries-config="/home/registry.conf" --continue-on-error --skip-missing
+    /home/oc-mirror --config "/home/imageset-config.yaml" docker://\${mirror_registry} --oci-registries-config="/home/registry.conf" --continue-on-error --skip-missing
+    /home/oc-mirror --config "/home/imageset-config.yaml" docker://\${mirror_registry} --oci-registries-config="/home/registry.conf" --continue-on-error --skip-missing
+    popd
+
+    echo "5: Create imageconentsourcepolicy and catalogsource"
+    for d in /home/oc-mirror-workspace/results* ; do sed -i "s|name: operator-0\$|name: operator-\${d#/home/oc-mirror-workspace/results-}|g" \${d}/imageContentSourcePolicy.yaml; done
+    find /home/oc-mirror-workspace -type d -name '*results*' -exec oc apply -f {}/*.yaml \;
+
+    echo "6: Waiting for the new ImageContentSourcePolicy to be updated on machines"
+    oc wait clusteroperators/machine-config --for=condition=Upgradeable=true --timeout=15m
+EOF
+}
+
+
+
+
+if [[ "${DISCONNECTED}" == "true" ]];
+then
+    mirror_odf
+else
+    name="redhat-operators-$(echo $REDHAT_OPERATORS_INDEX_TAG| sed "s/[.]/-/g")"
+
+    oc apply -f - <<EOF
+    apiVersion: operators.coreos.com/v1alpha1
+    kind: CatalogSource
+    metadata:
+      annotations:
+        operatorframework.io/managed-by: marketplace-operator
+        target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}'
+      generation: 5
+      name: $name
+      namespace: openshift-marketplace
+    spec:
+      displayName: Red Hat Operators
+      grpcPodConfig:
+        nodeSelector:
+          kubernetes.io/os: linux
+          node-role.kubernetes.io/master: ""
+        priorityClassName: system-cluster-critical
+        securityContextConfig: restricted
+        tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+          operator: Exists
+        - effect: NoExecute
+          key: node.kubernetes.io/unreachable
+          operator: Exists
+          tolerationSeconds: 120
+        - effect: NoExecute
+          key: node.kubernetes.io/not-ready
+          operator: Exists
+          tolerationSeconds: 120
+      icon:
+        base64data: ""
+        mediatype: ""
+      image: registry.redhat.io/redhat/redhat-operator-index:${REDHAT_OPERATORS_INDEX_TAG}
+      priority: -100
+      publisher: Red Hat
+      sourceType: grpc
+      updateStrategy:
+        registryPoll:
+          interval: 10m
+EOF
+
+    for i in $(seq 1 120); do
+        state=$(oc get catalogsources/$name -n openshift-marketplace -o jsonpath='{.status.connectionState.lastObservedState}')
+        echo $state
+        if [ "$state" == "READY" ] ; then
+            echo "Catalogsource created successfully after waiting $((5*i)) seconds"
+            echo "current state of catalogsource is \"$state\""
+            created=true
+            break
+        fi
+        sleep 5
+    done
+    [ "$created" = "true" ]
+fi
 

ci-operator/step-registry/hypershift/kubevirt/baremetalds/redhat-operators/hypershift-kubevirt-baremetalds-redhat-operators-ref.yaml

@@ -1,6 +1,7 @@
 ref:
   as: hypershift-kubevirt-baremetalds-redhat-operators
-  from: cli
+  from: upi-installer
+  cli: latest
   grace_period: 10m
   commands: hypershift-kubevirt-baremetalds-redhat-operators-commands.sh
   resources:
@@ -12,5 +13,13 @@ ref:
   - name: REDHAT_OPERATORS_INDEX_TAG
     documentation: Redhat operators catalog source index tag
     default: v4.14
+  - name: DISCONNECTED
+    default: "false"
+  - name: ODF_OPERATOR_SUB_PACKAGE
+    documentation: The package name of the ODF Operator to install.
+    default: odf-operator
+  - name: ODF_OPERATOR_SUB_CHANNEL
+    documentation: The channel from which to install the package.
+    default: stable-4.14
   documentation: |-
     This step to deploy a specific redhat-operators catalog source