diff --git a/ci-operator/config/openshift/cluster-kube-scheduler-operator/openshift-cluster-kube-scheduler-operator-main.yaml b/ci-operator/config/openshift/cluster-kube-scheduler-operator/openshift-cluster-kube-scheduler-operator-main.yaml index 02ef1f49cf083..92910d6d1ae9c 100644 --- a/ci-operator/config/openshift/cluster-kube-scheduler-operator/openshift-cluster-kube-scheduler-operator-main.yaml +++ b/ci-operator/config/openshift/cluster-kube-scheduler-operator/openshift-cluster-kube-scheduler-operator-main.yaml @@ -82,38 +82,13 @@ tests: optional: true steps: workflow: openshift-ci-security -- always_run: false - as: e2e-aws-operator-serial-ote - optional: true - steps: - cluster_profile: openshift-org-aws - env: - TEST_SUITE: openshift/cluster-kube-scheduler-operator/operator/serial - test: - - ref: openshift-e2e-test - workflow: ipi-aws - timeout: 8h0m0s -- always_run: false - as: e2e-aws-operator-parallel-ote - optional: true +- as: e2e-aws-operator-ote-combined steps: cluster_profile: openshift-org-aws - env: - TEST_SUITE: openshift/cluster-kube-scheduler-operator/operator/parallel test: - - ref: openshift-e2e-test + - ref: cluster-kube-scheduler-operator-e2e-ote workflow: ipi-aws timeout: 8h0m0s -- always_run: false - as: e2e-aws-preferred-host-serial-ote - optional: true - steps: - cluster_profile: openshift-org-aws - env: - TEST_SUITE: openshift/cluster-kube-scheduler-operator/preferred-host/serial - test: - - ref: openshift-e2e-test - workflow: ipi-aws zz_generated_metadata: branch: main org: openshift diff --git a/ci-operator/jobs/openshift/cluster-kube-scheduler-operator/openshift-cluster-kube-scheduler-operator-main-presubmits.yaml b/ci-operator/jobs/openshift/cluster-kube-scheduler-operator/openshift-cluster-kube-scheduler-operator-main-presubmits.yaml index caaeee1f199e5..7484591fca332 100644 --- a/ci-operator/jobs/openshift/cluster-kube-scheduler-operator/openshift-cluster-kube-scheduler-operator-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/cluster-kube-scheduler-operator/openshift-cluster-kube-scheduler-operator-main-presubmits.yaml @@ -81,12 +81,12 @@ presubmits: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-aws-operator,?($|\s.*) - agent: kubernetes - always_run: false + always_run: true branches: - ^main$ - ^main- cluster: build10 - context: ci/prow/e2e-aws-operator-parallel-ote + context: ci/prow/e2e-aws-operator-ote-combined decorate: true decoration_config: timeout: 8h0m0s @@ -95,9 +95,8 @@ presubmits: ci-operator.openshift.io/cloud-cluster-profile: openshift-org-aws ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-openshift-cluster-kube-scheduler-operator-main-e2e-aws-operator-parallel-ote - optional: true - rerun_command: /test e2e-aws-operator-parallel-ote + name: pull-ci-openshift-cluster-kube-scheduler-operator-main-e2e-aws-operator-ote-combined + rerun_command: /test e2e-aws-operator-ote-combined spec: containers: - args: @@ -106,7 +105,7 @@ presubmits: - --lease-server-credentials-file=/etc/boskos/credentials - --report-credentials-file=/etc/report/credentials - --secret-dir=/secrets/ci-pull-credentials - - --target=e2e-aws-operator-parallel-ote + - --target=e2e-aws-operator-ote-combined command: - ci-operator env: @@ -162,7 +161,7 @@ presubmits: - name: result-aggregator secret: secretName: result-aggregator - trigger: (?m)^/test( | .* )e2e-aws-operator-parallel-ote,?($|\s.*) + trigger: (?m)^/test( | .* )e2e-aws-operator-ote-combined,?($|\s.*) - agent: kubernetes always_run: true branches: @@ -243,89 +242,6 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-aws-operator-preferred-host,?($|\s.*) - - agent: kubernetes - always_run: false - branches: - - ^main$ - - ^main- - cluster: build10 - context: ci/prow/e2e-aws-operator-serial-ote - decorate: true - decoration_config: - timeout: 8h0m0s - labels: - ci-operator.openshift.io/cloud: aws - ci-operator.openshift.io/cloud-cluster-profile: openshift-org-aws - ci.openshift.io/generator: prowgen - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-openshift-cluster-kube-scheduler-operator-main-e2e-aws-operator-serial-ote - optional: true - rerun_command: /test e2e-aws-operator-serial-ote - spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --secret-dir=/secrets/ci-pull-credentials - - --target=e2e-aws-operator-serial-ote - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/ci-pull-credentials - name: ci-pull-credentials - readOnly: true - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: ci-pull-credentials - secret: - secretName: ci-pull-credentials - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - trigger: (?m)^/test( | .* )e2e-aws-operator-serial-ote,?($|\s.*) - agent: kubernetes always_run: true branches: @@ -648,87 +564,6 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-aws-ovn-upgrade,?($|\s.*) - - agent: kubernetes - always_run: false - branches: - - ^main$ - - ^main- - cluster: build10 - context: ci/prow/e2e-aws-preferred-host-serial-ote - decorate: true - labels: - ci-operator.openshift.io/cloud: aws - ci-operator.openshift.io/cloud-cluster-profile: openshift-org-aws - ci.openshift.io/generator: prowgen - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-openshift-cluster-kube-scheduler-operator-main-e2e-aws-preferred-host-serial-ote - optional: true - rerun_command: /test e2e-aws-preferred-host-serial-ote - spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --secret-dir=/secrets/ci-pull-credentials - - --target=e2e-aws-preferred-host-serial-ote - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/ci-pull-credentials - name: ci-pull-credentials - readOnly: true - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: ci-pull-credentials - secret: - secretName: ci-pull-credentials - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - trigger: (?m)^/test( | .* )e2e-aws-preferred-host-serial-ote,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/step-registry/cluster-kube-scheduler-operator/OWNERS b/ci-operator/step-registry/cluster-kube-scheduler-operator/OWNERS new file mode 100644 index 0000000000000..df0651142caa3 --- /dev/null +++ b/ci-operator/step-registry/cluster-kube-scheduler-operator/OWNERS @@ -0,0 +1,8 @@ +approvers: +- ingvagabund +- p0lyn0mial +reviewers: +- ingvagabund +- p0lyn0mial +- ardaguclu +- xueqzhan \ No newline at end of file diff --git a/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/OWNERS b/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/OWNERS new file mode 100644 index 0000000000000..df0651142caa3 --- /dev/null +++ b/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/OWNERS @@ -0,0 +1,8 @@ +approvers: +- ingvagabund +- p0lyn0mial +reviewers: +- ingvagabund +- p0lyn0mial +- ardaguclu +- xueqzhan \ No newline at end of file diff --git a/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-commands.sh b/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-commands.sh new file mode 100755 index 0000000000000..523da3e277df2 --- /dev/null +++ b/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-commands.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +# Setup cloud credentials and environment +export AWS_SHARED_CREDENTIALS_FILE=${CLUSTER_PROFILE_DIR}/.awscred +export AZURE_AUTH_LOCATION=${CLUSTER_PROFILE_DIR}/osServicePrincipal.json +export GCP_SHARED_CREDENTIALS_FILE=${CLUSTER_PROFILE_DIR}/gce.json +export ALIBABA_CLOUD_CREDENTIALS_FILE=${SHARED_DIR}/alibabacreds.ini +export HOME=/tmp/home +export PATH=/usr/libexec/origin:$PATH + +echo "Starting cluster-kube-scheduler-operator OTE test suite execution" + +# Grant access for image pulling from the build farm +echo "Granting access for image pulling from the build farm..." +KUBECONFIG_BAK=$KUBECONFIG +unset KUBECONFIG +oc adm policy add-role-to-group system:image-puller system:unauthenticated --namespace "${NAMESPACE}" || echo "Failed to grant image puller access, continuing..." +export KUBECONFIG=$KUBECONFIG_BAK + +# Enable retry strategy for presubmits to reduce retests +# Use array for safe argument expansion (handles whitespace/metacharacters) +TEST_ARGS=() +if [[ "${JOB_TYPE:-}" == "presubmit" && ( "${PULL_BASE_REF:-}" == "main" || "${PULL_BASE_REF:-}" == "master" ) ]]; then + if openshift-tests run --help 2>/dev/null | grep -q 'retry-strategy'; then + TEST_ARGS+=(--retry-strategy=aggressive) + echo "Enabled aggressive retry strategy for presubmit" + fi +fi + +# Handle HyperShift clusters (treat as AWS) +if [[ "${CLUSTER_TYPE}" == "hypershift" ]]; then + export CLUSTER_TYPE="aws" + echo "Overriding 'hypershift' cluster type to be 'aws'" +fi + +# Load proxy configuration if present +if test -f "${SHARED_DIR}/proxy-conf.sh"; then + # shellcheck disable=SC1090 + source "${SHARED_DIR}/proxy-conf.sh" + echo "Loaded proxy configuration" +fi + +# Handle External platform type +STATUS_PLATFORM_NAME="$(oc get Infrastructure cluster -o jsonpath='{.status.platform}' 2>/dev/null || true)" +if [[ "${STATUS_PLATFORM_NAME-}" == "External" ]]; then + export CLUSTER_TYPE="external" + echo "Detected External platform, setting CLUSTER_TYPE=external" +fi + +# Setup cleanup trap +function cleanup() { + echo "$(date +%s)" > "${SHARED_DIR}/TEST_TIME_TEST_END" || true + echo "Cleanup completed" +} +trap cleanup EXIT + +mkdir -p "${HOME}" + +# Setup test provider based on cluster type +case "${CLUSTER_TYPE}" in +gcp|gcp-arm64) + export GOOGLE_APPLICATION_CREDENTIALS="${GCP_SHARED_CREDENTIALS_FILE}" + export ENABLE_STORAGE_GCE_PD_DRIVER="yes" + export KUBE_SSH_USER=core + PROJECT="$(oc get -o jsonpath='{.status.platformStatus.gcp.projectID}' infrastructure cluster)" + REGION="$(oc get -o jsonpath='{.status.platformStatus.gcp.region}' infrastructure cluster)" + export TEST_PROVIDER="{\"type\":\"gce\",\"region\":\"${REGION}\",\"multizone\":true,\"multimaster\":true,\"projectid\":\"${PROJECT}\"}" + ;; +aws|aws-arm64|aws-eusc) + export PROVIDER_ARGS="-provider=aws -gce-zone=us-east-1" + REGION="$(oc get -o jsonpath='{.status.platformStatus.aws.region}' infrastructure cluster)" + ZONE="$(oc get -o jsonpath='{.items[0].metadata.labels.failure-domain\.beta\.kubernetes\.io/zone}' nodes)" + export TEST_PROVIDER="{\"type\":\"aws\",\"region\":\"${REGION}\",\"zone\":\"${ZONE}\",\"multizone\":true,\"multimaster\":true}" + export KUBE_SSH_USER=core + ;; +azure4|azure-arm64) + export TEST_PROVIDER=azure + ;; +azurestack) + export TEST_PROVIDER="none" + export AZURE_AUTH_LOCATION=${SHARED_DIR}/osServicePrincipal.json + ;; +vsphere) + # shellcheck disable=SC1090 + source "${SHARED_DIR}/govc.sh" + export TEST_PROVIDER=vsphere + ;; +openstack*) + # shellcheck disable=SC1090 + source "${SHARED_DIR}/cinder_credentials.sh" + if test -n "${HTTP_PROXY:-}" -o -n "${HTTPS_PROXY:-}"; then + export TEST_PROVIDER='{"type":"openstack","disconnected":true}' + else + export TEST_PROVIDER='{"type":"openstack"}' + fi + ;; +ovirt) + export TEST_PROVIDER='{"type":"ovirt"}' + ;; +ibmcloud*) + export TEST_PROVIDER='{"type":"ibmcloud"}' + ;; +nutanix) + export TEST_PROVIDER='{"type":"nutanix"}' + ;; +external) + export TEST_PROVIDER='{"type":"external"}' + ;; +*) + echo "Using default cluster type: ${CLUSTER_TYPE}" + export TEST_PROVIDER="${CLUSTER_TYPE}" + ;; +esac + +echo "TEST_PROVIDER configured as: ${TEST_PROVIDER}" + +# Create working directory +mkdir -p /tmp/output +cd /tmp/output + +# Record test start time +echo "$(date +%s)" > "${SHARED_DIR}/TEST_TIME_TEST_START" + +# Wait for cluster to be stable +echo "$(date) - Waiting for ClusterVersion to stabilize..." +oc wait --for=condition=Progressing=False --timeout=2m clusterversion/version || echo "ClusterVersion check timed out, continuing..." + +echo "$(date) - Waiting for cluster operators to finish progressing..." +oc wait clusteroperators --all --for=condition=Progressing=false --timeout=10m || echo "ClusterOperators check timed out, continuing..." +echo "$(date) - Cluster operators check completed" + +# Wait for cluster stability if command is available +echo "$(date) - Checking for cluster stability..." +if oc adm wait-for-stable-cluster --minimum-stable-period 2m &>/dev/null; then + echo "$(date) - Cluster is stable" +else + echo "$(date) - wait-for-stable-cluster not available or failed, continuing..." +fi + +# Create output directories for each test suite +mkdir -p "${ARTIFACT_DIR}/junit-operator-serial" +mkdir -p "${ARTIFACT_DIR}/junit-operator-parallel" +mkdir -p "${ARTIFACT_DIR}/junit-preferred-host-serial" + +# Initialize return code to track failures across all test suites +rc=0 + +# Note on test isolation: openshift-tests is stateless and each test suite +# is self-cleaning. Tests create/destroy their own resources and do not +# depend on execution order. Running sequentially on the same cluster is safe. + +# Run operator/serial test suite +echo "========================================" +echo "$(date) - Running operator/serial test suite..." +echo "========================================" +TEST_SUITE="openshift/cluster-kube-scheduler-operator/operator/serial" +if openshift-tests run "${TEST_SUITE}" "${TEST_ARGS[@]}" \ + --provider "${TEST_PROVIDER}" \ + -o "${ARTIFACT_DIR}/e2e-operator-serial.log" \ + --junit-dir "${ARTIFACT_DIR}/junit-operator-serial"; then + echo "$(date) - ✓ operator/serial tests passed" +else + rc=1 + echo "$(date) - ✗ operator/serial tests failed" +fi + +# Run operator/parallel test suite +echo "========================================" +echo "$(date) - Running operator/parallel test suite..." +echo "========================================" +TEST_SUITE="openshift/cluster-kube-scheduler-operator/operator/parallel" +if openshift-tests run "${TEST_SUITE}" "${TEST_ARGS[@]}" \ + --provider "${TEST_PROVIDER}" \ + -o "${ARTIFACT_DIR}/e2e-operator-parallel.log" \ + --junit-dir "${ARTIFACT_DIR}/junit-operator-parallel"; then + echo "$(date) - ✓ operator/parallel tests passed" +else + rc=1 + echo "$(date) - ✗ operator/parallel tests failed" +fi + +# Run preferred-host/serial test suite +echo "========================================" +echo "$(date) - Running preferred-host/serial test suite..." +echo "========================================" +TEST_SUITE="openshift/cluster-kube-scheduler-operator/preferred-host/serial" +if openshift-tests run "${TEST_SUITE}" "${TEST_ARGS[@]}" \ + --provider "${TEST_PROVIDER}" \ + -o "${ARTIFACT_DIR}/e2e-preferred-host-serial.log" \ + --junit-dir "${ARTIFACT_DIR}/junit-preferred-host-serial"; then + echo "$(date) - ✓ preferred-host/serial tests passed" +else + rc=1 + echo "$(date) - ✗ preferred-host/serial tests failed" +fi + +echo "========================================" +if [ "$rc" -eq 0 ]; then + echo "$(date) - All cluster-kube-scheduler-operator OTE tests completed successfully!" +else + echo "$(date) - Some cluster-kube-scheduler-operator OTE tests failed (exit code: $rc)" +fi +echo "========================================" + +# Exit with the captured return code +exit "${rc}" diff --git a/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-ref.metadata.json b/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-ref.metadata.json new file mode 100644 index 0000000000000..bc39eb996a3c0 --- /dev/null +++ b/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-ref.yaml", + "owners": { + "approvers": [ + "ingvagabund", + "p0lyn0mial" + ], + "reviewers": [ + "ingvagabund", + "p0lyn0mial", + "ardaguclu", + "xueqzhan" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-ref.yaml b/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-ref.yaml new file mode 100644 index 0000000000000..a250cc0da3062 --- /dev/null +++ b/ci-operator/step-registry/cluster-kube-scheduler-operator/e2e-ote/cluster-kube-scheduler-operator-e2e-ote-ref.yaml @@ -0,0 +1,16 @@ +ref: + as: cluster-kube-scheduler-operator-e2e-ote + from: tests + grace_period: 10m + commands: cluster-kube-scheduler-operator-e2e-ote-commands.sh + timeout: 8h0m0s + resources: + requests: + cpu: "3" + memory: 600Mi + limits: + memory: 10Gi + documentation: |- + Runs all cluster-kube-scheduler-operator OTE test suites sequentially on a single cluster. + This consolidates operator/serial, operator/parallel, and preferred-host/serial test suites + to avoid spinning up multiple clusters for a small test suite (4-5 test cases total).