Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,16 @@ tests:
HYPERSHIFT_AZURE_LOCATION: centralus
TEST_CPO_OVERRIDE: "1"
workflow: hypershift-azure-aks-e2e
- always_run: false
as: e2e-azure-v2-self-managed
optional: true
pipeline_run_if_changed: ^(test/e2e/v2/tests/hosted_cluster_azure|test/e2e/util)
steps:
cluster_profile: hypershift-azure
env:
ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true"
HYPERSHIFT_AZURE_LOCATION: centralus
workflow: hypershift-azure-e2e-v2-self-managed
- always_run: false
as: e2e-aws
capabilities:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1305,6 +1305,89 @@ presubmits:
secret:
secretName: result-aggregator
trigger: (?m)^/test( | .* )e2e-azure-self-managed,?($|\s.*)
- agent: kubernetes
always_run: false
annotations:
pipeline_run_if_changed: ^(test/e2e/v2/tests/hosted_cluster_azure|test/e2e/util)
branches:
- ^main$
- ^main-
cluster: build01
context: ci/prow/e2e-azure-v2-self-managed
decorate: true
labels:
ci-operator.openshift.io/cloud: hypershift-azure
ci-operator.openshift.io/cloud-cluster-profile: hypershift-azure
ci.openshift.io/generator: prowgen
pj-rehearse.openshift.io/can-be-rehearsed: "true"
name: pull-ci-openshift-hypershift-main-e2e-azure-v2-self-managed
optional: true
rerun_command: /test e2e-azure-v2-self-managed
spec:
containers:
- args:
- --gcs-upload-secret=/secrets/gcs/service-account.json
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --lease-server-credentials-file=/etc/boskos/credentials
- --report-credentials-file=/etc/report/credentials
- --secret-dir=/secrets/ci-pull-credentials
- --target=e2e-azure-v2-self-managed
command:
- ci-operator
env:
- name: HTTP_SERVER_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
imagePullPolicy: Always
name: ""
ports:
- containerPort: 8080
name: http
resources:
requests:
cpu: 10m
volumeMounts:
- mountPath: /etc/boskos
name: boskos
readOnly: true
- mountPath: /secrets/ci-pull-credentials
name: ci-pull-credentials
readOnly: true
- mountPath: /secrets/gcs
name: gcs-credentials
readOnly: true
- mountPath: /secrets/manifest-tool
name: manifest-tool-local-pusher
readOnly: true
- mountPath: /etc/pull-secret
name: pull-secret
readOnly: true
- mountPath: /etc/report
name: result-aggregator
readOnly: true
serviceAccountName: ci-operator
volumes:
- name: boskos
secret:
items:
- key: credentials
path: credentials
secretName: boskos-credentials
- name: ci-pull-credentials
secret:
secretName: ci-pull-credentials
- name: manifest-tool-local-pusher
secret:
secretName: manifest-tool-local-pusher
- name: pull-secret
secret:
secretName: registry-pull-credentials
- name: result-aggregator
secret:
secretName: result-aggregator
trigger: (?m)^/test( | .* )e2e-azure-v2-self-managed,?($|\s.*)
- agent: kubernetes
always_run: false
branches:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
approvers:
- csrwng
- enxebre
- sjenning
- bryan-cox
options: {}
reviewers:
- csrwng
- enxebre
- sjenning
- bryan-cox
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"path": "hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-chain.yaml",
"owners": {
"approvers": [
"csrwng",
"enxebre",
"sjenning",
"bryan-cox"
],
"reviewers": [
"csrwng",
"enxebre",
"sjenning",
"bryan-cox"
]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
chain:
as: hypershift-azure-create-selfmanaged-guests
steps:
- as: create-guests
cli: latest
env:
- name: HYPERSHIFT_NODE_COUNT
default: "3"
documentation: "The number of nodes per guest cluster."
- name: HYPERSHIFT_BASE_DOMAIN
default: "hcp-sm-azure.azure.devcluster.openshift.com"
documentation: "The cluster's FQDN will be a subdomain of the base domain."
- name: HYPERSHIFT_AZURE_LOCATION
default: "centralus"
documentation: "Specifies the Azure location of the clusters."
- name: HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER
default: ""
documentation: "The Azure Marketplace image publisher."
- name: HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_OFFER
default: ""
documentation: "The Azure Marketplace image offer."
- name: HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_SKU
default: ""
documentation: "The Azure Marketplace image SKU."
- name: HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION
default: ""
documentation: "The Azure Marketplace image version."
commands: |-
set -exuo pipefail

# Use the nested management cluster kubeconfig
export KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig"

# Generate unique cluster names from job ID
PUBLIC_NAME="$(echo -n "${PROW_JOB_ID}-pub"|sha256sum|cut -c-20)"
PRIVATE_NAME="$(echo -n "${PROW_JOB_ID}-prv"|sha256sum|cut -c-20)"
OAUTH_LB_NAME="$(echo -n "${PROW_JOB_ID}-oau"|sha256sum|cut -c-20)"

# Self-managed Azure credentials
AZURE_CREDS="/etc/hypershift-ci-jobs-self-managed-azure/credentials.json"
AZURE_OIDC_ISSUER_URL="https://smazure.blob.core.windows.net/smazure"
AZURE_SA_TOKEN_ISSUER_KEY_PATH="/etc/hypershift-ci-jobs-self-managed-azure-e2e/serviceaccount-signer.private"
AZURE_WORKLOAD_IDENTITIES_FILE="/etc/hypershift-ci-jobs-self-managed-azure-e2e/workload-identities.json"

PULL_SECRET_PATH="/etc/ci-pull-credentials/.dockerconfigjson"

RELEASE_IMAGE="${RELEASE_IMAGE_LATEST}"
HC_LOCATION="${HYPERSHIFT_AZURE_LOCATION:-centralus}"

# Read private NAT subnet ID from SHARED_DIR (written by setup-private-link step)
if [[ ! -s "${SHARED_DIR}/azure_private_nat_subnet_id" ]]; then
echo "$(date) ERROR: azure_private_nat_subnet_id is required for the private guest cluster"
exit 1
fi
AZURE_PRIVATE_NAT_SUBNET_ID="$(cat "${SHARED_DIR}/azure_private_nat_subnet_id")"

# Marketplace image flags
MARKETPLACE_ARGS=""
if [[ -n "${HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER:-}" ]]; then
MARKETPLACE_ARGS="--marketplace-publisher=${HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER} --marketplace-offer=${HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_OFFER}"
if [[ -n "${HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_SKU:-}" ]]; then
MARKETPLACE_ARGS="${MARKETPLACE_ARGS} --marketplace-sku=${HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_SKU}"
elif [[ -f "${SHARED_DIR}/azure-marketplace-image-sku" ]]; then
MARKETPLACE_ARGS="${MARKETPLACE_ARGS} --marketplace-sku=$(cat "${SHARED_DIR}/azure-marketplace-image-sku")"
fi
if [[ -n "${HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION:-}" ]]; then
MARKETPLACE_ARGS="${MARKETPLACE_ARGS} --marketplace-version=${HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION}"
elif [[ -f "${SHARED_DIR}/azure-marketplace-image-version" ]]; then
MARKETPLACE_ARGS="${MARKETPLACE_ARGS} --marketplace-version=$(cat "${SHARED_DIR}/azure-marketplace-image-version")"
fi
fi

# Common flags for all self-managed clusters
COMMON_FLAGS="--node-pool-replicas=${HYPERSHIFT_NODE_COUNT} \
--base-domain=${HYPERSHIFT_BASE_DOMAIN} \
--pull-secret=${PULL_SECRET_PATH} \
--azure-creds=${AZURE_CREDS} \
--location=${HC_LOCATION} \
--release-image=${RELEASE_IMAGE} \
--oidc-issuer-url=${AZURE_OIDC_ISSUER_URL} \
--sa-token-issuer-private-key-path=${AZURE_SA_TOKEN_ISSUER_KEY_PATH} \
--workload-identities-file=${AZURE_WORKLOAD_IDENTITIES_FILE} \
--generate-ssh \
${MARKETPLACE_ARGS}"

# Create public cluster
echo "$(date) Creating public self-managed cluster: ${PUBLIC_NAME}"
/usr/bin/hypershift create cluster azure \
--name="${PUBLIC_NAME}" \
${COMMON_FLAGS} &
PUBLIC_PID=$!

# Create private cluster
PRIVATE_EXTRA="--endpoint-access-private-nat-subnet-id=${AZURE_PRIVATE_NAT_SUBNET_ID}"
echo "$(date) Creating private self-managed cluster: ${PRIVATE_NAME}"
/usr/bin/hypershift create cluster azure \
--name="${PRIVATE_NAME}" \
--endpoint-access=Private \
${COMMON_FLAGS} \
${PRIVATE_EXTRA} &
PRIVATE_PID=$!

# Create OAuth LoadBalancer cluster
echo "$(date) Creating OAuth LB self-managed cluster: ${OAUTH_LB_NAME}"
/usr/bin/hypershift create cluster azure \
--name="${OAUTH_LB_NAME}" \
--oauth-publishing-strategy=LoadBalancer \
${COMMON_FLAGS} &
OAUTH_LB_PID=$!

# Wait for create commands to complete
echo "$(date) Waiting for cluster create commands to finish..."
FAILED=0
wait ${PUBLIC_PID} || FAILED=1
echo "$(date) Public cluster create command completed"
wait ${PRIVATE_PID} || FAILED=1
echo "$(date) Private cluster create command completed"
wait ${OAUTH_LB_PID} || FAILED=1
echo "$(date) OAuth LB cluster create command completed"
if [[ ${FAILED} -ne 0 ]]; then
echo "$(date) ERROR: One or more cluster create commands failed"
exit 1
fi

# Wait for clusters to become available
echo "$(date) Waiting for public cluster to become available..."
oc wait --timeout=30m --for=condition=Available --namespace=clusters "hostedcluster/${PUBLIC_NAME}"
echo "$(date) Public cluster is available"

echo "$(date) Waiting for private cluster to become available..."
oc wait --timeout=30m --for=condition=Available --namespace=clusters "hostedcluster/${PRIVATE_NAME}"
echo "$(date) Private cluster is available"

echo "$(date) Waiting for OAuth LB cluster to become available..."
oc wait --timeout=30m --for=condition=Available --namespace=clusters "hostedcluster/${OAUTH_LB_NAME}"
echo "$(date) OAuth LB cluster is available"

# Wait for version rollout to complete on each cluster (via management API, same as AWS/GCP v2)
FAILED_READY=0
for CLUSTER in "${PUBLIC_NAME}" "${PRIVATE_NAME}" "${OAUTH_LB_NAME}"; do
echo "$(date) Waiting for version rollout on ${CLUSTER}..."
set +e
CLUSTER_CHECK="${CLUSTER}" timeout 30m bash -c '
until [[ "$(oc get -n clusters hostedcluster/${CLUSTER_CHECK} -o jsonpath='"'"'{.status.version.history[?(@.state!="")].state}'"'"')" = "Completed" ]]; do
sleep 15
done
'
ROLLOUT_RC=$?
set -e
if [[ ${ROLLOUT_RC} -ne 0 ]]; then
echo "$(date) ERROR: version rollout timed out for ${CLUSTER}"
cat << EOF > "${ARTIFACT_DIR}/junit_hosted_cluster_${CLUSTER}.xml"
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="hypershift install ${CLUSTER}" tests="1" failures="1">
<testcase name="hosted cluster version rollout succeeds">
<failure message="hosted cluster version rollout never completed">
<![CDATA[
error: hosted cluster version rollout never completed for ${CLUSTER}
Degraded: $(oc get -n clusters hostedcluster/${CLUSTER} -o jsonpath='{.status.conditions[?(@.type=="Degraded")].message}')
ClusterVersionSucceeding: $(oc get -n clusters hostedcluster/${CLUSTER} -o jsonpath='{.status.conditions[?(@.type=="ClusterVersionSucceeding")].message}')
]]>
</failure>
</testcase>
</testsuite>
EOF
FAILED_READY=1
else
echo "$(date) Version rollout completed for ${CLUSTER}"
cat << EOF > "${ARTIFACT_DIR}/junit_hosted_cluster_${CLUSTER}.xml"
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="hypershift install ${CLUSTER}" tests="1" failures="0">
<testcase name="hosted cluster version rollout succeeds">
<system-out>
<![CDATA[
info: hosted cluster version rollout completed successfully for ${CLUSTER}
]]>
</system-out>
</testcase>
</testsuite>
EOF
fi
done
if [[ ${FAILED_READY} -ne 0 ]]; then
exit 1
fi

# Write cluster names to shared dir
echo "${PUBLIC_NAME}" > "${SHARED_DIR}/cluster-name-public"
echo "${PRIVATE_NAME}" > "${SHARED_DIR}/cluster-name-private"
echo "${OAUTH_LB_NAME}" > "${SHARED_DIR}/cluster-name-oauth-lb"

echo "$(date) All self-managed guest clusters are ready"
from: hypershift-operator
grace_period: 5m0s
resources:
requests:
cpu: 100m
memory: 100Mi
timeout: 120m0s
credentials:
- mount_path: /etc/ci-pull-credentials
name: ci-pull-credentials
namespace: test-credentials
- mount_path: /etc/hypershift-ci-jobs-self-managed-azure
name: hypershift-ci-jobs-self-managed-azure
namespace: test-credentials
- mount_path: /etc/hypershift-ci-jobs-self-managed-azure-e2e
name: hypershift-ci-jobs-self-managed-azure-e2e
namespace: test-credentials
dependencies:
- name: "release:latest"
env: RELEASE_IMAGE_LATEST
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
approvers:
- csrwng
- enxebre
- sjenning
- bryan-cox
options: {}
reviewers:
- csrwng
- enxebre
- sjenning
- bryan-cox
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"path": "hypershift/azure/destroy-selfmanaged-guests/hypershift-azure-destroy-selfmanaged-guests-chain.yaml",
"owners": {
"approvers": [
"csrwng",
"enxebre",
"sjenning",
"bryan-cox"
],
"reviewers": [
"csrwng",
"enxebre",
"sjenning",
"bryan-cox"
]
}
}
Loading