forked from openshift/assisted-service
/
deploy_capi_cluster.sh
executable file
·236 lines (206 loc) · 12.2 KB
/
deploy_capi_cluster.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
#!/usr/bin/env bash
__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
__root="$(realpath ${__dir}/../../..)"
source ${__dir}/../common.sh
source ${__dir}/../utils.sh
set -x
export ASSISTED_CLUSTER_NAME="${ASSISTED_CLUSTER_NAME:-assisted-test-cluster}"
export ASSISTED_CLUSTER_DEPLOYMENT_NAME="${ASSISTED_CLUSTER_DEPLOYMENT_NAME:-assisted-test-cluster}"
export ASSISTED_AGENT_CLUSTER_INSTALL_NAME="${ASSISTED_AGENT_CLUSTER_INSTALL_NAME:-assisted-agent-cluster-install}"
export ASSISTED_INFRAENV_NAME="${ASSISTED_INFRAENV_NAME:-assisted-infra-env}"
export ASSISTED_PULLSECRET_NAME="${ASSISTED_PULLSECRET_NAME:-assisted-pull-secret}"
export ASSISTED_PULLSECRET_JSON="${ASSISTED_PULLSECRET_JSON:-${PULL_SECRET_FILE}}"
export ASSISTED_PRIVATEKEY_NAME="${ASSISTED_PRIVATEKEY_NAME:-assisted-ssh-private-key}"
export EXTRA_BAREMETALHOSTS_FILE="${EXTRA_BAREMETALHOSTS_FILE:-/home/test/dev-scripts/ocp/ostest/extra_baremetalhosts.json}"
export SPOKE_CONTROLPLANE_AGENTS="${SPOKE_CONTROLPLANE_AGENTS:-1}"
export ASSISTED_STOP_AFTER_AGENT_DISCOVERY="${ASSISTED_STOP_AFTER_AGENT_DISCOVERY:-false}"
export ASSISTED_UPGRADE_OPERATOR="${ASSISTED_UPGRADE_OPERATOR:-false}"
export SPAWN_NONE_PLATFORM_LOAD_BALANCER="${SPAWN_NONE_PLATFORM_LOAD_BALANCER:-false}"
export ADD_NONE_PLATFORM_LIBVIRT_DNS="${ADD_NONE_PLATFORM_LIBVIRT_DNS:-false}"
export LIBVIRT_NONE_PLATFORM_NETWORK="${LIBVIRT_NONE_PLATFORM_NETWORK:-ostestbm}"
export LOAD_BALANCER_IP="${LOAD_BALANCER_IP:-192.168.111.1}"
export HYPERSHIFT_IMAGE="${HYPERSHIFT_IMAGE:-quay.io/hypershift/hypershift-operator:latest}"
export CONTROL_PLANE_OPERATOR_IMAGE="${CONTROL_PLANE_OPERATOR_IMAGE:-}"
export PROVIDER_IMAGE="${PROVIDER_IMAGE:-}"
export EXTRA_HYPERSHIFT_INSTALL_FLAGS="${EXTRA_HYPERSHIFT_INSTALL_FLAGS:-}"
export EXTRA_HYPERSHIFT_CREATE_COMMANDS="${EXTRA_HYPERSHIFT_CREATE_COMMANDS:-}"
if [[ ${SPOKE_CONTROLPLANE_AGENTS} -eq 1 ]]; then
export USER_MANAGED_NETWORKING="true"
else
export USER_MANAGED_NETWORKING="${USER_MANAGED_NETWORKING:-false}"
fi
if [[ "${IP_STACK}" == "v4" ]]; then
export CLUSTER_SUBNET="${CLUSTER_SUBNET_V4}"
export CLUSTER_HOST_PREFIX="${CLUSTER_HOST_PREFIX_V4}"
if [ "${USER_MANAGED_NETWORKING}" != "true" ] || [ ${SPOKE_CONTROLPLANE_AGENTS} -eq 1 ] ; then
export EXTERNAL_SUBNET="${EXTERNAL_SUBNET_V4}"
else
unset EXTERNAL_SUBNET
fi
export SERVICE_SUBNET="${SERVICE_SUBNET_V4}"
elif [[ "${IP_STACK}" == "v6" ]]; then
export CLUSTER_SUBNET="${CLUSTER_SUBNET_V6}"
export CLUSTER_HOST_PREFIX="${CLUSTER_HOST_PREFIX_V6}"
export EXTERNAL_SUBNET="${EXTERNAL_SUBNET_V6}"
export SERVICE_SUBNET="${SERVICE_SUBNET_V6}"
# IPv6 requires hypershift create cluster cidr override
export EXTRA_HYPERSHIFT_CREATE_COMMANDS="$EXTRA_HYPERSHIFT_CREATE_COMMANDS --cluster-cidr fd01::/48"
elif [[ "${IP_STACK}" == "v4v6" ]]; then
export CLUSTER_SUBNET="${CLUSTER_SUBNET_V4}"
export CLUSTER_HOST_PREFIX="${CLUSTER_HOST_PREFIX_V4}"
export EXTERNAL_SUBNET="${EXTERNAL_SUBNET_V4}"
export SERVICE_SUBNET="${SERVICE_SUBNET_V4}"
export CLUSTER_SUBNET_ADDITIONAL="${CLUSTER_SUBNET_V6}"
export CLUSTER_HOST_PREFIX_ADDITIONAL="${CLUSTER_HOST_PREFIX_V6}"
export EXTERNAL_SUBNET_ADDITIONAL="${EXTERNAL_SUBNET_V6}"
export SERVICE_SUBNET_ADDITIONAL="${SERVICE_SUBNET_V6}"
# IPv6 requires hypershift create cluster and service cidr overrides
export EXTRA_HYPERSHIFT_CREATE_COMMANDS="$EXTRA_HYPERSHIFT_CREATE_COMMANDS --cluster-cidr fd01::/48 --service-cidr fd02::/112"
fi
if [ "${DISCONNECTED}" = "true" ]; then
export DISCONNECTED_ASSISTED_OPENSHIFT_INSTALL_RELEASE_IMAGE="${LOCAL_REGISTRY}/ocp/ocp-release:latest"
# Disconnected hypershift requires:
# 1. pull secret in hypershift namespace for the hypershift operator
oc get namespace hypershift || oc create namespace hypershift
oc get secret "${ASSISTED_PULLSECRET_NAME}" -n hypershift || \
oc create secret generic "${ASSISTED_PULLSECRET_NAME}" --from-file=.dockerconfigjson="${ASSISTED_PULLSECRET_JSON}" --type=kubernetes.io/dockerconfigjson -n hypershift
# 2. mirrored hypershift operator image to local registry
HYPERSHIFT_LOCAL_IMAGE="${LOCAL_REGISTRY}/localimages/hypershift:latest"
oc image mirror -a "${PULL_SECRET_FILE}" "${HYPERSHIFT_IMAGE}" "${HYPERSHIFT_LOCAL_IMAGE}"
export HYPERSHIFT_IMAGE="${HYPERSHIFT_LOCAL_IMAGE}"
# 3. the hypershift cli must be available on the local environment
id=$(podman create $HYPERSHIFT_LOCAL_IMAGE)
mkdir -p ./hypershift-cli
podman cp $id:/usr/bin/hypershift ./hypershift-cli
export PATH="$PATH":"$PWD"/hypershift-cli
# 4. mirrored openshift release to local registry
# disconnected openshift release image will be used as release-image flag for hypershift create cluster
oc image mirror -a "${PULL_SECRET_FILE}" "${ASSISTED_OPENSHIFT_INSTALL_RELEASE_IMAGE}" "${DISCONNECTED_ASSISTED_OPENSHIFT_INSTALL_RELEASE_IMAGE}"
export ASSISTED_OPENSHIFT_INSTALL_RELEASE_IMAGE="${DISCONNECTED_ASSISTED_OPENSHIFT_INSTALL_RELEASE_IMAGE}"
# 5. mirrored capi agent image to local registry
if [ -z "$PROVIDER_IMAGE" ]
then
export PROVIDER_LOCAL_IMAGE="${LOCAL_REGISTRY}/localimages/cluster-api-provider-agent:latest"
oc image mirror -a "${PULL_SECRET_FILE}" "${PROVIDER_IMAGE}" "${PROVIDER_LOCAL_IMAGE}"
export PROVIDER_IMAGE="${PROVIDER_LOCAL_IMAGE}"
fi
# 6. ImageContentPolicy for local registry
cat << EOM >> icsp.yaml
apiVersion: operator.openshift.io/v1alpha1
kind: ImageContentSourcePolicy
metadata:
name: example
spec:
repositoryDigestMirrors:
- mirrors:
- ${LOCAL_REGISTRY}/openshift-release-dev/ocp-release
source: quay.io/openshift-release-dev/ocp-release
- mirrors:
- ${LOCAL_REGISTRY}/openshift-release-dev/ocp-release
source: quay.io/openshift-release-dev/ocp-v4.0-art-dev
EOM
oc apply -f icsp.yaml
# disconnected requires the additional trust bundle containing the local registry certificate
export EXTRA_HYPERSHIFT_CREATE_COMMANDS="$EXTRA_HYPERSHIFT_CREATE_COMMANDS --additional-trust-bundle ${REGISTRY_DIR}/certs/${REGISTRY_CRT}"
fi
# TODO: make SSH public key configurable
set -o nounset
set -o pipefail
set -o errexit
set -o xtrace
echo "Running Ansible playbook to create kubernetes objects"
ansible-playbook "${__dir}/assisted-installer-crds-playbook.yaml"
oc get namespace "${SPOKE_NAMESPACE}" || oc create namespace "${SPOKE_NAMESPACE}"
oc get secret "${ASSISTED_PULLSECRET_NAME}" -n "${SPOKE_NAMESPACE}" || \
oc create secret generic "${ASSISTED_PULLSECRET_NAME}" --from-file=.dockerconfigjson="${ASSISTED_PULLSECRET_JSON}" --type=kubernetes.io/dockerconfigjson -n "${SPOKE_NAMESPACE}"
oc get secret "${ASSISTED_PRIVATEKEY_NAME}" -n "${SPOKE_NAMESPACE}" || \
oc create secret generic "${ASSISTED_PRIVATEKEY_NAME}" --from-file=ssh-privatekey=/root/.ssh/id_rsa --type=kubernetes.io/ssh-auth -n "${SPOKE_NAMESPACE}"
for manifest in $(find ${__dir}/generated -type f); do
tee < "${manifest}" >(oc apply -f -)
done
wait_for_condition "infraenv/${ASSISTED_INFRAENV_NAME}" "ImageCreated" "5m" "${SPOKE_NAMESPACE}"
echo "Waiting until at least ${SPOKE_CONTROLPLANE_AGENTS} agents are available..."
function get_agents() {
oc get agent -n ${SPOKE_NAMESPACE} --no-headers
}
export -f wait_for_cmd_amount
export -f get_agents
timeout 20m bash -c "wait_for_cmd_amount ${SPOKE_CONTROLPLANE_AGENTS} 30 get_agents"
echo "All ${SPOKE_CONTROLPLANE_AGENTS} agents have been discovered!"
if [[ "${ASSISTED_STOP_AFTER_AGENT_DISCOVERY}" == "true" ]]; then
echo "Agents have been discovered, do not wait for the cluster installtion to finish."
exit
fi
# We need a storage for etcd of the hosted cluster
oc patch storageclass assisted-service -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
### Hypershift CLI needs access to the kubeconfig, pull-secret and public SSH key
function hypershift_cli() {
if command -v hypershift &> /dev/null
then
hypershift "$@"
else
podman run -it --net host --rm --entrypoint /usr/bin/hypershift -v $KUBECONFIG:/root/.kube/config -v $ASSISTED_PULLSECRET_JSON:$ASSISTED_PULLSECRET_JSON -v /root/.ssh/id_rsa.pub:/root/.ssh/id_rsa.pub $HYPERSHIFT_IMAGE "$@"
fi
}
echo "Installing HyperShift using upstream image"
hypershift_cli install --hypershift-image $HYPERSHIFT_IMAGE --namespace hypershift $EXTRA_HYPERSHIFT_INSTALL_FLAGS
if [ "${DISCONNECTED}" = "true" ]; then
# disconnected hypershift requires patching the operator deployment with the local image mirror of the capi agent
oc patch deploy/operator -n hypershift --type=strategic --patch="{\"spec\": {\"template\": {\"spec\": {\"containers\": [{\"name\": \"operator\",\"env\":[{\"name\":\"IMAGE_AGENT_CAPI_PROVIDER\",\"value\":\"${PROVIDER_IMAGE}\"}]}]}}}}"
# delete all rs since patching the deployment doesn't actually remove the running rs
oc delete rs --all -n hypershift
fi
wait_for_pods "hypershift"
if [ -z "$PROVIDER_IMAGE" ]
then
echo "PROVIDER_IMAGE override not set"
export PROVIDER_FLAG_FOR_CREATE_COMMAND=""
else
echo "PROVIDER_IMAGE override: $PROVIDER_IMAGE"
export PROVIDER_FLAG_FOR_CREATE_COMMAND=" --annotations hypershift.openshift.io/capi-provider-agent-image=$PROVIDER_IMAGE"
fi
if [ -z "$CONTROL_PLANE_OPERATOR_IMAGE" ]
then
echo "CONTROL_PLANE_OPERATOR_IMAGE override not set"
export CONTROL_PLANE_OPERATOR_FLAG_FOR_CREATE_COMMAND=""
else
echo "CONTROL_PLANE_OPERATOR_IMAGE override: $CONTROL_PLANE_OPERATOR_IMAGE"
export CONTROL_PLANE_OPERATOR_FLAG_FOR_CREATE_COMMAND=" --control-plane-operator-image $CONTROL_PLANE_OPERATOR_IMAGE"
fi
echo "Creating HostedCluster"
hypershift_cli create cluster agent --name $ASSISTED_CLUSTER_NAME --base-domain redhat.example --pull-secret $ASSISTED_PULLSECRET_JSON \
--ssh-key /root/.ssh/id_rsa.pub --agent-namespace $SPOKE_NAMESPACE --namespace $SPOKE_NAMESPACE \
--release-image ${ASSISTED_OPENSHIFT_INSTALL_RELEASE_IMAGE:-${RELEASE_IMAGE}} \
$CONTROL_PLANE_OPERATOR_FLAG_FOR_CREATE_COMMAND \
$PROVIDER_FLAG_FOR_CREATE_COMMAND \
$EXTRA_HYPERSHIFT_CREATE_COMMANDS
if [ "${DISCONNECTED}" = "true" ]; then
# Disconnected requires annotating the hosted cluster with the mirrored hypershift operator image (with digest instead of tag)
export HYPERSHIFT_IMAGE_WITH_DIGEST="${LOCAL_REGISTRY}/localimages/hypershift@$(oc image info $HYPERSHIFT_IMAGE -o json | jq -r '.digest')"
oc annotate hostedcluster $ASSISTED_CLUSTER_NAME hypershift.openshift.io/control-plane-operator-image=$HYPERSHIFT_IMAGE_WITH_DIGEST -n $SPOKE_NAMESPACE
fi
# Wait for a hypershift hostedcontrolplane to report ready status
wait_for_resource "hostedcontrolplane/${ASSISTED_CLUSTER_NAME}" "${SPOKE_NAMESPACE}-${ASSISTED_CLUSTER_NAME}"
wait_for_boolean_field "hostedcontrolplane/${ASSISTED_CLUSTER_NAME}" status.ready "${SPOKE_NAMESPACE}-${ASSISTED_CLUSTER_NAME}"
wait_for_condition "nodepool/$ASSISTED_CLUSTER_NAME" "Ready" "10m" "$SPOKE_NAMESPACE"
wait_for_condition "hostedcluster/$ASSISTED_CLUSTER_NAME" "Available" "10m" "$SPOKE_NAMESPACE"
# Scale up
echo "Scaling the hosted cluster up to contain ${SPOKE_CONTROLPLANE_AGENTS} worker nodes"
oc scale nodepool/$ASSISTED_CLUSTER_NAME -n $SPOKE_NAMESPACE --replicas=${SPOKE_CONTROLPLANE_AGENTS}
# Wait for node to appear in the CAPI-deployed cluster
oc extract -n $SPOKE_NAMESPACE secret/$ASSISTED_CLUSTER_NAME-admin-kubeconfig --to=- > /tmp/$ASSISTED_CLUSTER_NAME-kubeconfig
export HUB_KUBECONFIG=${KUBECONFIG}
export KUBECONFIG=/tmp/$ASSISTED_CLUSTER_NAME-kubeconfig
wait_for_object_amount node ${SPOKE_CONTROLPLANE_AGENTS} 10
echo "Worker nodes have been detected successfuly in the created cluster!"
echo "verify the BMH on the HUB cluster is detached"
export KUBECONFIG=${HUB_KUBECONFIG}
if [ $(oc get baremetalhost -n ${SPOKE_NAMESPACE} -o json | jq -c '.items[].metadata.annotations."baremetalhost.metal3.io/detached"| select("assisted-service-controller")' | wc -l) -ne ${SPOKE_CONTROLPLANE_AGENTS} ]; then
echo "The amount of detached BMHs on the HUB cluster doesn't match the amount of expected installed nodes in the spoke: ${SPOKE_CONTROLPLANE_AGENTS}"
echo "HUB cluster BMHs: "
oc get baremetalhost -n "${SPOKE_NAMESPACE}"
return 1
fi
echo "Destroy the hosted cluster"
hypershift_cli destroy cluster agent --name $ASSISTED_CLUSTER_NAME --namespace $SPOKE_NAMESPACE --cluster-grace-period 60m
echo "Successfully destroyed the hosted cluster"