forked from kubernetes/kubernetes
/
common.sh
336 lines (279 loc) · 14.3 KB
/
common.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# required:
# KUBE_ROOT: path of the root of the Kubernetes reposiitory
# optional override
# FEDERATION_IMAGE_REPO_BASE: repo which federated images are tagged under (default gcr.io/google_containers)
# FEDERATION_NAMESPACE: name of the namespace will created for the federated components in the underlying cluster.
# KUBE_PLATFORM
# KUBE_ARCH
# KUBE_BUILD_STAGE
: "${KUBE_ROOT?Must set KUBE_ROOT env var}"
# Provides the $KUBERNETES_PROVIDER variable and detect-project function
source "${KUBE_ROOT}/cluster/kube-util.sh"
# If $FEDERATION_PUSH_REPO_BASE isn't set, then set the GCR registry name
# based on the detected project name for gce and gke providers.
FEDERATION_PUSH_REPO_BASE=${FEDERATION_PUSH_REPO_BASE:-}
if [[ -z "${FEDERATION_PUSH_REPO_BASE}" ]]; then
if [[ "${KUBERNETES_PROVIDER}" == "gke" || "${KUBERNETES_PROVIDER}" == "gce" ]]; then
# Populates $PROJECT
detect-project
if [[ ${PROJECT} == *':'* ]]; then
echo "${PROJECT} contains ':' and can not be used as FEDERATION_PUSH_REPO_BASE. Please set FEDERATION_PUSH_REPO_BASE explicitly."
exit 1
fi
FEDERATION_PUSH_REPO_BASE=gcr.io/${PROJECT}
else
echo "Must set FEDERATION_PUSH_REPO_BASE env var"
exit 1
fi
fi
FEDERATION_IMAGE_REPO_BASE=${FEDERATION_IMAGE_REPO_BASE:-'gcr.io/google_containers'}
FEDERATION_NAMESPACE=${FEDERATION_NAMESPACE:-federation}
KUBE_PLATFORM=${KUBE_PLATFORM:-linux}
KUBE_ARCH=${KUBE_ARCH:-amd64}
KUBE_BUILD_STAGE=${KUBE_BUILD_STAGE:-release-stage}
source "${KUBE_ROOT}/cluster/common.sh"
host_kubectl="${KUBE_ROOT}/cluster/kubectl.sh --namespace=${FEDERATION_NAMESPACE}"
# required:
# FEDERATION_PUSH_REPO_BASE: repo to which federated container images will be pushed
# FEDERATION_IMAGE_TAG: reference and pull all federated images with this tag.
function create-federation-api-objects {
(
: "${FEDERATION_PUSH_REPO_BASE?Must set FEDERATION_PUSH_REPO_BASE env var}"
: "${FEDERATION_IMAGE_TAG?Must set FEDERATION_IMAGE_TAG env var}"
export FEDERATION_APISERVER_DEPLOYMENT_NAME="federation-apiserver"
export FEDERATION_APISERVER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube"
export FEDERATION_APISERVER_IMAGE_TAG="${FEDERATION_IMAGE_TAG}"
export FEDERATION_CONTROLLER_MANAGER_DEPLOYMENT_NAME="federation-controller-manager"
export FEDERATION_CONTROLLER_MANAGER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube"
export FEDERATION_CONTROLLER_MANAGER_IMAGE_TAG="${FEDERATION_IMAGE_TAG}"
if [[ -z "${FEDERATION_DNS_PROVIDER:-}" ]]; then
# Set the appropriate value based on cloud provider.
if [[ "$KUBERNETES_PROVIDER" == "gce" || "${KUBERNETES_PROVIDER}" == "gke" ]]; then
echo "setting dns provider to google-clouddns"
export FEDERATION_DNS_PROVIDER="google-clouddns"
elif [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
echo "setting dns provider to aws-route53"
export FEDERATION_DNS_PROVIDER="aws-route53"
else
echo "Must set FEDERATION_DNS_PROVIDER env var"
exit 1
fi
fi
export FEDERATION_SERVICE_CIDR=${FEDERATION_SERVICE_CIDR:-"10.10.0.0/24"}
#Only used for providers that require a nodeport service (vagrant for now)
#We will use loadbalancer services where we can
export FEDERATION_API_NODEPORT=32111
export FEDERATION_NAMESPACE
export FEDERATION_NAME="${FEDERATION_NAME:-federation}"
export DNS_ZONE_NAME="${DNS_ZONE_NAME:-federation.example.}" # See https://tools.ietf.org/html/rfc2606
template="go run ${KUBE_ROOT}/federation/cluster/template.go"
FEDERATION_KUBECONFIG_PATH="${KUBE_ROOT}/federation/cluster/kubeconfig"
federation_kubectl="${KUBE_ROOT}/cluster/kubectl.sh --context=federation-cluster --namespace=default"
manifests_root="${KUBE_ROOT}/federation/manifests/"
$template "${manifests_root}/federation-ns.yaml" | $host_kubectl apply -f -
cleanup-federation-api-objects
export FEDERATION_API_HOST=""
export KUBE_MASTER_IP=""
if [[ "$KUBERNETES_PROVIDER" == "vagrant" ]];then
# The vagrant approach is to use a nodeport service, and point kubectl at one of the nodes
$template "${manifests_root}/federation-apiserver-nodeport-service.yaml" | $host_kubectl create -f -
node_addresses=`$host_kubectl get nodes -o=jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}'`
FEDERATION_API_HOST=`printf "$node_addresses" | cut -d " " -f1`
KUBE_MASTER_IP="${FEDERATION_API_HOST}:${FEDERATION_API_NODEPORT}"
elif [[ "$KUBERNETES_PROVIDER" == "gce" || "$KUBERNETES_PROVIDER" == "gke" || "$KUBERNETES_PROVIDER" == "aws" ]];then
# any capable providers should use a loadbalancer service
# we check for ingress.ip and ingress.hostname, so should work for any loadbalancer-providing provider
# allows 30x5 = 150 seconds for loadbalancer creation
$template "${manifests_root}/federation-apiserver-lb-service.yaml" | $host_kubectl create -f -
for i in {1..30};do
echo "attempting to get federation-apiserver loadbalancer hostname ($i / 30)"
for field in ip hostname;do
FEDERATION_API_HOST=`${host_kubectl} get -o=jsonpath svc/${FEDERATION_APISERVER_DEPLOYMENT_NAME} --template '{.status.loadBalancer.ingress[*].'"${field}}"`
if [[ ! -z "${FEDERATION_API_HOST// }" ]];then
break 2
fi
done
if [[ $i -eq 30 ]];then
echo "Could not find ingress hostname for federation-apiserver loadbalancer service"
exit 1
fi
sleep 5
done
KUBE_MASTER_IP="${FEDERATION_API_HOST}:443"
else
echo "provider ${KUBERNETES_PROVIDER} is not (yet) supported for e2e testing"
exit 1
fi
echo "Found federation-apiserver host at $FEDERATION_API_HOST"
FEDERATION_API_TOKEN="$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)"
export FEDERATION_API_KNOWN_TOKENS="${FEDERATION_API_TOKEN},admin,admin"
gen-kube-basicauth
export FEDERATION_API_BASIC_AUTH="${KUBE_PASSWORD},${KUBE_USER},admin"
# Create a kubeconfig with credentails for federation-apiserver. We will
# then use this kubeconfig to create a secret which the federation
# controller manager can use to talk to the federation-apiserver.
# Note that the file name should be "kubeconfig" so that the secret key gets the same name.
KUBECONFIG_DIR=$(dirname ${KUBECONFIG:-$DEFAULT_KUBECONFIG})
CONTEXT=federation-cluster \
KUBE_BEARER_TOKEN="$FEDERATION_API_TOKEN" \
KUBE_USER="${KUBE_USER}" \
KUBE_PASSWORD="${KUBE_PASSWORD}" \
KUBECONFIG="${KUBECONFIG_DIR}/federation/federation-apiserver/kubeconfig" \
create-kubeconfig
# Create secret with federation-apiserver's kubeconfig
$host_kubectl create secret generic federation-apiserver-kubeconfig --from-file="${KUBECONFIG_DIR}/federation/federation-apiserver/kubeconfig" --namespace="${FEDERATION_NAMESPACE}"
# Create secrets with all the kubernetes-apiserver's kubeconfigs.
# Note: This is used only by the test setup (where kubernetes clusters are
# brought up with FEDERATION=true). Users are expected to create this secret
# themselves.
for dir in ${KUBECONFIG_DIR}/federation/kubernetes-apiserver/*; do
# We create a secret with the same name as the directory name (which is
# same as cluster name in kubeconfig).
# Massage the name so that it is valid (should not contain "_" and max 253
# chars)
name=$(basename $dir)
name=$(echo "$name" | sed -e "s/_/-/g") # Replace "_" by "-"
name=${name:0:252}
echo "Creating secret with name: $name"
$host_kubectl create secret generic ${name} --from-file="${dir}/kubeconfig" --namespace="${FEDERATION_NAMESPACE}"
done
# Create server certificates.
ensure-temp-dir
echo "Creating federation apiserver certs for IP: $FEDERATION_API_HOST"
MASTER_NAME="federation-apiserver" create-federation-apiserver-certs ${FEDERATION_API_HOST}
export FEDERATION_APISERVER_CA_CERT_BASE64="${FEDERATION_APISERVER_CA_CERT_BASE64}"
export FEDERATION_APISERVER_CERT_BASE64="${FEDERATION_APISERVER_CERT_BASE64}"
export FEDERATION_APISERVER_KEY_BASE64="${FEDERATION_APISERVER_KEY_BASE64}"
# Enable the NamespaceLifecycle admission control by default.
export FEDERATION_ADMISSION_CONTROL="${FEDERATION_ADMISSION_CONTROL:-NamespaceLifecycle}"
for file in federation-etcd-pvc.yaml federation-apiserver-{deployment,secrets}.yaml federation-controller-manager-deployment.yaml; do
$template "${manifests_root}/${file}" | $host_kubectl create -f -
done
# Update the users kubeconfig to include federation-apiserver credentials.
CONTEXT=federation-cluster \
KUBE_BEARER_TOKEN="$FEDERATION_API_TOKEN" \
KUBE_USER="${KUBE_USER}" \
KUBE_PASSWORD="${KUBE_PASSWORD}" \
SECONDARY_KUBECONFIG=true \
create-kubeconfig
# Don't finish provisioning until federation-apiserver pod is running
for i in {1..30};do
#TODO(colhom): in the future this needs to scale out for N pods. This assumes just one pod
phase="$($host_kubectl get -o=jsonpath pods -lapp=federated-cluster,module=federation-apiserver --template '{.items[*].status.phase}')"
echo "Waiting for federation-apiserver to be running...(phase= $phase)"
if [[ "$phase" == "Running" ]];then
echo "federation-apiserver pod is running!"
break
fi
if [[ $i -eq 30 ]];then
echo "federation-apiserver pod is not running! giving up."
exit 1
fi
sleep 4
done
# Verify that federation-controller-manager pod is running.
for i in {1..30};do
#TODO(colhom): in the future this needs to scale out for N pods. This assumes just one pod
phase="$($host_kubectl get -o=jsonpath pods -lapp=federated-cluster,module=federation-controller-manager --template '{.items[*].status.phase}')"
echo "Waiting for federation-controller-manager to be running...(phase= $phase)"
if [[ "$phase" == "Running" ]];then
echo "federation-controller-manager pod is running!"
break
fi
if [[ $i -eq 30 ]];then
echo "federation-controller-manager pod is not running! giving up."
exit 1
fi
sleep 4
done
)
}
# Creates the required certificates for federation apiserver.
# $1: The public IP for the master.
#
# Assumed vars
# KUBE_TEMP
# MASTER_NAME
#
function create-federation-apiserver-certs {
local -r primary_cn="${1}"
local sans="IP:${1},DNS:${MASTER_NAME}"
echo "Generating certs for alternate-names: ${sans}"
local kube_temp="${KUBE_TEMP}/federation"
mkdir -p "${kube_temp}"
KUBE_TEMP="${kube_temp}" PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs
local cert_dir="${kube_temp}/easy-rsa-master/easyrsa3"
# By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces.
# Note 'base64 -w0' doesn't work on Mac OS X, which has different flags.
FEDERATION_APISERVER_CA_CERT_BASE64=$(cat "${cert_dir}/pki/ca.crt" | base64 | tr -d '\r\n')
FEDERATION_APISERVER_CERT_BASE64=$(cat "${cert_dir}/pki/issued/${MASTER_NAME}.crt" | base64 | tr -d '\r\n')
FEDERATION_APISERVER_KEY_BASE64=$(cat "${cert_dir}/pki/private/${MASTER_NAME}.key" | base64 | tr -d '\r\n')
}
# Required
# FEDERATION_PUSH_REPO_BASE: the docker repo where federated images will be pushed
# FEDERATION_IMAGE_TAG: the tag of the image to be pushed
function push-federation-images {
: "${FEDERATION_PUSH_REPO_BASE?Must set FEDERATION_PUSH_REPO_BASE env var}"
: "${FEDERATION_IMAGE_TAG?Must set FEDERATION_IMAGE_TAG env var}"
source "${KUBE_ROOT}/build/common.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
local FEDERATION_BINARIES=${FEDERATION_BINARIES:-"hyperkube"}
local bin_dir="${KUBE_ROOT}/_output/${KUBE_BUILD_STAGE}/server/${KUBE_PLATFORM}-${KUBE_ARCH}/kubernetes/server/bin"
if [[ ! -d "${bin_dir}" ]];then
echo "${bin_dir} does not exist! Run make quick-release or make release"
exit 1
fi
for binary in ${FEDERATION_BINARIES}; do
local bin_path="${bin_dir}/${binary}"
if [[ ! -f "${bin_path}" ]]; then
echo "${bin_path} does not exist!"
exit 1
fi
local docker_build_path="${bin_path}.dockerbuild"
local docker_file_path="${docker_build_path}/Dockerfile"
rm -rf ${docker_build_path}
mkdir -p ${docker_build_path}
ln "${bin_path}" "${docker_build_path}/${binary}"
printf " FROM debian:jessie \n ADD ${binary} /usr/local/bin/${binary}\n" > ${docker_file_path}
local docker_image_tag="${FEDERATION_PUSH_REPO_BASE}/${binary}:${FEDERATION_IMAGE_TAG}"
# Build the docker image on-the-fly.
#
# NOTE: This is only a temporary fix until the proposal in issue
# https://github.com/kubernetes/kubernetes/issues/28630 is implemented.
# Also, the new turn up mechanism completely obviates this step.
#
# TODO(madhusudancs): Remove this code when the new turn up mechanism work
# is merged.
kube::log::status "Building docker image ${docker_image_tag} from the binary"
docker build -q -t "${docker_image_tag}" ${docker_build_path} >/dev/null
rm -rf ${docker_build_path}
kube::log::status "Pushing ${docker_image_tag}"
if [[ "${FEDERATION_PUSH_REPO_BASE}" == "gcr.io/"* ]]; then
echo " -> GCR repository detected. Using gcloud"
gcloud docker push "${docker_image_tag}"
else
docker push "${docker_image_tag}"
fi
kube::log::status "Deleting docker image ${docker_image_tag}"
docker rmi "${docker_image_tag}" 2>/dev/null || true
done
}
function cleanup-federation-api-objects {
# Delete all resources with the federated-cluster label.
$host_kubectl delete pods,svc,rc,deployment,secret -lapp=federated-cluster
# Delete all resources in FEDERATION_NAMESPACE.
$host_kubectl delete pods,svc,rc,deployment,secret --namespace=${FEDERATION_NAMESPACE} --all
}