/
kubernetes
245 lines (225 loc) · 7.76 KB
/
kubernetes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
#!/bin/bash
KURYR_KUBEADMIN_IMAGE_REPOSITORY="registry.k8s.io"
function get_k8s_log_level {
if [[ ${ENABLE_DEBUG_LOG_LEVEL} == "True" ]]; then
echo "4"
else
echo "2"
fi
}
function kubeadm_install {
if ! is_ubuntu && ! is_fedora; then
(>&2 echo "WARNING: kubeadm installation is not supported in this \
distribution.")
return
fi
if is_ubuntu; then
apt_get install apt-transport-https gpg
sudo mkdir -p -m 755 /etc/apt/keyrings
curl -fsSL https://pkgs.k8s.io/core:/stable:/v${KURYR_KUBERNETES_VERSION%.*}/deb/Release.key | \
sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v'${KURYR_KUBERNETES_VERSION%.*}'/deb/ /' | \
sudo tee /etc/apt/sources.list.d/kubernetes.list
REPOS_UPDATED=False apt_get_update
# NOTE(gryf): kubectl will be installed alongside with the kubeadm as
# a dependency, although let's pin it to the k8s version as well.
kube_pkg_version=$(sudo apt-cache show kubeadm | grep "Version: $KURYR_KUBERNETES_VERSION-" | awk '{ print $2 }')
apt_get install \
kubelet="${kube_pkg_version}" \
kubeadm="${kube_pkg_version}" \
kubectl="${kube_pkg_version}"
sudo apt-mark hold kubelet kubeadm kubectl
# NOTE(hongbin): This work-around an issue that kubelet pick a wrong
# IP address if the node has multiple network interfaces.
# See https://github.com/kubernetes/kubeadm/issues/203
echo "KUBELET_EXTRA_ARGS=--node-ip=$HOST_IP" | sudo tee -a \
/etc/default/kubelet
sudo systemctl daemon-reload && sudo systemctl restart kubelet
fi
if is_fedora; then
source /etc/os-release
os_VENDOR=$(echo $NAME | tr -d '[:space:]')
if [[ $os_VENDOR =~ "CentOS" ]]; then
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg \
https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF
sudo chmod 755 /etc/yum.repos.d/kubernetes.repo
sudo dnf install kubeadm -y
fi
fi
}
function kubeadm_init {
local cluster_ip_ranges
local output_dir="${DATA_DIR}/kuryr-kubernetes"
local cgroup_driver
local cri_socket
mkdir -p "${output_dir}"
if [[ ${CONTAINER_ENGINE} == 'crio' ]]; then
local crio_conf="/etc/crio/crio.conf"
cgroup_driver=$(iniget ${crio_conf} crio.runtime cgroup_manager)
cri_socket="unix:///var/run/crio/crio.sock"
else
# docker is used
cgroup_driver=$(docker info -f '{{.CgroupDriver}}')
cri_socket="/var/run/dockershim.sock"
fi
cluster_ip_ranges=()
for service_subnet_id in ${KURYR_SERVICE_SUBNETS_IDS[@]}; do
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$service_subnet_id" \
-c cidr -f value)
cluster_ip_ranges+=($(split_subnet "$service_cidr" | cut -f1))
done
# TODO(gryf): take care of cri-o case aswell
rm -f ${output_dir}/kubeadm-init.yaml
cat >> ${output_dir}/kubeadm-init.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
imageRepository: "${KURYR_KUBEADMIN_IMAGE_REPOSITORY}"
etcd:
external:
endpoints:
- "http://${SERVICE_HOST}:${ETCD_PORT}"
networking:
serviceSubnet: "$(IFS=, ; echo "${cluster_ip_ranges[*]}")"
apiServer:
extraArgs:
endpoint-reconciler-type: "none"
min-request-timeout: "300"
allow-privileged: "true"
v: "$(get_k8s_log_level)"
controllerManager:
extraArgs:
master: "$KURYR_K8S_API_URL"
min-resync-period: "3m"
v: "$(get_k8s_log_level)"
leader-elect: "false"
scheduler:
extraArgs:
master: "${KURYR_K8S_API_URL}"
v: "$(get_k8s_log_level)"
leader-elect: "false"
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
bootstrapTokens:
- token: "${KURYR_K8S_TOKEN}"
ttl: 0s
localAPIEndpoint:
advertiseAddress: "${K8S_API_SERVER_IP}"
bindPort: ${K8S_API_SERVER_PORT}
nodeRegistration:
criSocket: "$cri_socket"
kubeletExtraArgs:
enable-server: "true"
taints:
[]
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
failSwapOn: false
address: "0.0.0.0"
enableServer: true
cgroupDriver: $cgroup_driver
EOF
sudo kubeadm config images pull --image-repository=${KURYR_KUBEADMIN_IMAGE_REPOSITORY}
args="--config ${output_dir}/kubeadm-init.yaml"
# NOTE(gryf): skip installing kube proxy, kuryr will handle services.
args+=" --skip-phases=addon/kube-proxy"
args+=" --ignore-preflight-errors Swap"
if ! is_service_enabled coredns; then
# FIXME(gryf): Do we need specific configuration for coredns?
args+=" --skip-phases=addon/coredns"
fi
sudo kubeadm init $args
local kube_config_file=$HOME/.kube/config
mkdir -p $(dirname ${kube_config_file})
sudo cp /etc/kubernetes/admin.conf $kube_config_file
safe_chown $STACK_USER:$STACK_USER $kube_config_file
}
function kubeadm_join {
local output_dir="${DATA_DIR}/kuryr-kubernetes"
local cgroup_driver
local cri_socket
mkdir -p "${output_dir}"
if [[ ${CONTAINER_ENGINE} == 'crio' ]]; then
local crio_conf="/etc/crio/crio.conf"
cgroup_driver=$(iniget ${crio_conf} crio.runtime cgroup_manager)
cri_socket="unix:///var/run/crio/crio.sock"
else
# docker is used
cgroup_driver=$(docker info -f '{{.CgroupDriver}}')
cri_socket="/var/run/dockershim.sock"
fi
cluster_ip_ranges=()
for service_subnet_id in ${KURYR_SERVICE_SUBNETS_IDS[@]}; do
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$service_subnet_id" \
-c cidr -f value)
cluster_ip_ranges+=($(split_subnet "$service_cidr" | cut -f1))
done
# TODO(gryf): take care of cri-o case aswell
rm -f ${output_dir}/kubeadm-join.yaml
cat >> ${output_dir}/kubeadm-join.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta3
discovery:
bootstrapToken:
apiServerEndpoint: ${SERVICE_HOST}:${KURYR_K8S_API_PORT}
token: "${KURYR_K8S_TOKEN}"
unsafeSkipCAVerification: true
tlsBootstrapToken: "${KURYR_K8S_TOKEN}"
kind: JoinConfiguration
nodeRegistration:
criSocket: "$cri_socket"
kubeletExtraArgs:
enable-server: "true"
taints:
[]
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
failSwapOn: false
address: "0.0.0.0"
enableServer: true
cgroupDriver: $cgroup_driver
EOF
sudo -E kubeadm join --ignore-preflight-errors Swap \
--config ${output_dir}/kubeadm-join.yaml
}
function get_k8s_apiserver {
# assumption is, there is no other cluster, so there is only one API
# server.
echo "$(kubectl config view -o jsonpath='{.clusters[].cluster.server}')"
}
function get_k8s_token {
local secret
secret=$(kubectl get secrets -o jsonpath='{.items[0].metadata.name}')
echo $(kubectl get secret $secret -o jsonpath='{.items[0].data.token}' | \
base64 -d)
}
function kubeadm_reset {
sudo kubeadm reset -f
sudo iptables -F
sudo iptables -t nat -F
sudo iptables -t mangle -F
sudo iptables -X
sudo ipvsadm -C
}
function kubeadm_uninstall {
sudo systemctl stop kubelet
apt_get purge --allow-change-held-packages. kubelet kubeadm kubeadm \
kubernetes-cni apt-transport-https
sudo add-apt-repository -r -y \
"deb https://apt.kubernetes.io/ kubernetes-xenial main"
REPOS_UPDATED=False apt_get_update
sudo rm -fr /etc/default/kubelet /etc/kubernetes
}