Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Make GKE kube-apiserver use host IP to connect to etcd #95312

Merged
merged 1 commit into from Oct 20, 2020
Merged

Make GKE kube-apiserver use host IP to connect to etcd #95312

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions cluster/gce/config-default.sh
View file
Open in desktop
Expand Up @@ -524,3 +524,8 @@ export KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE="${KUBE_KONNECTIVITY_SERVICE_PRO

# Optional: Enable Windows CSI-Proxy
export ENABLE_CSI_PROXY="${ENABLE_CSI_PROXY:-true}"

# ETCD_LISTEN_ON_HOST_IP decides whether etcd servers should also listen on host IP,
# in addition to listening to 127.0.0.1, and whether kube-apiserver should connect to etcd servers
# through host IP.
export ETCD_LISTEN_ON_HOST_IP="${ETCD_LISTEN_ON_HOST_IP:-false}"
5 changes: 5 additions & 0 deletions cluster/gce/config-test.sh
View file
Open in desktop
Expand Up @@ -571,3 +571,8 @@ export GCE_UPLOAD_KUBCONFIG_TO_MASTER_METADATA=true

# Optoinal: Enable Windows CSI-Proxy
export ENABLE_CSI_PROXY="${ENABLE_CSI_PROXY:-true}"

# ETCD_LISTEN_ON_HOST_IP decides whether etcd servers should also listen on host IP,
# in addition to listening to 127.0.0.1, and whether kube-apiserver should connect to etcd servers
# through host IP.
export ETCD_LISTEN_ON_HOST_IP="${ETCD_LISTEN_ON_HOST_IP:-false}"
15 changes: 14 additions & 1 deletion cluster/gce/gci/apiserver_etcd_test.go
View file
Open in desktop
Expand Up @@ -37,6 +37,8 @@ type kubeAPIServeETCDEnv struct {
StorageBackend string
StorageMediaType string
CompactionInterval string
HostPrimaryIP string
ETCDListenOnHostIP string
}

func TestServerOverride(t *testing.T) {
Expand All @@ -52,7 +54,7 @@ func TestServerOverride(t *testing.T) {
},
},
{
desc: "ETCD-SERVERS and ETCD_SERVERS_OVERRIDES iare set",
desc: "ETCD-SERVERS and ETCD_SERVERS_OVERRIDES are set",
env: kubeAPIServeETCDEnv{
ETCDServers: "ETCDServers",
ETCDServersOverride: "ETCDServersOverrides",
Expand All @@ -61,6 +63,17 @@ func TestServerOverride(t *testing.T) {
"--etcd-servers-overrides=ETCDServersOverrides",
},
},
{
desc: "HOST_PRIMARY_IP is set and etcd is set to listen to host IP",
env: kubeAPIServeETCDEnv{
HostPrimaryIP: "HostPrimaryIP",
ETCDListenOnHostIP: "true",
},
want: []string{
"--etcd-servers-overrides=/events#http://HostPrimaryIP:4002",
"--etcd-servers=http://HostPrimaryIP:2379",
},
},
}

for _, tc := range testCases {
Expand Down
29 changes: 25 additions & 4 deletions cluster/gce/gci/configure-helper.sh
View file
Open in desktop
Expand Up @@ -1686,8 +1686,8 @@ function start-kube-proxy {
# Replaces the variables in the etcd manifest file with the real values, and then
# copy the file to the manifest dir
# $1: value for variable 'suffix'
# $2: value for variable 'port'
# $3: value for variable 'server_port'
# $2: value for variable 'port', for listening to clients
# $3: value for variable 'server_port', for etcd peering
# $4: value for variable 'cpulimit'
# $5: pod name, which should be either etcd or etcd-events
function prepare-etcd-manifest {
Expand All @@ -1706,19 +1706,39 @@ function prepare-etcd-manifest {
if [[ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]]; then
cluster_state="${INITIAL_ETCD_CLUSTER_STATE}"
fi

# Configure mTLS for etcd peers.
if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth "
etcd_protocol="https"
fi

# mTLS should only be enabled for etcd server but not etcd-events. if $1 suffix is empty, it's etcd server.
# host_primary_ip is the primary internal IP of the host.
# Override host primary IP if specifically provided.
local host_primary_ip
host_primary_ip="${HOST_PRIMARY_IP:-$(hostname -i)}"

# Configure mTLS for clients (e.g. kube-apiserver).
# mTLS should only be enabled for etcd server but not etcd-events. If $1 suffix is empty, it's etcd server.
local etcd_listen_metrics_urls=""
if [[ -z "${suffix}" && -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
etcd_apiserver_creds=" --client-cert-auth --trusted-ca-file ${ETCD_APISERVER_CA_CERT_PATH} --cert-file ${ETCD_APISERVER_SERVER_CERT_PATH} --key-file ${ETCD_APISERVER_SERVER_KEY_PATH} "
etcd_apiserver_protocol="https"
etcd_livenessprobe_port="2382"
etcd_extra_args+=" --listen-metrics-urls=http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_livenessprobe_port} "
etcd_listen_metrics_urls="http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_livenessprobe_port}"
if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]]; then
etcd_listen_metrics_urls+=",http://${host_primary_ip}:${etcd_livenessprobe_port}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we cannot connect to non-local IPs over http

fi
etcd_extra_args+=" --listen-metrics-urls=${etcd_listen_metrics_urls} "
fi

# If etcd is configured to listen on host IP, an additional client listening URL is added.
local etcd_listen_client_urls="${etcd_apiserver_protocol}://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:$2"
if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]] ; then
benhxy marked this conversation as resolved.
Show resolved Hide resolved
etcd_listen_client_urls+=",${etcd_apiserver_protocol}://${host_primary_ip}:$2"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is also not ok, it means there is the possibility to connect to a non-local IP over http

fi

# Generate etcd member URLs.
for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do
etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3"
if [[ -n "${etcd_cluster}" ]]; then
Expand All @@ -1738,6 +1758,7 @@ function prepare-etcd-manifest {
sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}"
sed -i -e "s@{{ *liveness_probe_initial_delay *}}@${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${temp_file}"
sed -i -e "s@{{ *listen_client_ip *}}@${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}@g" "${temp_file}"
sed -i -e "s@{{ *etcd_listen_client_urls *}}@${etcd_listen_client_urls:-}@g" "${temp_file}"
# Get default storage backend from manifest file.
local -r default_storage_backend=$( \
grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" "${temp_file}" | \
Expand Down
17 changes: 13 additions & 4 deletions cluster/gce/gci/configure-kubeapiserver.sh
View file
Open in desktop
Expand Up @@ -14,25 +14,34 @@
# limitations under the License.


# Configures etcd related flags of kube-apiserver.
# Configures etcd related parameters of kube-apiserver.
function configure-etcd-params {
local -n params_ref=$1

local host_ip="127.0.0.1"
# If etcd is configured to listen on host IP,
# host_ip is set to the primary internal IP of host VM.
if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]] ; then
host_ip="${HOST_PRIMARY_IP:-$(hostname -i)}"
fi

# Configure the main etcd.
if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://127.0.0.1:2379}"
params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://${host_ip}:2379}"
params_ref+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}"
params_ref+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}"
params_ref+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}"
elif [[ -z "${ETCD_APISERVER_CA_KEY:-}" && -z "${ETCD_APISERVER_CA_CERT:-}" && -z "${ETCD_APISERVER_SERVER_KEY:-}" && -z "${ETCD_APISERVER_SERVER_CERT:-}" && -z "${ETCD_APISERVER_CLIENT_KEY:-}" && -z "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}"
params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://${host_ip}:2379}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is not ok... this means kube-apiserver is connecting to etcd over a potentially public IP over http.

echo "WARNING: ALL of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver is not enabled."
else
echo "ERROR: Some of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver cannot be enabled. Please provide all mTLS credential."
exit 1
fi

# Configure the event log etcd.
if [[ -z "${ETCD_SERVERS:-}" ]]; then
params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://127.0.0.1:4002}"
params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://${host_ip}:4002}"
elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then
params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}"
fi
Expand Down
2 changes: 2 additions & 0 deletions cluster/gce/gci/testdata/kube-apiserver/etcd.template
View file
Open in desktop
Expand Up @@ -13,3 +13,5 @@ readonly ETCD_SERVERS_OVERRIDES={{.ETCDServersOverride}}
readonly STORAGE_BACKEND={{.StorageBackend}}
readonly STORAGE_MEDIA_TYPE={{.StorageMediaType}}
readonly ETCD_COMPACTION_INTERVAL_SEC={{.CompactionInterval}}
readonly HOST_PRIMARY_IP={{.HostPrimaryIP}}
readonly ETCD_LISTEN_ON_HOST_IP={{.ETCDListenOnHostIP}}
7 changes: 5 additions & 2 deletions cluster/gce/manifests/etcd.manifest
View file
Open in desktop
Expand Up @@ -26,7 +26,7 @@
"command": [
"/bin/sh",
"-c",
"set -o errexit; if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls {{ etcd_apiserver_protocol }}://127.0.0.1:{{ port }} --listen-client-urls {{ etcd_apiserver_protocol }}://{{ listen_client_ip }}:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
"set -o errexit; if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls {{ etcd_apiserver_protocol }}://127.0.0.1:{{ port }} --listen-client-urls {{ etcd_listen_client_urls }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
],
"env": [
{ "name": "TARGET_STORAGE",
Expand Down Expand Up @@ -58,8 +58,11 @@
},
{ "name": "ETCD_HOSTNAME",
"value": "{{ hostname }}"
},
{ "name": "LISTEN_CLIENT_URLS",
"value": "{{ etcd_listen_client_urls }}"
}
],
],
"livenessProbe": {
"httpGet": {
"host": "127.0.0.1",
Expand Down