diff --git a/cluster/common.sh b/cluster/common.sh index 01c1817440e21..00f00f1706001 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -702,6 +702,10 @@ NUM_NODES: $(yaml-quote ${NUM_NODES}) STORAGE_BACKEND: $(yaml-quote ${STORAGE_BACKEND:-etcd2}) ENABLE_GARBAGE_COLLECTOR: $(yaml-quote ${ENABLE_GARBAGE_COLLECTOR:-}) MASTER_ADVERTISE_ADDRESS: $(yaml-quote ${MASTER_ADVERTISE_ADDRESS:-}) +ETCD_CA_KEY: $(yaml-quote ${ETCD_CA_KEY_BASE64:-}) +ETCD_CA_CERT: $(yaml-quote ${ETCD_CA_CERT_BASE64:-}) +ETCD_PEER_KEY: $(yaml-quote ${ETCD_PEER_KEY_BASE64:-}) +ETCD_PEER_CERT: $(yaml-quote ${ETCD_PEER_CERT_BASE64:-}) EOF # ETCD_IMAGE (if set) allows to use a custom etcd image. if [ -n "${ETCD_IMAGE:-}" ]; then diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index f6190b6a2562d..b4bf9fd7c6d25 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -457,6 +457,7 @@ num_nodes: $(echo "${NUM_NODES:-}" | sed -e "s/'/''/g") e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")' kube_uid: '$(echo "${KUBE_UID}" | sed -e "s/'/''/g")' initial_etcd_cluster: '$(echo "${INITIAL_ETCD_CLUSTER:-}" | sed -e "s/'/''/g")' + hostname: $(hostname -s) EOF if [ -n "${STORAGE_BACKEND:-}" ]; then @@ -482,6 +483,15 @@ EOF if [ -n "${ETCD_VERSION:-}" ]; then cat <>/srv/salt-overlay/pillar/cluster-params.sls etcd_version: '$(echo "$ETCD_VERSION" | sed -e "s/'/''/g")' +EOF + fi + if [[ -n "${ETCD_CA_KEY:-}" && -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then + cat <>/srv/salt-overlay/pillar/cluster-params.sls +etcd_over_ssl: 'true' +EOF + else + cat <>/srv/salt-overlay/pillar/cluster-params.sls +etcd_over_ssl: 'false' EOF fi # Configuration changes for test clusters @@ -1054,6 +1064,15 @@ function run-user-script() { fi } +function create-salt-master-etcd-auth { + if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then + local -r auth_dir="/srv/kubernetes" + echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt" + echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key" + echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt" + fi +} + # This script is re-used on AWS. Some of the above functions will be replaced. # The AWS kube-up script looks for this marker: #+AWS_OVERRIDES_HERE @@ -1074,6 +1093,7 @@ if [[ -z "${is_push}" ]]; then create-salt-pillar if [[ "${KUBERNETES_MASTER}" == "true" ]]; then create-salt-master-auth + create-salt-master-etcd-auth create-salt-master-kubelet-auth else create-salt-kubelet-auth diff --git a/cluster/gce/debian/master-helper.sh b/cluster/gce/debian/master-helper.sh index c2a0d5312d8ab..52607aa9a4073 100755 --- a/cluster/gce/debian/master-helper.sh +++ b/cluster/gce/debian/master-helper.sh @@ -52,6 +52,17 @@ function replicate-master-instance() { # Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering. kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER")" kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER: '${existing_master_replicas},${REPLICA_NAME}'")" + + ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//")" + ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//")" + + create-etcd-certs "${ETCD_CA_CERT}" "${ETCD_CA_KEY}" + + kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")" + kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")" + kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")" + kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")" + echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > ${KUBE_TEMP}/cluster-name.txt get-metadata "${existing_master_zone}" "${existing_master_name}" startup-script > ${KUBE_TEMP}/configure-vm.sh diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 54677e583842a..24aba9d0322d5 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -375,6 +375,15 @@ current-context: service-account-context EOF } +function create-master-etcd-auth { + if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then + local -r auth_dir="/etc/srv/kubernetes" + echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt" + echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key" + echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt" + fi +} + function assemble-docker-flags { echo "Assemble docker command line flags" local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" @@ -606,14 +615,23 @@ function prepare-etcd-manifest { local host_name=$(hostname) local etcd_cluster="" local cluster_state="new" + local etcd_protocol="http" + local etcd_creds="" + + if [[ -n "${ETCD_CA_KEY:-}" && -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then + etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth " + etcd_protocol="https" + fi + for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do - etcd_host="etcd-${host}=http://${host}:$3" + etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3" if [[ -n "${etcd_cluster}" ]]; then etcd_cluster+="," cluster_state="existing" fi etcd_cluster+="${etcd_host}" done + local -r temp_file="/tmp/$5" cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd.manifest" "${temp_file}" remove-salt-config-comments "${temp_file}" @@ -622,6 +640,7 @@ function prepare-etcd-manifest { sed -i -e "s@{{ *server_port *}}@$3@g" "${temp_file}" sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${temp_file}" sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}" + sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" if [[ "${STORAGE_BACKEND:-}" == "etcd3" ]]; then @@ -635,6 +654,8 @@ function prepare-etcd-manifest { else sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${temp_file}" fi + sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}" + sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}" if [[ -n "${ETCD_VERSION:-}" ]]; then sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${temp_file}" else @@ -1209,6 +1230,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then mount-master-pd create-master-auth create-master-kubelet-auth + create-master-etcd-auth else create-kubelet-kubeconfig create-kubeproxy-kubeconfig diff --git a/cluster/gce/gci/master-helper.sh b/cluster/gce/gci/master-helper.sh index 7c15643dd9499..e83bb18ef29fc 100755 --- a/cluster/gce/gci/master-helper.sh +++ b/cluster/gce/gci/master-helper.sh @@ -49,6 +49,16 @@ function replicate-master-instance() { # Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering. kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER")" kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER: '${existing_master_replicas},${REPLICA_NAME}'")" + ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//")" + ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//")" + + create-etcd-certs "${ETCD_CA_CERT}" "${ETCD_CA_KEY}" + + kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")" + kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")" + kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")" + kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")" + echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt" get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt" @@ -58,6 +68,7 @@ function replicate-master-instance() { create-master-instance-internal "${REPLICA_NAME}" } + function create-master-instance-internal() { local -r master_name="${1}" local -r address_option="${2:-}" diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index 720f2a6fb2f9f..0b7edfe3de112 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -428,6 +428,15 @@ create_master_kubelet_auth() { fi } +function create-master-etcd-auth { + if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then + local -r auth_dir="/etc/srv/kubernetes" + echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt" + echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key" + echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt" + fi +} + # Replaces the variables in the etcd manifest file with the real values, and then # copy the file to the manifest dir # $1: value for variable 'suffix' @@ -439,14 +448,23 @@ prepare_etcd_manifest() { local host_name=$(hostname) local etcd_cluster="" local cluster_state="new" + local etcd_protocol="http" + local etcd_creds="" + + if [[ -n "${ETCD_CA_KEY:-}" && -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then + etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth " + etcd_protocol="https" + fi + for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do - etcd_host="etcd-${host}=http://${host}:$3" + etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3" if [[ -n "${etcd_cluster}" ]]; then etcd_cluster+="," cluster_state="existing" fi etcd_cluster+="${etcd_host}" done + etcd_temp_file="/tmp/$5" cp /home/kubernetes/kube-manifests/kubernetes/gci-trusty/etcd.manifest "${etcd_temp_file}" remove_salt_config_comments "${etcd_temp_file}" @@ -454,6 +472,7 @@ prepare_etcd_manifest() { sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}" sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}" sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${etcd_temp_file}" + sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${etcd_temp_file}" sed -i -e "s@{{ *hostname *}}@$host_name@g" "${etcd_temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${etcd_temp_file}" sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" @@ -468,6 +487,8 @@ prepare_etcd_manifest() { else sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${etcd_temp_file}" fi + sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${etcd_temp_file}" + sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${etcd_temp_file}" if [[ -n "${ETCD_VERSION:-}" ]]; then sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${etcd_temp_file}" else diff --git a/cluster/gce/trusty/master.yaml b/cluster/gce/trusty/master.yaml index 73cf72c3bb5e7..1c6b9f78438c9 100644 --- a/cluster/gce/trusty/master.yaml +++ b/cluster/gce/trusty/master.yaml @@ -65,6 +65,8 @@ script create_master_auth echo "Creating master instance kubelet auth file" create_master_kubelet_auth + echo "Creating auth files for etcd" + create-master-etcd-auth echo "Assemble kubelet command line" # Kubelet command flags will be written in /etc/default/kubelet assemble_kubelet_flags diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index e2069510c5847..93f6995d40a4f 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -741,6 +741,86 @@ function get-master-disk-size() { fi } +# Generates SSL certificates for etcd cluster. Uses cfssl program. +# +# Assumed vars: +# KUBE_TEMP: temporary directory +# +# Args: +# $1: CA certificate +# $2: CA key +# +# If CA cert/key is empty, the function will also generate certs for CA. +# +# Vars set: +# ETCD_CA_KEY_BASE64 +# ETCD_CA_CERT_BASE64 +# ETCD_PEER_KEY_BASE64 +# ETCD_PEER_CERT_BASE64 +# +function create-etcd-certs { + local ca_cert=${1:-} + local ca_key=${2:-} + + mkdir -p "${KUBE_TEMP}/cfssl" + pushd "${KUBE_TEMP}/cfssl" + + kernel=$(uname -s) + case "${kernel}" in + Linux) + curl -s -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 + curl -s -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 + ;; + Darwin) + curl -s -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64 + curl -s -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64 + ;; + *) + echo "Unknown, unsupported platform: ${kernel}." >&2 + echo "Supported platforms: Linux, Darwin." >&2 + exit 2 + esac + + chmod +x cfssl + chmod +x cfssljson + + cat >ca-config.json < ca-key.pem + echo "${ca_cert}" | base64 --decode | gunzip > ca.pem + else + ./cfssl print-defaults csr > ca-csr.json + ./cfssl gencert -initca ca-csr.json | ./cfssljson -bare ca - + fi + + echo '{"CN":"'"${MASTER_NAME}"'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \ + | ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client-server -hostname="${MASTER_NAME}" - \ + | ./cfssljson -bare etcd + + ETCD_CA_KEY_BASE64=$(cat "ca-key.pem" | base64 | tr -d '\r\n') + ETCD_CA_CERT_BASE64=$(cat "ca.pem" | gzip | base64 | tr -d '\r\n') + ETCD_PEER_KEY_BASE64=$(cat "etcd-key.pem" | base64 | tr -d '\r\n') + ETCD_PEER_CERT_BASE64=$(cat "etcd.pem" | gzip | base64 | tr -d '\r\n') + popd +} + function create-master() { echo "Starting master and configuring firewalls" gcloud compute firewall-rules create "${MASTER_NAME}-https" \ @@ -798,6 +878,7 @@ function create-master() { MASTER_ADVERTISE_ADDRESS="${MASTER_RESERVED_IP}" create-certs "${MASTER_RESERVED_IP}" + create-etcd-certs # Sets MASTER_ROOT_DISK_SIZE that is used by create-master-instance get-master-root-disk-size @@ -823,7 +904,7 @@ function add-replica-to-etcd() { --project "${PROJECT}" \ --zone "${EXISTING_MASTER_ZONE}" \ --command \ - "curl localhost:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"http://${REPLICA_NAME}:${internal_port}\"]}'" + "curl localhost:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"https://${REPLICA_NAME}:${internal_port}\"]}'" return $? } diff --git a/cluster/saltbase/salt/etcd/etcd.manifest b/cluster/saltbase/salt/etcd/etcd.manifest index 5b6bf1d3426d9..ced139e64c0b7 100644 --- a/cluster/saltbase/salt/etcd/etcd.manifest +++ b/cluster/saltbase/salt/etcd/etcd.manifest @@ -1,3 +1,9 @@ +{% set etcd_protocol = 'http' -%} +{% set etcd_creds = '' -%} +{% if pillar.get('etcd_over_ssl', '').lower() == 'true' -%} + {% set etcd_protocol = 'https' -%} + {% set etcd_creds = '--peer-trusted-ca-file /srv/kubernetes/etcd-ca.crt --peer-cert-file /srv/kubernetes/etcd-peer.crt --peer-key-file /srv/kubernetes/etcd-peer.key -peer-client-cert-auth' -%} +{% endif -%} {% set cluster_state = 'new' -%} {% set hostname = pillar.get('hostname', '') -%} {% set etcd_cluster_array = (pillar.get('initial_etcd_cluster') or hostname).split(',') -%} @@ -9,7 +15,7 @@ {% set cluster_state = 'existing' -%} {% set etcd_cluster = etcd_cluster ~ ',' -%} {% endif -%} - {% set etcd_cluster = etcd_cluster ~ 'etcd-' ~ host ~ '=http://' ~ host ~ ':' ~ server_port -%} + {% set etcd_cluster = etcd_cluster ~ 'etcd-' ~ host ~ '=' ~ etcd_protocol ~'://' ~ host ~ ':' ~ server_port -%} {% do vars.update({'etcd_cluster': etcd_cluster, 'cluster_state': cluster_state}) -%} {% endfor -%} {% set etcd_cluster = vars.etcd_cluster -%} @@ -19,6 +25,7 @@ {% if pillar.get('storage_backend', 'etcd2') == 'etcd3' -%} {% set quota_bytes = '--quota-backend-bytes=4294967296' -%} {% endif -%} +{% set srv_kube_path = "/srv/kubernetes" -%} { "apiVersion": "v1", @@ -41,7 +48,7 @@ "command": [ "/bin/sh", "-c", - "if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls http://{{ hostname }}:{{ server_port }} --initial-advertise-peer-urls http://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" + "if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" ], "env": [ { "name": "TARGET_STORAGE", @@ -81,8 +88,12 @@ { "name": "varlogetcd", "mountPath": "/var/log/etcd{{ suffix }}.log", "readOnly": false + }, + { "name": "etc", + "mountPath": "{{ srv_kube_path }}", + "readOnly": false } - ] + ] } ], "volumes":[ @@ -93,6 +104,10 @@ { "name": "varlogetcd", "hostPath": { "path": "/var/log/etcd{{ suffix }}.log"} + }, + { "name": "etc", + "hostPath": { + "path": "{{ srv_kube_path }}"} } ] }}