Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SSL certificates for etcd cluster. #35516

Merged
merged 1 commit into from Nov 10, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions cluster/common.sh
Expand Up @@ -702,6 +702,10 @@ NUM_NODES: $(yaml-quote ${NUM_NODES})
STORAGE_BACKEND: $(yaml-quote ${STORAGE_BACKEND:-etcd2})
ENABLE_GARBAGE_COLLECTOR: $(yaml-quote ${ENABLE_GARBAGE_COLLECTOR:-})
MASTER_ADVERTISE_ADDRESS: $(yaml-quote ${MASTER_ADVERTISE_ADDRESS:-})
ETCD_CA_KEY: $(yaml-quote ${ETCD_CA_KEY_BASE64:-})
ETCD_CA_CERT: $(yaml-quote ${ETCD_CA_CERT_BASE64:-})
ETCD_PEER_KEY: $(yaml-quote ${ETCD_PEER_KEY_BASE64:-})
ETCD_PEER_CERT: $(yaml-quote ${ETCD_PEER_CERT_BASE64:-})
EOF
# ETCD_IMAGE (if set) allows to use a custom etcd image.
if [ -n "${ETCD_IMAGE:-}" ]; then
Expand Down
20 changes: 20 additions & 0 deletions cluster/gce/configure-vm.sh
Expand Up @@ -457,6 +457,7 @@ num_nodes: $(echo "${NUM_NODES:-}" | sed -e "s/'/''/g")
e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")'
kube_uid: '$(echo "${KUBE_UID}" | sed -e "s/'/''/g")'
initial_etcd_cluster: '$(echo "${INITIAL_ETCD_CLUSTER:-}" | sed -e "s/'/''/g")'

hostname: $(hostname -s)
EOF
if [ -n "${STORAGE_BACKEND:-}" ]; then
Expand All @@ -482,6 +483,15 @@ EOF
if [ -n "${ETCD_VERSION:-}" ]; then
cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
etcd_version: '$(echo "$ETCD_VERSION" | sed -e "s/'/''/g")'
EOF
fi
if [[ -n "${ETCD_CA_KEY:-}" && -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
etcd_over_ssl: 'true'
EOF
else
cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
etcd_over_ssl: 'false'
EOF
fi
# Configuration changes for test clusters
Expand Down Expand Up @@ -1054,6 +1064,15 @@ function run-user-script() {
fi
}

function create-salt-master-etcd-auth {
if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
local -r auth_dir="/srv/kubernetes"
echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt"
echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key"
echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt"
fi
}

# This script is re-used on AWS. Some of the above functions will be replaced.
# The AWS kube-up script looks for this marker:
#+AWS_OVERRIDES_HERE
Expand All @@ -1074,6 +1093,7 @@ if [[ -z "${is_push}" ]]; then
create-salt-pillar
if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
create-salt-master-auth
create-salt-master-etcd-auth
create-salt-master-kubelet-auth
else
create-salt-kubelet-auth
Expand Down
11 changes: 11 additions & 0 deletions cluster/gce/debian/master-helper.sh
Expand Up @@ -52,6 +52,17 @@ function replicate-master-instance() {
# Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering.
kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER")"
kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER: '${existing_master_replicas},${REPLICA_NAME}'")"

ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//")"
ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//")"

create-etcd-certs "${ETCD_CA_CERT}" "${ETCD_CA_KEY}"

kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")"
kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")"
kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")"
kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")"

echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml
get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > ${KUBE_TEMP}/cluster-name.txt
get-metadata "${existing_master_zone}" "${existing_master_name}" startup-script > ${KUBE_TEMP}/configure-vm.sh
Expand Down
24 changes: 23 additions & 1 deletion cluster/gce/gci/configure-helper.sh
Expand Up @@ -375,6 +375,15 @@ current-context: service-account-context
EOF
}

function create-master-etcd-auth {
if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
local -r auth_dir="/etc/srv/kubernetes"
echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt"
echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key"
echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt"
fi
}

function assemble-docker-flags {
echo "Assemble docker command line flags"
local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false"
Expand Down Expand Up @@ -606,14 +615,23 @@ function prepare-etcd-manifest {
local host_name=$(hostname)
local etcd_cluster=""
local cluster_state="new"
local etcd_protocol="http"
local etcd_creds=""

if [[ -n "${ETCD_CA_KEY:-}" && -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth "
etcd_protocol="https"
fi

for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do
etcd_host="etcd-${host}=http://${host}:$3"
etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3"
if [[ -n "${etcd_cluster}" ]]; then
etcd_cluster+=","
cluster_state="existing"
fi
etcd_cluster+="${etcd_host}"
done

local -r temp_file="/tmp/$5"
cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd.manifest" "${temp_file}"
remove-salt-config-comments "${temp_file}"
Expand All @@ -622,6 +640,7 @@ function prepare-etcd-manifest {
sed -i -e "s@{{ *server_port *}}@$3@g" "${temp_file}"
sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${temp_file}"
sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}"
sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${temp_file}"
sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}"
sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}"
if [[ "${STORAGE_BACKEND:-}" == "etcd3" ]]; then
Expand All @@ -635,6 +654,8 @@ function prepare-etcd-manifest {
else
sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${temp_file}"
fi
sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}"
sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}"
if [[ -n "${ETCD_VERSION:-}" ]]; then
sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${temp_file}"
else
Expand Down Expand Up @@ -1209,6 +1230,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
mount-master-pd
create-master-auth
create-master-kubelet-auth
create-master-etcd-auth
else
create-kubelet-kubeconfig
create-kubeproxy-kubeconfig
Expand Down
11 changes: 11 additions & 0 deletions cluster/gce/gci/master-helper.sh
Expand Up @@ -49,6 +49,16 @@ function replicate-master-instance() {
# Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering.
kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER")"
kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER: '${existing_master_replicas},${REPLICA_NAME}'")"
ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//")"
ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//")"

create-etcd-certs "${ETCD_CA_CERT}" "${ETCD_CA_KEY}"

kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")"
kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")"
kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")"
kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")"

echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml
get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt"
get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt"
Expand All @@ -58,6 +68,7 @@ function replicate-master-instance() {
create-master-instance-internal "${REPLICA_NAME}"
}


function create-master-instance-internal() {
local -r master_name="${1}"
local -r address_option="${2:-}"
Expand Down
23 changes: 22 additions & 1 deletion cluster/gce/trusty/configure-helper.sh
Expand Up @@ -428,6 +428,15 @@ create_master_kubelet_auth() {
fi
}

function create-master-etcd-auth {
if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
local -r auth_dir="/etc/srv/kubernetes"
echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt"
echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key"
echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt"
fi
}

# Replaces the variables in the etcd manifest file with the real values, and then
# copy the file to the manifest dir
# $1: value for variable 'suffix'
Expand All @@ -439,21 +448,31 @@ prepare_etcd_manifest() {
local host_name=$(hostname)
local etcd_cluster=""
local cluster_state="new"
local etcd_protocol="http"
local etcd_creds=""

if [[ -n "${ETCD_CA_KEY:-}" && -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth "
etcd_protocol="https"
fi

for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do
etcd_host="etcd-${host}=http://${host}:$3"
etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3"
if [[ -n "${etcd_cluster}" ]]; then
etcd_cluster+=","
cluster_state="existing"
fi
etcd_cluster+="${etcd_host}"
done

etcd_temp_file="/tmp/$5"
cp /home/kubernetes/kube-manifests/kubernetes/gci-trusty/etcd.manifest "${etcd_temp_file}"
remove_salt_config_comments "${etcd_temp_file}"
sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}"
sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}"
sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}"
sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${etcd_temp_file}"
sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${etcd_temp_file}"
sed -i -e "s@{{ *hostname *}}@$host_name@g" "${etcd_temp_file}"
sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${etcd_temp_file}"
sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}"
Expand All @@ -468,6 +487,8 @@ prepare_etcd_manifest() {
else
sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${etcd_temp_file}"
fi
sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${etcd_temp_file}"
sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${etcd_temp_file}"
if [[ -n "${ETCD_VERSION:-}" ]]; then
sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${etcd_temp_file}"
else
Expand Down
2 changes: 2 additions & 0 deletions cluster/gce/trusty/master.yaml
Expand Up @@ -65,6 +65,8 @@ script
create_master_auth
echo "Creating master instance kubelet auth file"
create_master_kubelet_auth
echo "Creating auth files for etcd"
create-master-etcd-auth
echo "Assemble kubelet command line"
# Kubelet command flags will be written in /etc/default/kubelet
assemble_kubelet_flags
Expand Down
83 changes: 82 additions & 1 deletion cluster/gce/util.sh
Expand Up @@ -741,6 +741,86 @@ function get-master-disk-size() {
fi
}

# Generates SSL certificates for etcd cluster. Uses cfssl program.
#
# Assumed vars:
# KUBE_TEMP: temporary directory
#
# Args:
# $1: CA certificate
# $2: CA key
#
# If CA cert/key is empty, the function will also generate certs for CA.
#
# Vars set:
# ETCD_CA_KEY_BASE64
# ETCD_CA_CERT_BASE64
# ETCD_PEER_KEY_BASE64
# ETCD_PEER_CERT_BASE64
#
function create-etcd-certs {
local ca_cert=${1:-}
local ca_key=${2:-}

mkdir -p "${KUBE_TEMP}/cfssl"
pushd "${KUBE_TEMP}/cfssl"

kernel=$(uname -s)
case "${kernel}" in
Linux)
curl -s -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
curl -s -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
;;
Darwin)
curl -s -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64
curl -s -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64
;;
*)
echo "Unknown, unsupported platform: ${kernel}." >&2
echo "Supported platforms: Linux, Darwin." >&2
exit 2
esac

chmod +x cfssl
chmod +x cfssljson

cat >ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "168h"
},
"profiles": {
"client-server": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment"
]
}
}
}
}
EOF
if [[ ! -z "${ca_key}" && ! -z "${ca_cert}" ]]; then
echo "${ca_key}" | base64 --decode > ca-key.pem
echo "${ca_cert}" | base64 --decode | gunzip > ca.pem
else
./cfssl print-defaults csr > ca-csr.json
./cfssl gencert -initca ca-csr.json | ./cfssljson -bare ca -
fi

echo '{"CN":"'"${MASTER_NAME}"'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \
| ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client-server -hostname="${MASTER_NAME}" - \
| ./cfssljson -bare etcd

ETCD_CA_KEY_BASE64=$(cat "ca-key.pem" | base64 | tr -d '\r\n')
ETCD_CA_CERT_BASE64=$(cat "ca.pem" | gzip | base64 | tr -d '\r\n')
ETCD_PEER_KEY_BASE64=$(cat "etcd-key.pem" | base64 | tr -d '\r\n')
ETCD_PEER_CERT_BASE64=$(cat "etcd.pem" | gzip | base64 | tr -d '\r\n')
popd
}

function create-master() {
echo "Starting master and configuring firewalls"
gcloud compute firewall-rules create "${MASTER_NAME}-https" \
Expand Down Expand Up @@ -798,6 +878,7 @@ function create-master() {
MASTER_ADVERTISE_ADDRESS="${MASTER_RESERVED_IP}"

create-certs "${MASTER_RESERVED_IP}"
create-etcd-certs

# Sets MASTER_ROOT_DISK_SIZE that is used by create-master-instance
get-master-root-disk-size
Expand All @@ -823,7 +904,7 @@ function add-replica-to-etcd() {
--project "${PROJECT}" \
--zone "${EXISTING_MASTER_ZONE}" \
--command \
"curl localhost:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"http://${REPLICA_NAME}:${internal_port}\"]}'"
"curl localhost:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"https://${REPLICA_NAME}:${internal_port}\"]}'"
return $?
}

Expand Down
21 changes: 18 additions & 3 deletions cluster/saltbase/salt/etcd/etcd.manifest
@@ -1,3 +1,9 @@
{% set etcd_protocol = 'http' -%}
{% set etcd_creds = '' -%}
{% if pillar.get('etcd_over_ssl', '').lower() == 'true' -%}
{% set etcd_protocol = 'https' -%}
{% set etcd_creds = '--peer-trusted-ca-file /srv/kubernetes/etcd-ca.crt --peer-cert-file /srv/kubernetes/etcd-peer.crt --peer-key-file /srv/kubernetes/etcd-peer.key -peer-client-cert-auth' -%}
{% endif -%}
{% set cluster_state = 'new' -%}
{% set hostname = pillar.get('hostname', '') -%}
{% set etcd_cluster_array = (pillar.get('initial_etcd_cluster') or hostname).split(',') -%}
Expand All @@ -9,7 +15,7 @@
{% set cluster_state = 'existing' -%}
{% set etcd_cluster = etcd_cluster ~ ',' -%}
{% endif -%}
{% set etcd_cluster = etcd_cluster ~ 'etcd-' ~ host ~ '=http://' ~ host ~ ':' ~ server_port -%}
{% set etcd_cluster = etcd_cluster ~ 'etcd-' ~ host ~ '=' ~ etcd_protocol ~'://' ~ host ~ ':' ~ server_port -%}
{% do vars.update({'etcd_cluster': etcd_cluster, 'cluster_state': cluster_state}) -%}
{% endfor -%}
{% set etcd_cluster = vars.etcd_cluster -%}
Expand All @@ -19,6 +25,7 @@
{% if pillar.get('storage_backend', 'etcd2') == 'etcd3' -%}
{% set quota_bytes = '--quota-backend-bytes=4294967296' -%}
{% endif -%}
{% set srv_kube_path = "/srv/kubernetes" -%}

{
"apiVersion": "v1",
Expand All @@ -41,7 +48,7 @@
"command": [
"/bin/sh",
"-c",
"if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls http://{{ hostname }}:{{ server_port }} --initial-advertise-peer-urls http://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
"if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
],
"env": [
{ "name": "TARGET_STORAGE",
Expand Down Expand Up @@ -81,8 +88,12 @@
{ "name": "varlogetcd",
"mountPath": "/var/log/etcd{{ suffix }}.log",
"readOnly": false
},
{ "name": "etc",
"mountPath": "{{ srv_kube_path }}",
"readOnly": false
}
]
]
}
],
"volumes":[
Expand All @@ -93,6 +104,10 @@
{ "name": "varlogetcd",
"hostPath": {
"path": "/var/log/etcd{{ suffix }}.log"}
},
{ "name": "etc",
"hostPath": {
"path": "{{ srv_kube_path }}"}
}
]
}}