diff --git a/build/common.sh b/build/common.sh index c85345c378178..38096f5b26b3e 100755 --- a/build/common.sh +++ b/build/common.sh @@ -946,6 +946,7 @@ function kube::release::package_kube_manifests_tarball() { cp "${salt_dir}/kube-addons/namespace.yaml" "${release_stage}/trusty" cp "${salt_dir}/kube-addons/kube-addons.sh" "${release_stage}/trusty" cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${release_stage}/trusty" + cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${release_stage}/trusty" cp -r "${salt_dir}/kube-admission-controls/limit-range" "${release_stage}/trusty" local objects objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo) diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh new file mode 100644 index 0000000000000..75fb2a31ccf57 --- /dev/null +++ b/cluster/gce/trusty/configure-helper.sh @@ -0,0 +1,671 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script contains functions for configuring instances to run kubernetes +# master and nodes. It is uploaded in kube-manifests tarball, and used by +# upstart jobs in cluster/gce/trusty/.yaml + + +config_hostname() { + # Set the hostname to the short version. + host_name=$(curl --silent "http://metadata.google.internal/computeMetadata/v1/instance/hostname" -H "Metadata-Flavor: Google") + short_hostname=$(echo "${host_name}" | cut -d. -f1) + hostname "${short_hostname}" +} + +config_ip_firewall() { + # We have seen that GCE image may have strict host firewall rules which drop + # most inbound/forwarded packets. In such a case, add rules to accept all + # TCP/UDP packets. + if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then + echo "Add rules to accpet all inbound TCP/UDP packets" + iptables -A INPUT -w -p TCP -j ACCEPT + iptables -A INPUT -w -p UDP -j ACCEPT + fi + if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then + echo "Add rules to accpet all forwarded TCP/UDP packets" + iptables -A FORWARD -w -p TCP -j ACCEPT + iptables -A FORWARD -w -p UDP -j ACCEPT + fi +} + +create_dirs() { + # Create required directories. + mkdir -p /var/lib/kubelet + mkdir -p /etc/kubernetes/manifests + if [ "${KUBERNETES_MASTER:-}" = "false" ]; then + mkdir -p /var/lib/kube-proxy + fi +} + +create_kubelet_kubeconfig() { + # Create the kubelet kubeconfig file. + if [ -z "${KUBELET_CA_CERT:-}" ]; then + KUBELET_CA_CERT="${CA_CERT}" + fi + cat > /var/lib/kubelet/kubeconfig << EOF +apiVersion: v1 +kind: Config +users: +- name: kubelet + user: + client-certificate-data: "${KUBELET_CERT}" + client-key-data: "${KUBELET_KEY}" +clusters: +- name: local + cluster: + certificate-authority-data: "${KUBELET_CA_CERT}" +contexts: +- context: + cluster: local + user: kubelet + name: service-account-context +current-context: service-account-context +EOF +} + +create_kubeproxy_kubeconfig() { + # Create the kube-proxy config file. + cat > /var/lib/kube-proxy/kubeconfig << EOF +apiVersion: v1 +kind: Config +users: +- name: kube-proxy + user: + token: "${KUBE_PROXY_TOKEN}" +clusters: +- name: local + cluster: + certificate-authority-data: "${CA_CERT}" +contexts: +- context: + cluster: local + user: kube-proxy + name: service-account-context +current-context: service-account-context +EOF +} + +# Installs the critical packages that are required by spinning up a cluster. +install_critical_packages() { + apt-get update + # Install docker and brctl if they are not in the image. + if ! which docker > /dev/null; then + echo "Do not find docker. Install it." + curl -fsSL https://get.docker.com/ | sh + fi + if ! which brctl > /dev/null; then + echo "Do not find brctl. Install it." + apt-get install --yes bridge-utils + fi +} + +# Install the packages that are useful but not required by spinning up a cluster. +install_additional_packages() { + if ! which socat > /dev/null; then + echo "Do not find socat. Install it." + apt-get install --yes socat + fi + if ! which nsenter > /dev/null; then + echo "Do not find nsenter. Install it." + mkdir -p /tmp/nsenter-install + cd /tmp/nsenter-install + curl https://www.kernel.org/pub/linux/utils/util-linux/v2.24/util-linux-2.24.tar.gz | tar -zxf- + apt-get --yes install make + apt-get --yes install gcc + cd util-linux-2.24 + ./configure --without-ncurses + make nsenter + cp nsenter /usr/local/bin + rm -rf /tmp/nsenter-install + fi +} + +# Assembles kubelet command line flags. +# It should be called by master and nodes before running kubelet process. The caller +# needs to source the config file /etc/kube-env. This function sets the following +# variable that will be used in kubelet command line. +# KUBELET_CMD_FLAGS +assemble_kubelet_flags() { + KUBELET_CMD_FLAGS="--v=2" + if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_TEST_LOG_LEVEL}" + fi + if [ -n "${KUBELET_PORT:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --port=${KUBELET_PORT}" + fi + if [ -n "${KUBELET_TEST_ARGS:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} ${KUBELET_TEST_ARGS}" + fi + if [ "${KUBERNETES_MASTER:-}" = "true" ] && \ + [ ! -z "${KUBELET_APISERVER:-}" ] && \ + [ ! -z "${KUBELET_CERT:-}" ] && \ + [ ! -z "${KUBELET_KEY:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER}" + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --register-schedulable=false --reconcile-cidr=false" + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=10.123.45.0/30" + fi + if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}" + fi + if [ "${KUBERNETES_MASTER:-}" = "true" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=none" + elif [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \ + [ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \ + [ "${HAIRPIN_MODE:-}" = "none" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}" + fi + if [ -n "${ENABLE_CUSTOM_METRICS:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" + fi + if [ -n "${NODE_LABELS:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --node-labels=${NODE_LABELS}" + fi +} + +restart_docker_daemon() { + readonly DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false" + echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker + # Make sure the network interface cbr0 is created before restarting docker daemon + while ! [ -L /sys/class/net/cbr0 ]; do + echo "Sleep 1 second to wait for cbr0" + sleep 1 + done + # Remove docker0 + ifconfig docker0 down + brctl delbr docker0 + # Ensure docker daemon is really functional before exiting. Operations afterwards may + # assume it is running. + while ! docker version > /dev/null; do + echo "Sleep 1 second to wait for docker daemon" + sleep 1 + done +} + +# Create the log file and set its properties. +# +# $1 is the file to create +prepare_log_file() { + touch $1 + chmod 644 $1 + chown root:root $1 +} + +# It monitors the health of several master and node components. +health_monitoring() { + sleep_seconds=10 + max_seconds=10 + # We simply kill the process when there is a failure. Another upstart job will automatically + # restart the process. + while [ 1 ]; do + if ! timeout 10 docker version > /dev/null; then + echo "Docker daemon failed!" + pkill docker + # Wait for a while, as we don't want to kill it again before it is really up. + sleep 30 + fi + if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then + echo "Kubelet is unhealthy!" + pkill kubelet + # Wait for a while, as we don't want to kill it again before it is really up. + sleep 60 + fi + sleep "${sleep_seconds}" + done +} + + +########## The functions below are for master only ########## + +# Mounts a persistent disk (formatting if needed) to store the persistent data +# on the master -- etcd's data, a few settings, and security certs/keys/tokens. +# safe_format_and_mount only formats an unformatted disk, and mkdir -p will +# leave a directory be if it already exists. +mount_master_pd() { + readonly pd_path="/dev/disk/by-id/google-master-pd" + readonly mount_point="/mnt/disks/master-pd" + + # TODO(zmerlynn): GKE is still lagging in master-pd creation + if [ ! -e "${pd_path}" ]; then + return + fi + # Format and mount the disk, create directories on it for all of the master's + # persistent data, and link them to where they're used. + mkdir -p "${mount_point}" + /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${pd_path}" "${mount_point}" >/var/log/master-pd-mount.log || \ + { echo "!!! master-pd mount failed, review /var/log/master-pd-mount.log !!!"; return 1; } + # Contains all the data stored in etcd + mkdir -m 700 -p "${mount_point}/var/etcd" + # Contains the dynamically generated apiserver auth certs and keys + mkdir -p "${mount_point}/etc/srv/kubernetes" + # Directory for kube-apiserver to store SSH key (if necessary) + mkdir -p "${mount_point}/etc/srv/sshproxy" + ln -s -f "${mount_point}/var/etcd" /var/etcd + mkdir -p /etc/srv + ln -s -f "${mount_point}/etc/srv/kubernetes" /etc/srv/kubernetes + ln -s -f "${mount_point}/etc/srv/sshproxy" /etc/srv/sshproxy + + if ! id etcd &>/dev/null; then + useradd -s /sbin/nologin -d /var/etcd etcd + fi + chown -R etcd "${mount_point}/var/etcd" + chgrp -R etcd "${mount_point}/var/etcd" +} + +# A helper function that adds an entry to a token file. +# $1: account information +# $2: token file +add_token_entry() { + current_token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + echo "${current_token},$1,$1" >> $2 +} + +# After the first boot and on upgrade, these files exists on the master-pd +# and should never be touched again (except perhaps an additional service +# account, see NB below.) +create_master_auth() { + readonly auth_dir="/etc/srv/kubernetes" + if [ ! -e "${auth_dir}/ca.crt" ]; then + if [ ! -z "${CA_CERT:-}" ] && [ ! -z "${MASTER_CERT:-}" ] && [ ! -z "${MASTER_KEY:-}" ]; then + echo "${CA_CERT}" | base64 -d > "${auth_dir}/ca.crt" + echo "${MASTER_CERT}" | base64 -d > "${auth_dir}/server.cert" + echo "${MASTER_KEY}" | base64 -d > "${auth_dir}/server.key" + # Kubecfg cert/key are optional and included for backwards compatibility. + # TODO(roberthbailey): Remove these two lines once GKE no longer requires + # fetching clients certs from the master VM. + echo "${KUBECFG_CERT:-}" | base64 -d > "${auth_dir}/kubecfg.crt" + echo "${KUBECFG_KEY:-}" | base64 -d > "${auth_dir}/kubecfg.key" + fi + fi + readonly basic_auth_csv="${auth_dir}/basic_auth.csv" + if [ ! -e "${basic_auth_csv}" ]; then + echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${basic_auth_csv}" + fi + readonly known_tokens_csv="${auth_dir}/known_tokens.csv" + if [ ! -e "${known_tokens_csv}" ]; then + echo "${KUBE_BEARER_TOKEN},admin,admin" > "${known_tokens_csv}" + echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${known_tokens_csv}" + echo "${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy" >> "${known_tokens_csv}" + + # Generate tokens for other "service accounts". Append to known_tokens. + # + # NB: If this list ever changes, this script actually has to + # change to detect the existence of this file, kill any deleted + # old tokens and add any new tokens (to handle the upgrade case). + add_token_entry "system:scheduler" "${known_tokens_csv}" + add_token_entry "system:controller_manager" "${known_tokens_csv}" + add_token_entry "system:logging" "${known_tokens_csv}" + add_token_entry "system:monitoring" "${known_tokens_csv}" + add_token_entry "system:dns" "${known_tokens_csv}" + fi + + if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then + cat </etc/gce.conf +[global] +token-url = "${TOKEN_URL}" +token-body = "${TOKEN_BODY}" +project-id = "${PROJECT_ID}" +network-name = "${NODE_NETWORK}" +EOF + fi +} + +# Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY +# to generate a kubeconfig file for the kubelet to securely connect to the apiserver. +create_master_kubelet_auth() { + # Only configure the kubelet on the master if the required variables are + # set in the environment. + if [ -n "${KUBELET_APISERVER:-}" ] && [ -n "${KUBELET_CERT:-}" ] && [ -n "${KUBELET_KEY:-}" ]; then + create_kubelet_kubeconfig + fi +} + +# Replaces the variables in the etcd manifest file with the real values, and then +# copy the file to the manifest dir +# $1: value for variable 'suffix' +# $2: value for variable 'port' +# $3: value for variable 'server_port' +# $4: value for variable 'cpulimit' +# $5: pod name, which should be either etcd or etcd-events +prepare_etcd_manifest() { + etcd_temp_file="/tmp/$5" + cp /run/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}" + sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}" + sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}" + sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}" + sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${etcd_temp_file}" + # Replace the volume host path + sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${etcd_temp_file}" + mv "${etcd_temp_file}" /etc/kubernetes/manifests +} + +# Starts etcd server pod (and etcd-events pod if needed). +# More specifically, it prepares dirs and files, sets the variable value +# in the manifests, and copies them to /etc/kubernetes/manifests. +start_etcd_servers() { + if [ -d /etc/etcd ]; then + rm -rf /etc/etcd + fi + if [ -e /etc/default/etcd ]; then + rm -f /etc/default/etcd + fi + if [ -e /etc/systemd/system/etcd.service ]; then + rm -f /etc/systemd/system/etcd.service + fi + if [ -e /etc/init.d/etcd ]; then + rm -f /etc/init.d/etcd + fi + prepare_log_file /var/log/etcd.log + prepare_etcd_manifest "" "4001" "2380" "200m" "etcd.manifest" + + prepare_log_file /var/log/etcd-events.log + prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" +} + +# Calculates the following variables based on env variables, which will be used +# by the manifests of several kube-master components. +# CLOUD_CONFIG_VOLUME +# CLOUD_CONFIG_MOUNT +# DOCKER_REGISTRY +compute_master_manifest_variables() { + CLOUD_CONFIG_VOLUME="" + CLOUD_CONFIG_MOUNT="" + if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then + CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\"}}," + CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," + fi + DOCKER_REGISTRY="gcr.io/google_containers" + if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then + DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}" + fi +} + +# A helper function for removing salt configuration and comments from a file. +# This is mainly for preparing a manifest file. +# $1: Full path of the file to manipulate +remove_salt_config_comments() { + # Remove salt configuration + sed -i "/^[ |\t]*{[#|%]/d" $1 + # Remove comments + sed -i "/^[ |\t]*#/d" $1 +} + +# Starts k8s apiserver. +# It prepares the log file, loads the docker image, calculates variables, sets them +# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. +# +# Assumed vars (which are calculated in function compute_master_manifest_variables) +# CLOUD_CONFIG_VOLUME +# CLOUD_CONFIG_MOUNT +# DOCKER_REGISTRY +start_kube_apiserver() { + prepare_log_file /var/log/kube-apiserver.log + # Load the docker image from file. + echo "Try to load docker image file kube-apiserver.tar" + timeout 30 docker load -i /run/kube-docker-files/kube-apiserver.tar + + # Calculate variables and assemble the command line. + params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true" + params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002" + if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then + params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" + fi + if [ -n "${ADMISSION_CONTROL:-}" ]; then + params="${params} --admission-control=${ADMISSION_CONTROL}" + fi + if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then + params="${params} --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" + fi + if [ -n "${RUNTIME_CONFIG:-}" ]; then + params="${params} --runtime-config=${RUNTIME_CONFIG}" + fi + if [ -n "${APISERVER_TEST_ARGS:-}" ]; then + params="${params} ${APISERVER_TEST_ARGS}" + fi + log_level="--v=2" + if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then + log_level="${API_SERVER_TEST_LOG_LEVEL}" + fi + params="${params} ${log_level}" + + if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then + readonly vm_external_ip=$(curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip") + params="${params} --cloud-config=/etc/gce.conf --advertise-address=${vm_external_ip} --ssh-user=${PROXY_SSH_USER} --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" + fi + readonly kube_apiserver_docker_tag=$(cat /run/kube-docker-files/kube-apiserver.docker_tag) + + src_file="/run/kube-manifests/kubernetes/trusty/kube-apiserver.manifest" + remove_salt_config_comments "${src_file}" + # Evaluate variables + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" + sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}" + sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" + sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}" + sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" + sed -i -e "s@{{secure_port}}@443@g" "${src_file}" + sed -i -e "s@{{secure_port}}@8080@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests +} + +# Starts k8s controller manager. +# It prepares the log file, loads the docker image, calculates variables, sets them +# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. +# +# Assumed vars (which are calculated in function compute_master_manifest_variables) +# CLOUD_CONFIG_VOLUME +# CLOUD_CONFIG_MOUNT +# DOCKER_REGISTRY +start_kube_controller_manager() { + prepare_log_file /var/log/kube-controller-manager.log + # Load the docker image from file. + echo "Try to load docker image file kube-controller-manager.tar" + timeout 30 docker load -i /run/kube-docker-files/kube-controller-manager.tar + + # Calculate variables and assemble the command line. + params="--master=127.0.0.1:8080 --cloud-provider=gce --root-ca-file=/etc/srv/kubernetes/ca.crt --service-account-private-key-file=/etc/srv/kubernetes/server.key" + if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then + params="${params} --cloud-config=/etc/gce.conf" + fi + if [ -n "${INSTANCE_PREFIX:-}" ]; then + params="${params} --cluster-name=${INSTANCE_PREFIX}" + fi + if [ -n "${CLUSTER_IP_RANGE:-}" ]; then + params="${params} --cluster-cidr=${CLUSTER_IP_RANGE}" + fi + if [ "${ALLOCATE_NODE_CIDRS:-}" = "true" ]; then + params="${params} --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" + fi + if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then + params="${params} --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" + fi + log_level="--v=2" + if [ -n "${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-}" ]; then + log_level="${CONTROLLER_MANAGER_TEST_LOG_LEVEL}" + fi + params="${params} ${log_level}" + if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then + params="${params} ${CONTROLLER_MANAGER_TEST_ARGS}" + fi + readonly kube_rc_docker_tag=$(cat /run/kube-docker-files/kube-controller-manager.docker_tag) + + src_file="/run/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest" + remove_salt_config_comments "${src_file}" + # Evaluate variables + sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}" + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" + sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests +} + +# Starts k8s scheduler. +# It prepares the log file, loads the docker image, calculates variables, sets them +# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. +# +# Assumed vars (which are calculated in compute_master_manifest_variables()) +# DOCKER_REGISTRY +start_kube_scheduler() { + prepare_log_file /var/log/kube-scheduler.log + # Load the docker image from file. + echo "Try to load docker image file kube-scheduler.tar" + timeout 30 docker load -i /run/kube-docker-files/kube-scheduler.tar + + # Calculate variables and set them in the manifest. + params="" + log_level="--v=2" + if [ -n "${SCHEDULER_TEST_LOG_LEVEL:-}" ]; then + log_level="${SCHEDULER_TEST_LOG_LEVEL}" + fi + params="${params} ${log_level}" + if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then + params="${params} ${SCHEDULER_TEST_ARGS}" + fi + readonly kube_scheduler_docker_tag=$(cat /run/kube-docker-files/kube-scheduler.docker_tag) + + # Remove salt comments and replace variables with values + src_file="/run/kube-manifests/kubernetes/trusty/kube-scheduler.manifest" + remove_salt_config_comments "${src_file}" + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests +} + +# Starts a fluentd static pod for logging. +start_fluentd() { + if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then + if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then + cp /run/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/ + elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then + cp /run/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/ + fi + fi +} + +# A helper function for copying addon manifests and set dir/files +# permissions. +# $1: addon category under /etc/kubernetes +# $2: manifest source dir +setup_addon_manifests() { + src_dir="/run/kube-manifests/kubernetes/trusty/$2" + dst_dir="/etc/kubernetes/$1/$2" + if [ ! -d "${dst_dir}" ]; then + mkdir -p "${dst_dir}" + fi + files=$(find "${src_dir}" -name "*.yaml") + if [ -n "${files}" ]; then + cp "${src_dir}/"*.yaml "${dst_dir}" + fi + files=$(find "${src_dir}" -name "*.json") + if [ -n "${files}" ]; then + cp "${src_dir}/"*.json "${dst_dir}" + fi + files=$(find "${src_dir}" -name "*.yaml.in") + if [ -n "${files}" ]; then + cp "${src_dir}/"*.yaml.in "${dst_dir}" + fi + chown -R root:root "${dst_dir}" + chmod 755 "${dst_dir}" + chmod 644 "${dst_dir}"/* +} + +# Prepares the manifests of k8s addons static pods. +prepare_kube_addons() { + addon_src_dir="/run/kube-manifests/kubernetes/trusty" + addon_dst_dir="/etc/kubernetes/addons" + # Set up manifests of other addons. + if [ "${ENABLE_CLUSTER_MONITORING:-}" = "influxdb" ] || \ + [ "${ENABLE_CLUSTER_MONITORING:-}" = "google" ] || \ + [ "${ENABLE_CLUSTER_MONITORING:-}" = "standalone" ] || \ + [ "${ENABLE_CLUSTER_MONITORING:-}" = "googleinfluxdb" ]; then + file_dir="cluster-monitoring/${ENABLE_CLUSTER_MONITORING}" + setup_addon_manifests "addons" "${file_dir}" + # Replace the salt configurations with variable values. + metrics_memory="200Mi" + eventer_memory="200Mi" + if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -ge 1 ]; then + num_kube_nodes="$((${NUM_NODES}-1))" + metrics_memory="$((${num_kube_nodes} * 4 + 200))Mi" + eventer_memory="$((${num_kube_nodes} * 500 + 200 * 1024))Ki" + fi + controller_yaml="${addon_dst_dir}/${file_dir}" + if [ "${ENABLE_CLUSTER_MONITORING:-}" = "googleinfluxdb" ]; then + controller_yaml="${controller_yaml}/heapster-controller-combined.yaml" + else + controller_yaml="${controller_yaml}/heapster-controller.yaml" + fi + remove_salt_config_comments "${controller_yaml}" + sed -i -e "s@{{ *metrics_memory *}}@${metrics_memory}@g" "${controller_yaml}" + sed -i -e "s@{{ *eventer_memory *}}@${eventer_memory}@g" "${controller_yaml}" + fi + cp "${addon_src_dir}/namespace.yaml" "${addon_dst_dir}" + if [ "${ENABLE_L7_LOADBALANCING:-}" = "glbc" ]; then + setup_addon_manifests "addons" "cluster-loadbalancing/glbc" + fi + if [ "${ENABLE_CLUSTER_DNS:-}" = "true" ]; then + setup_addon_manifests "addons" "dns" + dns_rc_file="${addon_dst_dir}/dns/skydns-rc.yaml" + dns_svc_file="${addon_dst_dir}/dns/skydns-svc.yaml" + mv "${addon_dst_dir}/dns/skydns-rc.yaml.in" "${dns_rc_file}" + mv "${addon_dst_dir}/dns/skydns-svc.yaml.in" "${dns_svc_file}" + # Replace the salt configurations with variable values. + sed -i -e "s@{{ *pillar\['dns_replicas'\] *}}@${DNS_REPLICAS}@g" "${dns_rc_file}" + sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${dns_rc_file}" + sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${dns_svc_file}" + fi + if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then + setup_addon_manifests "addons" "registry" + registry_pv_file="${addon_dst_dir}/registry/registry-pv.yaml" + registry_pvc_file="${addon_dst_dir}/registry/registry-pvc.yaml" + mv "${addon_dst_dir}/registry/registry-pv.yaml.in" "${registry_pv_file}" + mv "${addon_dst_dir}/registry/registry-pvc.yaml.in" "${registry_pvc_file}" + # Replace the salt configurations with variable values. + remove_salt_config_comments "${controller_yaml}" + sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pv_file}" + sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pvc_file}" + sed -i -e "s@{{ *pillar\['cluster_registry_disk_name'\] *}}@${CLUSTER_REGISTRY_DISK}@g" "${registry_pvc_file}" + fi + if [ "${ENABLE_NODE_LOGGING:-}" = "true" ] && \ + [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ] && \ + [ "${ENABLE_CLUSTER_LOGGING:-}" = "true" ]; then + setup_addon_manifests "addons" "fluentd-elasticsearch" + fi + if [ "${ENABLE_CLUSTER_UI:-}" = "true" ]; then + setup_addon_manifests "addons" "dashboard" + fi + if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then + setup_addon_manifests "admission-controls" "limit-range" + fi + + # Prepare the scripts for running addons. + addon_script_dir="/var/lib/cloud/scripts/kubernetes" + mkdir -p "${addon_script_dir}" + cp "${addon_src_dir}/kube-addons.sh" "${addon_script_dir}" + cp "${addon_src_dir}/kube-addon-update.sh" "${addon_script_dir}" + chmod 544 "${addon_script_dir}/"*.sh + # In case that some GCE customized trusty may have a read-only /root. + mount -t tmpfs tmpfs /root + mount --bind -o remount,rw,noexec /root +} diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh index e8cf21e69e68f..c934430df8cf9 100644 --- a/cluster/gce/trusty/configure.sh +++ b/cluster/gce/trusty/configure.sh @@ -20,38 +20,6 @@ # of needed functions. The script itself is not supposed to be executed in # other manners. -config_hostname() { - # Set the hostname to the short version. - host_name=$(curl --silent "http://metadata.google.internal/computeMetadata/v1/instance/hostname" -H "Metadata-Flavor: Google") - short_hostname=$(echo "${host_name}" | cut -d. -f1) - hostname "${short_hostname}" -} - -config_ip_firewall() { - # We have seen that GCE image may have strict host firewall rules which drop - # most inbound/forwarded packets. In such a case, add rules to accept all - # TCP/UDP packets. - if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then - echo "Add rules to accpet all inbound TCP/UDP packets" - iptables -A INPUT -w -p TCP -j ACCEPT - iptables -A INPUT -w -p UDP -j ACCEPT - fi - if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then - echo "Add rules to accpet all forwarded TCP/UDP packets" - iptables -A FORWARD -w -p TCP -j ACCEPT - iptables -A FORWARD -w -p UDP -j ACCEPT - fi -} - -create_dirs() { - # Create required directories. - mkdir -p /var/lib/kubelet - mkdir -p /etc/kubernetes/manifests - if [ "${KUBERNETES_MASTER:-}" = "false" ]; then - mkdir -p /var/lib/kube-proxy - fi -} - download_kube_env() { # Fetch kube-env from GCE metadata server. readonly tmp_install_dir="/var/cache/kubernetes-install" @@ -68,89 +36,6 @@ for k,v in yaml.load(sys.stdin).iteritems(): ''' < "${tmp_install_dir}/kube_env.yaml" > /etc/kube-env) } -create_kubelet_kubeconfig() { - # Create the kubelet kubeconfig file. - if [ -z "${KUBELET_CA_CERT:-}" ]; then - KUBELET_CA_CERT="${CA_CERT}" - fi - cat > /var/lib/kubelet/kubeconfig << EOF -apiVersion: v1 -kind: Config -users: -- name: kubelet - user: - client-certificate-data: "${KUBELET_CERT}" - client-key-data: "${KUBELET_KEY}" -clusters: -- name: local - cluster: - certificate-authority-data: "${KUBELET_CA_CERT}" -contexts: -- context: - cluster: local - user: kubelet - name: service-account-context -current-context: service-account-context -EOF -} - -create_kubeproxy_kubeconfig() { - # Create the kube-proxy config file. - cat > /var/lib/kube-proxy/kubeconfig << EOF -apiVersion: v1 -kind: Config -users: -- name: kube-proxy - user: - token: "${KUBE_PROXY_TOKEN}" -clusters: -- name: local - cluster: - certificate-authority-data: "${CA_CERT}" -contexts: -- context: - cluster: local - user: kube-proxy - name: service-account-context -current-context: service-account-context -EOF -} - -# Installs the critical packages that are required by spinning up a cluster. -install_critical_packages() { - apt-get update - # Install docker and brctl if they are not in the image. - if ! which docker > /dev/null; then - echo "Do not find docker. Install it." - curl -fsSL https://get.docker.com/ | sh - fi - if ! which brctl > /dev/null; then - echo "Do not find brctl. Install it." - apt-get install --yes bridge-utils - fi -} - -# Install the packages that are useful but not required by spinning up a cluster. -install_additional_packages() { - if ! which socat > /dev/null; then - echo "Do not find socat. Install it." - apt-get install --yes socat - fi - if ! which nsenter > /dev/null; then - echo "Do not find nsenter. Install it." - mkdir -p /tmp/nsenter-install - cd /tmp/nsenter-install - curl https://www.kernel.org/pub/linux/utils/util-linux/v2.24/util-linux-2.24.tar.gz | tar -zxf- - apt-get --yes install make - apt-get --yes install gcc - cd util-linux-2.24 - ./configure --without-ncurses - make nsenter - cp nsenter /usr/local/bin - rm -rf /tmp/nsenter-install - fi -} - # Retry a download until we get it. # # $1 is the file to create @@ -237,542 +122,7 @@ install_kube_binary_config() { find /run/kube-manifests -name \*.manifest -or -name \*.json | \ xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@" fi + cp /run/kube-manifests/kubernetes/trusty/configure-helper.sh /etc/kube-configure-helper.sh rm "/run/kube-manifests/${manifests_sha1}" rm "/run/kube-manifests/${manifests_tar}" } - -# Assembles kubelet command line flags. -# It should be called by master and nodes before running kubelet process. The caller -# needs to source the config file /etc/kube-env. This function sets the following -# variable that will be used in kubelet command line. -# KUBELET_CMD_FLAGS -assemble_kubelet_flags() { - KUBELET_CMD_FLAGS="--v=2" - if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then - KUBELET_CMD_FLAGS="${KUBELET_TEST_LOG_LEVEL}" - fi - if [ -n "${KUBELET_PORT:-}" ]; then - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --port=${KUBELET_PORT}" - fi - if [ -n "${KUBELET_TEST_ARGS:-}" ]; then - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} ${KUBELET_TEST_ARGS}" - fi - if [ "${KUBERNETES_MASTER:-}" = "true" ] && \ - [ ! -z "${KUBELET_APISERVER:-}" ] && \ - [ ! -z "${KUBELET_CERT:-}" ] && \ - [ ! -z "${KUBELET_KEY:-}" ]; then - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER}" - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --register-schedulable=false --reconcile-cidr=false" - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=10.123.45.0/30" - fi - if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}" - fi - if [ "${KUBERNETES_MASTER:-}" = "true" ]; then - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=none" - elif [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \ - [ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \ - [ "${HAIRPIN_MODE:-}" = "none" ]; then - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}" - fi - if [ -n "${ENABLE_CUSTOM_METRICS:-}" ]; then - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" - fi - if [ -n "${NODE_LABELS:-}" ]; then - KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --node-labels=${NODE_LABELS}" - fi -} - -restart_docker_daemon() { - readonly DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false" - echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker - # Make sure the network interface cbr0 is created before restarting docker daemon - while ! [ -L /sys/class/net/cbr0 ]; do - echo "Sleep 1 second to wait for cbr0" - sleep 1 - done - # Remove docker0 - ifconfig docker0 down - brctl delbr docker0 - # Ensure docker daemon is really functional before exiting. Operations afterwards may - # assume it is running. - while ! docker version > /dev/null; do - echo "Sleep 1 second to wait for docker daemon" - sleep 1 - done -} - -# Create the log file and set its properties. -# -# $1 is the file to create -prepare_log_file() { - touch $1 - chmod 644 $1 - chown root:root $1 -} - -# It monitors the health of several master and node components. -health_monitoring() { - sleep_seconds=10 - max_seconds=10 - # We simply kill the process when there is a failure. Another upstart job will automatically - # restart the process. - while [ 1 ]; do - if ! timeout 10 docker version > /dev/null; then - echo "Docker daemon failed!" - pkill docker - # Wait for a while, as we don't want to kill it again before it is really up. - sleep 30 - fi - if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then - echo "Kubelet is unhealthy!" - pkill kubelet - # Wait for a while, as we don't want to kill it again before it is really up. - sleep 60 - fi - sleep "${sleep_seconds}" - done -} - - -########## The functions below are for master only ########## - -# Mounts a persistent disk (formatting if needed) to store the persistent data -# on the master -- etcd's data, a few settings, and security certs/keys/tokens. -# safe_format_and_mount only formats an unformatted disk, and mkdir -p will -# leave a directory be if it already exists. -mount_master_pd() { - readonly pd_path="/dev/disk/by-id/google-master-pd" - readonly mount_point="/mnt/disks/master-pd" - - # TODO(zmerlynn): GKE is still lagging in master-pd creation - if [ ! -e "${pd_path}" ]; then - return - fi - # Format and mount the disk, create directories on it for all of the master's - # persistent data, and link them to where they're used. - mkdir -p "${mount_point}" - /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${pd_path}" "${mount_point}" >/var/log/master-pd-mount.log || \ - { echo "!!! master-pd mount failed, review /var/log/master-pd-mount.log !!!"; return 1; } - # Contains all the data stored in etcd - mkdir -m 700 -p "${mount_point}/var/etcd" - # Contains the dynamically generated apiserver auth certs and keys - mkdir -p "${mount_point}/etc/srv/kubernetes" - # Directory for kube-apiserver to store SSH key (if necessary) - mkdir -p "${mount_point}/etc/srv/sshproxy" - ln -s -f "${mount_point}/var/etcd" /var/etcd - mkdir -p /etc/srv - ln -s -f "${mount_point}/etc/srv/kubernetes" /etc/srv/kubernetes - ln -s -f "${mount_point}/etc/srv/sshproxy" /etc/srv/sshproxy - - if ! id etcd &>/dev/null; then - useradd -s /sbin/nologin -d /var/etcd etcd - fi - chown -R etcd "${mount_point}/var/etcd" - chgrp -R etcd "${mount_point}/var/etcd" -} - -# A helper function that adds an entry to a token file. -# $1: account information -# $2: token file -add_token_entry() { - current_token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - echo "${current_token},$1,$1" >> $2 -} - -# After the first boot and on upgrade, these files exists on the master-pd -# and should never be touched again (except perhaps an additional service -# account, see NB below.) -create_master_auth() { - readonly auth_dir="/etc/srv/kubernetes" - if [ ! -e "${auth_dir}/ca.crt" ]; then - if [ ! -z "${CA_CERT:-}" ] && [ ! -z "${MASTER_CERT:-}" ] && [ ! -z "${MASTER_KEY:-}" ]; then - echo "${CA_CERT}" | base64 -d > "${auth_dir}/ca.crt" - echo "${MASTER_CERT}" | base64 -d > "${auth_dir}/server.cert" - echo "${MASTER_KEY}" | base64 -d > "${auth_dir}/server.key" - # Kubecfg cert/key are optional and included for backwards compatibility. - # TODO(roberthbailey): Remove these two lines once GKE no longer requires - # fetching clients certs from the master VM. - echo "${KUBECFG_CERT:-}" | base64 -d > "${auth_dir}/kubecfg.crt" - echo "${KUBECFG_KEY:-}" | base64 -d > "${auth_dir}/kubecfg.key" - fi - fi - readonly basic_auth_csv="${auth_dir}/basic_auth.csv" - if [ ! -e "${basic_auth_csv}" ]; then - echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${basic_auth_csv}" - fi - readonly known_tokens_csv="${auth_dir}/known_tokens.csv" - if [ ! -e "${known_tokens_csv}" ]; then - echo "${KUBE_BEARER_TOKEN},admin,admin" > "${known_tokens_csv}" - echo "${KUBELET_TOKEN},kubelet,kubelet" >> "${known_tokens_csv}" - echo "${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy" >> "${known_tokens_csv}" - - # Generate tokens for other "service accounts". Append to known_tokens. - # - # NB: If this list ever changes, this script actually has to - # change to detect the existence of this file, kill any deleted - # old tokens and add any new tokens (to handle the upgrade case). - add_token_entry "system:scheduler" "${known_tokens_csv}" - add_token_entry "system:controller_manager" "${known_tokens_csv}" - add_token_entry "system:logging" "${known_tokens_csv}" - add_token_entry "system:monitoring" "${known_tokens_csv}" - add_token_entry "system:dns" "${known_tokens_csv}" - fi - - if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then - cat </etc/gce.conf -[global] -token-url = "${TOKEN_URL}" -token-body = "${TOKEN_BODY}" -project-id = "${PROJECT_ID}" -network-name = "${NODE_NETWORK}" -EOF - fi -} - -# Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY -# to generate a kubeconfig file for the kubelet to securely connect to the apiserver. -create_master_kubelet_auth() { - # Only configure the kubelet on the master if the required variables are - # set in the environment. - if [ -n "${KUBELET_APISERVER:-}" ] && [ -n "${KUBELET_CERT:-}" ] && [ -n "${KUBELET_KEY:-}" ]; then - create_kubelet_kubeconfig - fi -} - -# Replaces the variables in the etcd manifest file with the real values, and then -# copy the file to the manifest dir -# $1: value for variable 'suffix' -# $2: value for variable 'port' -# $3: value for variable 'server_port' -# $4: value for variable 'cpulimit' -# $5: pod name, which should be either etcd or etcd-events -prepare_etcd_manifest() { - etcd_temp_file="/tmp/$5" - cp /run/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}" - sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}" - sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}" - sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}" - sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${etcd_temp_file}" - # Replace the volume host path - sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${etcd_temp_file}" - mv "${etcd_temp_file}" /etc/kubernetes/manifests -} - -# Starts etcd server pod (and etcd-events pod if needed). -# More specifically, it prepares dirs and files, sets the variable value -# in the manifests, and copies them to /etc/kubernetes/manifests. -start_etcd_servers() { - if [ -d /etc/etcd ]; then - rm -rf /etc/etcd - fi - if [ -e /etc/default/etcd ]; then - rm -f /etc/default/etcd - fi - if [ -e /etc/systemd/system/etcd.service ]; then - rm -f /etc/systemd/system/etcd.service - fi - if [ -e /etc/init.d/etcd ]; then - rm -f /etc/init.d/etcd - fi - prepare_log_file /var/log/etcd.log - prepare_etcd_manifest "" "4001" "2380" "200m" "etcd.manifest" - - prepare_log_file /var/log/etcd-events.log - prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" -} - -# Calculates the following variables based on env variables, which will be used -# by the manifests of several kube-master components. -# CLOUD_CONFIG_VOLUME -# CLOUD_CONFIG_MOUNT -# DOCKER_REGISTRY -compute_master_manifest_variables() { - CLOUD_CONFIG_VOLUME="" - CLOUD_CONFIG_MOUNT="" - if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then - CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\"}}," - CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," - fi - DOCKER_REGISTRY="gcr.io/google_containers" - if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then - DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}" - fi -} - -# A helper function for removing salt configuration and comments from a file. -# This is mainly for preparing a manifest file. -# $1: Full path of the file to manipulate -remove_salt_config_comments() { - # Remove salt configuration - sed -i "/^[ |\t]*{[#|%]/d" $1 - # Remove comments - sed -i "/^[ |\t]*#/d" $1 -} - -# Starts k8s apiserver. -# It prepares the log file, loads the docker image, calculates variables, sets them -# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. -# -# Assumed vars (which are calculated in function compute_master_manifest_variables) -# CLOUD_CONFIG_VOLUME -# CLOUD_CONFIG_MOUNT -# DOCKER_REGISTRY -start_kube_apiserver() { - prepare_log_file /var/log/kube-apiserver.log - # Load the docker image from file. - echo "Try to load docker image file kube-apiserver.tar" - timeout 30 docker load -i /run/kube-docker-files/kube-apiserver.tar - - # Calculate variables and assemble the command line. - params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true" - params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002" - if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then - params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" - fi - if [ -n "${ADMISSION_CONTROL:-}" ]; then - params="${params} --admission-control=${ADMISSION_CONTROL}" - fi - if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then - params="${params} --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" - fi - if [ -n "${RUNTIME_CONFIG:-}" ]; then - params="${params} --runtime-config=${RUNTIME_CONFIG}" - fi - if [ -n "${APISERVER_TEST_ARGS:-}" ]; then - params="${params} ${APISERVER_TEST_ARGS}" - fi - log_level="--v=2" - if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then - log_level="${API_SERVER_TEST_LOG_LEVEL}" - fi - params="${params} ${log_level}" - - if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then - readonly vm_external_ip=$(curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip") - params="${params} --cloud-config=/etc/gce.conf --advertise-address=${vm_external_ip} --ssh-user=${PROXY_SSH_USER} --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" - fi - readonly kube_apiserver_docker_tag=$(cat /run/kube-docker-files/kube-apiserver.docker_tag) - - src_file="/run/kube-manifests/kubernetes/trusty/kube-apiserver.manifest" - remove_salt_config_comments "${src_file}" - # Evaluate variables - sed -i -e "s@{{params}}@${params}@g" "${src_file}" - sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" - sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}" - sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" - sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}" - sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" - sed -i -e "s@{{secure_port}}@443@g" "${src_file}" - sed -i -e "s@{{secure_port}}@8080@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" - cp "${src_file}" /etc/kubernetes/manifests -} - -# Starts k8s controller manager. -# It prepares the log file, loads the docker image, calculates variables, sets them -# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. -# -# Assumed vars (which are calculated in function compute_master_manifest_variables) -# CLOUD_CONFIG_VOLUME -# CLOUD_CONFIG_MOUNT -# DOCKER_REGISTRY -start_kube_controller_manager() { - prepare_log_file /var/log/kube-controller-manager.log - # Load the docker image from file. - echo "Try to load docker image file kube-controller-manager.tar" - timeout 30 docker load -i /run/kube-docker-files/kube-controller-manager.tar - - # Calculate variables and assemble the command line. - params="--master=127.0.0.1:8080 --cloud-provider=gce --root-ca-file=/etc/srv/kubernetes/ca.crt --service-account-private-key-file=/etc/srv/kubernetes/server.key" - if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then - params="${params} --cloud-config=/etc/gce.conf" - fi - if [ -n "${INSTANCE_PREFIX:-}" ]; then - params="${params} --cluster-name=${INSTANCE_PREFIX}" - fi - if [ -n "${CLUSTER_IP_RANGE:-}" ]; then - params="${params} --cluster-cidr=${CLUSTER_IP_RANGE}" - fi - if [ "${ALLOCATE_NODE_CIDRS:-}" = "true" ]; then - params="${params} --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" - fi - if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then - params="${params} --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" - fi - log_level="--v=2" - if [ -n "${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-}" ]; then - log_level="${CONTROLLER_MANAGER_TEST_LOG_LEVEL}" - fi - params="${params} ${log_level}" - if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then - params="${params} ${CONTROLLER_MANAGER_TEST_ARGS}" - fi - readonly kube_rc_docker_tag=$(cat /run/kube-docker-files/kube-controller-manager.docker_tag) - - src_file="/run/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest" - remove_salt_config_comments "${src_file}" - # Evaluate variables - sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" - sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}" - sed -i -e "s@{{params}}@${params}@g" "${src_file}" - sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" - sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" - sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" - cp "${src_file}" /etc/kubernetes/manifests -} - -# Starts k8s scheduler. -# It prepares the log file, loads the docker image, calculates variables, sets them -# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. -# -# Assumed vars (which are calculated in compute_master_manifest_variables()) -# DOCKER_REGISTRY -start_kube_scheduler() { - prepare_log_file /var/log/kube-scheduler.log - # Load the docker image from file. - echo "Try to load docker image file kube-scheduler.tar" - timeout 30 docker load -i /run/kube-docker-files/kube-scheduler.tar - - # Calculate variables and set them in the manifest. - params="" - log_level="--v=2" - if [ -n "${SCHEDULER_TEST_LOG_LEVEL:-}" ]; then - log_level="${SCHEDULER_TEST_LOG_LEVEL}" - fi - params="${params} ${log_level}" - if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then - params="${params} ${SCHEDULER_TEST_ARGS}" - fi - readonly kube_scheduler_docker_tag=$(cat /run/kube-docker-files/kube-scheduler.docker_tag) - - # Remove salt comments and replace variables with values - src_file="/run/kube-manifests/kubernetes/trusty/kube-scheduler.manifest" - remove_salt_config_comments "${src_file}" - sed -i -e "s@{{params}}@${params}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" - sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}" - cp "${src_file}" /etc/kubernetes/manifests -} - -# Starts a fluentd static pod for logging. -start_fluentd() { - if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then - if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then - cp /run/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/ - elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then - cp /run/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/ - fi - fi -} - -# A helper function for copying addon manifests and set dir/files -# permissions. -# $1: addon category under /etc/kubernetes -# $2: manifest source dir -setup_addon_manifests() { - src_dir="/run/kube-manifests/kubernetes/trusty/$2" - dst_dir="/etc/kubernetes/$1/$2" - if [ ! -d "${dst_dir}" ]; then - mkdir -p "${dst_dir}" - fi - files=$(find "${src_dir}" -name "*.yaml") - if [ -n "${files}" ]; then - cp "${src_dir}/"*.yaml "${dst_dir}" - fi - files=$(find "${src_dir}" -name "*.json") - if [ -n "${files}" ]; then - cp "${src_dir}/"*.json "${dst_dir}" - fi - files=$(find "${src_dir}" -name "*.yaml.in") - if [ -n "${files}" ]; then - cp "${src_dir}/"*.yaml.in "${dst_dir}" - fi - chown -R root:root "${dst_dir}" - chmod 755 "${dst_dir}" - chmod 644 "${dst_dir}"/* -} - -# Prepares the manifests of k8s addons static pods. -prepare_kube_addons() { - addon_src_dir="/run/kube-manifests/kubernetes/trusty" - addon_dst_dir="/etc/kubernetes/addons" - # Set up manifests of other addons. - if [ "${ENABLE_CLUSTER_MONITORING:-}" = "influxdb" ] || \ - [ "${ENABLE_CLUSTER_MONITORING:-}" = "google" ] || \ - [ "${ENABLE_CLUSTER_MONITORING:-}" = "standalone" ] || \ - [ "${ENABLE_CLUSTER_MONITORING:-}" = "googleinfluxdb" ]; then - file_dir="cluster-monitoring/${ENABLE_CLUSTER_MONITORING}" - setup_addon_manifests "addons" "${file_dir}" - # Replace the salt configurations with variable values. - metrics_memory="200Mi" - eventer_memory="200Mi" - if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -ge 1 ]; then - num_kube_nodes="$((${NUM_NODES}-1))" - metrics_memory="$((${num_kube_nodes} * 4 + 200))Mi" - eventer_memory="$((${num_kube_nodes} * 500 + 200 * 1024))Ki" - fi - controller_yaml="${addon_dst_dir}/${file_dir}" - if [ "${ENABLE_CLUSTER_MONITORING:-}" = "googleinfluxdb" ]; then - controller_yaml="${controller_yaml}/heapster-controller-combined.yaml" - else - controller_yaml="${controller_yaml}/heapster-controller.yaml" - fi - remove_salt_config_comments "${controller_yaml}" - sed -i -e "s@{{ *metrics_memory *}}@${metrics_memory}@g" "${controller_yaml}" - sed -i -e "s@{{ *eventer_memory *}}@${eventer_memory}@g" "${controller_yaml}" - fi - cp "${addon_src_dir}/namespace.yaml" "${addon_dst_dir}" - if [ "${ENABLE_L7_LOADBALANCING:-}" = "glbc" ]; then - setup_addon_manifests "addons" "cluster-loadbalancing/glbc" - fi - if [ "${ENABLE_CLUSTER_DNS:-}" = "true" ]; then - setup_addon_manifests "addons" "dns" - dns_rc_file="${addon_dst_dir}/dns/skydns-rc.yaml" - dns_svc_file="${addon_dst_dir}/dns/skydns-svc.yaml" - mv "${addon_dst_dir}/dns/skydns-rc.yaml.in" "${dns_rc_file}" - mv "${addon_dst_dir}/dns/skydns-svc.yaml.in" "${dns_svc_file}" - # Replace the salt configurations with variable values. - sed -i -e "s@{{ *pillar\['dns_replicas'\] *}}@${DNS_REPLICAS}@g" "${dns_rc_file}" - sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${dns_rc_file}" - sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${dns_svc_file}" - fi - if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then - setup_addon_manifests "addons" "registry" - registry_pv_file="${addon_dst_dir}/registry/registry-pv.yaml" - registry_pvc_file="${addon_dst_dir}/registry/registry-pvc.yaml" - mv "${addon_dst_dir}/registry/registry-pv.yaml.in" "${registry_pv_file}" - mv "${addon_dst_dir}/registry/registry-pvc.yaml.in" "${registry_pvc_file}" - # Replace the salt configurations with variable values. - remove_salt_config_comments "${controller_yaml}" - sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pv_file}" - sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pvc_file}" - sed -i -e "s@{{ *pillar\['cluster_registry_disk_name'\] *}}@${CLUSTER_REGISTRY_DISK}@g" "${registry_pvc_file}" - fi - if [ "${ENABLE_NODE_LOGGING:-}" = "true" ] && \ - [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ] && \ - [ "${ENABLE_CLUSTER_LOGGING:-}" = "true" ]; then - setup_addon_manifests "addons" "fluentd-elasticsearch" - fi - if [ "${ENABLE_CLUSTER_UI:-}" = "true" ]; then - setup_addon_manifests "addons" "dashboard" - fi - if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then - setup_addon_manifests "admission-controls" "limit-range" - fi - - # Prepare the scripts for running addons. - addon_script_dir="/var/lib/cloud/scripts/kubernetes" - mkdir -p "${addon_script_dir}" - cp "${addon_src_dir}/kube-addons.sh" "${addon_script_dir}" - cp "${addon_src_dir}/kube-addon-update.sh" "${addon_script_dir}" - chmod 544 "${addon_script_dir}/"*.sh - # In case that some GCE customized trusty may have a read-only /root. - mount -t tmpfs tmpfs /root - mount --bind -o remount,rw,noexec /root -} diff --git a/cluster/gce/trusty/master.yaml b/cluster/gce/trusty/master.yaml index 2f2a922cdce03..ca4bc4ee2fda9 100644 --- a/cluster/gce/trusty/master.yaml +++ b/cluster/gce/trusty/master.yaml @@ -1,4 +1,4 @@ -From nobody Thu Mar 3 15:33:00 2016 +From nobody Thu Mar 10 10:33:00 2016 Content-Type: multipart/mixed; boundary="====================================" MIME-Version: 1.0 @@ -6,11 +6,11 @@ MIME-Version: 1.0 MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="kube-env.conf" +Content-Disposition: attachment; filename="kube-install-master.conf" #upstart-job -description "Prepare kube master environment" +description "Download and install k8s binaries and configurations" start on cloud-config @@ -18,20 +18,41 @@ script set -o errexit set -o nounset - # Fetch the script for configuring the instance. + # Fetch the script for installing master binary and configuration files. curl --fail --silent --show-error \ -H "X-Google-Metadata-Request: True" \ -o /etc/kube-configure.sh \ http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh . /etc/kube-configure.sh + echo "Downloading kube-env file" + download_kube_env + . /etc/kube-env + echo "Install kube master binary and configuration files" + install_kube_binary_config +end script +--==================================== +MIME-Version: 1.0 +Content-Type: text/upstart-job; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="kube-env.conf" + +#upstart-job + +description "Prepare kube master environment" + +start on stopped kube-install-master + +script + set -o errexit + set -o nounset + + . /etc/kube-configure-helper.sh + . /etc/kube-env echo "Configuring hostname" config_hostname echo "Configuring IP firewall rules" config_ip_firewall - echo "Downloading kube-env file" - download_kube_env - . /etc/kube-env echo "Creating required directories" create_dirs echo "Mount master PD" @@ -52,13 +73,13 @@ Content-Disposition: attachment; filename="kube-install-packages.conf" description "Install packages needed to run kubernetes" -start on stopped kube-env +start on stopped kube-install-master script set -o errexit set -o nounset - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh install_critical_packages end script @@ -78,31 +99,10 @@ script set -o errexit set -o nounset - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh install_additional_packages end script ---==================================== -MIME-Version: 1.0 -Content-Type: text/upstart-job; charset="us-ascii" -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="kube-install-master.conf" - -#upstart-job - -description "Download and install k8s binaries and configurations" - -start on stopped kube-env - -script - set -o errexit - set -o nounset - - . /etc/kube-configure.sh - . /etc/kube-env - install_kube_binary_config -end script - --==================================== MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" @@ -113,7 +113,7 @@ Content-Disposition: attachment; filename="kubelet.conf" description "Run kubelet service" -start on stopped kube-install-packages and stopped kube-install-master +start on stopped kube-install-packages and stopped kube-env respawn @@ -122,7 +122,7 @@ script set -o nounset echo "Start kubelet upstart job" - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh . /etc/kube-env # Assemble command line flags based on env variables, which will put the string # of flags in variable KUBELET_CMD_FLAGS @@ -163,7 +163,7 @@ script set -o errexit set -o nounset - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh . /etc/kube-env restart_docker_daemon end script @@ -184,7 +184,7 @@ script set -o errexit set -o nounset - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh . /etc/kube-env start_etcd_servers start_fluentd @@ -247,7 +247,7 @@ script # Wait for a minute to let docker and kubelet processes finish initialization. # TODO(andyzheng0831): replace it with a more reliable method if possible. sleep 60 - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh . /etc/kube-env health_monitoring end script diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml index fc072bb6c5374..0f610370d7124 100644 --- a/cluster/gce/trusty/node.yaml +++ b/cluster/gce/trusty/node.yaml @@ -1,4 +1,4 @@ -From nobody Thu Mar 3 15:33:00 2016 +From nobody Thu Mar 10 10:33:00 2016 Content-Type: multipart/mixed; boundary="====================================" MIME-Version: 1.0 @@ -6,11 +6,11 @@ MIME-Version: 1.0 MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="kube-env.conf" +Content-Disposition: attachment; filename="kube-install-minion.conf" #upstart-job -description "Prepare kube node environment" +description "Download and install k8s binaries and configurations" start on cloud-config @@ -18,87 +18,87 @@ script set -o errexit set -o nounset - # Fetch the script for configuring the instance. + # Fetch the script for installing nodes binary and configuration files. curl --fail --silent --show-error \ -H "X-Google-Metadata-Request: True" \ -o /etc/kube-configure.sh \ http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh . /etc/kube-configure.sh - - echo "Configuring hostname" - config_hostname - echo "Configuring IP firewall rules" - config_ip_firewall echo "Downloading kube-env file" download_kube_env - echo "Creating required directories" . /etc/kube-env - create_dirs - echo "Creating kubelet kubeconfig file" - create_kubelet_kubeconfig - echo "Creating kube-proxy kubeconfig file" - create_kubeproxy_kubeconfig + echo "Install kube nodes binary and configuration files" + install_kube_binary_config end script --==================================== MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="kube-install-packages.conf" +Content-Disposition: attachment; filename="kube-env.conf" #upstart-job -description "Install packages needed to run kubernetes" +description "Prepare kube node environment" -start on stopped kube-env +start on stopped kube-install-minion script set -o errexit set -o nounset - . /etc/kube-configure.sh - install_critical_packages + . /etc/kube-configure-helper.sh + . /etc/kube-env + echo "Configuring hostname" + config_hostname + echo "Configuring IP firewall rules" + config_ip_firewall + echo "Creating required directories" + create_dirs + echo "Creating kubelet kubeconfig file" + create_kubelet_kubeconfig + echo "Creating kube-proxy kubeconfig file" + create_kubeproxy_kubeconfig end script --==================================== MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="kube-install-additional-packages.conf" +Content-Disposition: attachment; filename="kube-install-packages.conf" #upstart-job -description "Install additional packages used by kubernetes" +description "Install packages needed to run kubernetes" -start on stopped kube-install-packages +start on stopped kube-install-minion script set -o errexit set -o nounset - . /etc/kube-configure.sh - install_additional_packages + . /etc/kube-configure-helper.sh + install_critical_packages end script --==================================== MIME-Version: 1.0 Content-Type: text/upstart-job; charset="us-ascii" Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="kube-install-minion.conf" +Content-Disposition: attachment; filename="kube-install-additional-packages.conf" #upstart-job -description "Download and install k8s binaries and configurations" +description "Install additional packages used by kubernetes" -start on stopped kube-env +start on stopped kube-install-packages script set -o errexit set -o nounset - . /etc/kube-configure.sh - . /etc/kube-env - install_kube_binary_config + . /etc/kube-configure-helper.sh + install_additional_packages end script --==================================== @@ -111,7 +111,7 @@ Content-Disposition: attachment; filename="kubelet.conf" description "Run kubelet service" -start on stopped kube-install-minion and stopped kube-install-packages +start on stopped kube-install-packages and stopped kube-env respawn @@ -120,7 +120,7 @@ script set -o nounset echo "Start kubelet upstart job" - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh . /etc/kube-env # Assemble command line flags based on env variables, which will put the string # of flags in variable KUBELET_CMD_FLAGS. @@ -162,7 +162,7 @@ script set -o errexit set -o nounset - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh . /etc/kube-env restart_docker_daemon end script @@ -183,7 +183,7 @@ script set -o errexit set -o nounset - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh . /etc/kube-env prepare_log_file "/var/log/kube-proxy.log" # Load the docker image from file /run/kube-docker-files/kube-proxy.tar. @@ -239,7 +239,7 @@ script set -o nounset # Kube-system pod manifest files are located at /run/kube-manifests/kubernetes. - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh . /etc/kube-env # Fluentd start_fluentd @@ -268,7 +268,7 @@ script # Wait for a minute to let docker, kubelet, and kube-proxy processes finish initialization. # TODO(andyzheng0831): replace it with a more reliable method if possible. sleep 60 - . /etc/kube-configure.sh + . /etc/kube-configure-helper.sh . /etc/kube-env health_monitoring end script diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 19a2560503e66..b75a511e82a43 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -13,7 +13,7 @@ cluster/gce/configure-vm.sh: cloud_config: ${CLOUD_CONFIG} cluster/gce/configure-vm.sh: env-to-grains "runtime_config" cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}' cluster/gce/coreos/helper.sh:# cloud_config yaml file should be passed -cluster/gce/trusty/configure.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" +cluster/gce/trusty/configure-helper.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" cluster/gce/util.sh: local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \ cluster/juju/layers/kubernetes/reactive/k8s.py: check_call(split(cmd.format(directory, cluster_name, public_address, cluster/juju/layers/kubernetes/reactive/k8s.py: check_call(split(cmd.format(directory, context, cluster_name, user))) @@ -24,7 +24,6 @@ cluster/lib/logging.sh: local source_file=${BASH_SOURCE[$frame_no]} cluster/lib/logging.sh: local source_file=${BASH_SOURCE[$stack_skip]} cluster/log-dump.sh: for node_name in "${NODE_NAMES[@]}"; do cluster/log-dump.sh: local -r node_name="${1}" -cluster/log-dump.sh: local -r node_name="${1}" cluster/log-dump.sh:readonly report_dir="${1:-_artifacts}" cluster/mesos/docker/km/build.sh: km_path=$(find-binary km darwin/amd64) cluster/rackspace/util.sh: local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \