Use presence of kubeconfig file to toggle standalone mode

build/debs/kubeadm-10.conf

@@ -1,5 +1,5 @@
 [Service]
-Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
+Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf"
 Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
 Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 Environment="KUBELET_DNS_ARGS=--cluster-dns=10.96.0.10 --cluster-domain=cluster.local"

build/rpms/10-kubeadm.conf

@@ -1,5 +1,5 @@
 [Service]
-Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
+Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf"
 Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
 Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 Environment="KUBELET_DNS_ARGS=--cluster-dns=10.96.0.10 --cluster-domain=cluster.local"

cluster/centos/node/scripts/kubelet.sh

@@ -14,12 +14,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 MASTER_ADDRESS=${1:-"8.8.8.18"}
 NODE_ADDRESS=${2:-"8.8.8.20"}
 DNS_SERVER_IP=${3:-"192.168.3.100"}
 DNS_DOMAIN=${4:-"cluster.local"}
-
+KUBECONFIG_DIR=${KUBECONFIG_DIR:-/opt/kubernetes/cfg}
+
+# Generate a kubeconfig file
+cat <<EOF > "${KUBECONFIG_DIR}/kubelet.kubeconfig"
+apiVersion: v1
+kind: Config
+clusters:
+  - cluster:
+      server: http://${MASTER_ADDRESS}:8080/
+    name: local
+contexts:
+  - context:
+      cluster: local
+    name: local
+current-context: local
+EOF
 
 cat <<EOF >/opt/kubernetes/cfg/kubelet
 # --logtostderr=true: log to standard error instead of files
@@ -37,9 +51,8 @@ NODE_PORT="--port=10250"
 # --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname.
 NODE_HOSTNAME="--hostname-override=${NODE_ADDRESS}"
 
-# --api-servers=[]: List of Kubernetes API servers for publishing events,
-# and reading pods and services. (ip:port), comma separated.
-KUBELET_API_SERVER="--api-servers=${MASTER_ADDRESS}:8080"
+# Path to a kubeconfig file, specifying how to connect to the API server.
+KUBELET_KUBECONFIG="--kubeconfig=${KUBECONFIG_DIR}/kubelet.kubeconfig"
 
 # --allow-privileged=false: If true, allow containers to request privileged mode. [default=false]
 KUBE_ALLOW_PRIV="--allow-privileged=false"
@@ -52,15 +65,15 @@ KUBELET_DNS_DOMAIN="--cluster-domain=${DNS_DOMAIN}"
 KUBELET_ARGS=""
 EOF
 
-KUBE_PROXY_OPTS="   \${KUBE_LOGTOSTDERR}     \\
+KUBELET_OPTS="      \${KUBE_LOGTOSTDERR}     \\
                     \${KUBE_LOG_LEVEL}       \\
                     \${NODE_ADDRESS}         \\
                     \${NODE_PORT}            \\
                     \${NODE_HOSTNAME}        \\
-                    \${KUBELET_API_SERVER}   \\
+                    \${KUBELET_KUBECONFIG}   \\
                     \${KUBE_ALLOW_PRIV}      \\
                     \${KUBELET__DNS_IP}      \\
-                    \${KUBELET_DNS_DOMAIN}      \\
+                    \${KUBELET_DNS_DOMAIN}   \\
                     \$KUBELET_ARGS"
 
 cat <<EOF >/usr/lib/systemd/system/kubelet.service
@@ -71,7 +84,7 @@ Requires=docker.service
 
 [Service]
 EnvironmentFile=-/opt/kubernetes/cfg/kubelet
-ExecStart=/opt/kubernetes/bin/kubelet ${KUBE_PROXY_OPTS}
+ExecStart=/opt/kubernetes/bin/kubelet ${KUBELET_OPTS}
 Restart=on-failure
 KillMode=process
 

cluster/gce/configure-vm.sh

@@ -576,7 +576,7 @@ EOF
       cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
 node_taints: '$(echo "${NODE_TAINTS}" | sed -e "s/'/''/g")'
 EOF
-    fi    
+    fi
     if [ -n "${EVICTION_HARD:-}" ]; then
       cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
 eviction_hard: '$(echo "${EVICTION_HARD}" | sed -e "s/'/''/g")'
@@ -753,12 +753,16 @@ EOF
 }
 
 function salt-node-role() {
+  local -r kubelet_bootstrap_kubeconfig="/srv/salt-overlay/salt/kubelet/bootstrap-kubeconfig"
+  local -r kubelet_kubeconfig="/srv/salt-overlay/salt/kubelet/kubeconfig"
   cat <<EOF >/etc/salt/minion.d/grains.conf
 grains:
   roles:
     - kubernetes-pool
   cloud: gce
   api_servers: '${KUBERNETES_MASTER_NAME}'
+  kubelet_bootstrap_kubeconfig: /var/lib/kubelet/bootstrap-kubeconfig
+  kubelet_kubeconfig: /var/lib/kubelet/kubeconfig
 EOF
 }
 

cluster/gce/container-linux/configure-helper.sh

@@ -341,7 +341,13 @@ EOF
   fi
 }
 
-function create-kubelet-kubeconfig {
+# Arg 1: the address of the API server
+function create-kubelet-kubeconfig() {
+  local apiserver_address="${1}"
+  if [[ -z "${apiserver_address}" ]]; then
+    echo "Must provide API server address to create Kubelet kubeconfig file!"
+    exit 1
+  fi
   echo "Creating kubelet kubeconfig file"
   if [[ -z "${KUBELET_CA_CERT:-}" ]]; then
     KUBELET_CA_CERT="${CA_CERT}"
@@ -357,6 +363,7 @@ users:
 clusters:
 - name: local
   cluster:
+    server: ${apiserver_address}
     certificate-authority-data: ${KUBELET_CA_CERT}
 contexts:
 - context:
@@ -376,7 +383,7 @@ function create-master-kubelet-auth {
   # set in the environment.
   if [[ -n "${KUBELET_APISERVER:-}" && -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then
     REGISTER_MASTER_KUBELET="true"
-    create-kubelet-kubeconfig
+    create-kubelet-kubeconfig "https://${KUBELET_APISERVER}"
   fi
 }
 
@@ -576,15 +583,15 @@ function start-kubelet {
     flags+=" --enable-debugging-handlers=false"
     flags+=" --hairpin-mode=none"
     if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
-      flags+=" --api-servers=https://${KUBELET_APISERVER}"
+      flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
       flags+=" --register-schedulable=false"
     else
       # Standalone mode (not widely used?)
       flags+=" --pod-cidr=${MASTER_IP_RANGE}"
     fi
   else # For nodes
     flags+=" --enable-debugging-handlers=true"
-    flags+=" --api-servers=https://${KUBERNETES_MASTER_NAME}"
+    flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
     if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
        [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
        [[ "${HAIRPIN_MODE:-}" == "none" ]]; then
@@ -1282,15 +1289,15 @@ function start-kube-addons {
   if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
     setup-addon-manifests "addons" "calico-policy-controller"
 
-    # Configure Calico based on cluster size and image type. 
+    # Configure Calico based on cluster size and image type.
     local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
     local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml"
     sed -i -e "s@__CALICO_CNI_DIR__@/opt/cni/bin@g" "${ds_file}"
     sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-node-cpu)@g" "${ds_file}"
     sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}"
     sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}"
   else
-    # If not configured to use Calico, the set the typha replica count to 0, but only if the 
+    # If not configured to use Calico, the set the typha replica count to 0, but only if the
     # addon is present.
     local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml"
     if [[ -e $typha_dep_file ]]; then
@@ -1439,7 +1446,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
   create-master-kubelet-auth
   create-master-etcd-auth
 else
-  create-kubelet-kubeconfig
+  create-kubelet-kubeconfig "https://${KUBERNETES_MASTER_NAME}"
   create-kubeproxy-kubeconfig
 fi
 

cluster/gce/gci/configure-helper.sh

@@ -624,7 +624,13 @@ EOF
   fi
 }
 
-function create-kubelet-kubeconfig {
+# Arg 1: the IP address of the API server
+function create-kubelet-kubeconfig() {
+  local apiserver_address="${1}"
+  if [[ -z "${apiserver_address}" ]]; then
+    echo "Must provide API server address to create Kubelet kubeconfig file!"
+    exit 1
+  fi
   echo "Creating kubelet kubeconfig file"
   cat <<EOF >/var/lib/kubelet/bootstrap-kubeconfig
 apiVersion: v1
@@ -637,6 +643,7 @@ users:
 clusters:
 - name: local
   cluster:
+    server: https://${apiserver_address}
     certificate-authority: ${CA_CERT_BUNDLE_PATH}
     server: https://${KUBERNETES_MASTER_NAME}
 contexts:
@@ -657,7 +664,7 @@ function create-master-kubelet-auth {
   # set in the environment.
   if [[ -n "${KUBELET_APISERVER:-}" && -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then
     REGISTER_MASTER_KUBELET="true"
-    create-kubelet-kubeconfig
+    create-kubelet-kubeconfig ${KUBELET_APISERVER}
   fi
 }
 
@@ -898,7 +905,6 @@ function start-kubelet {
       #flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
       #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
       flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
-      flags+=" --require-kubeconfig"
       flags+=" --register-schedulable=false"
     else
       # Standalone mode (not widely used?)
@@ -908,7 +914,6 @@ function start-kubelet {
     flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
     flags+=" --enable-debugging-handlers=true"
     flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
-    flags+=" --require-kubeconfig"
     flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
     if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
        [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
@@ -951,7 +956,7 @@ function start-kubelet {
   fi
   if [[ -n "${NODE_TAINTS:-}" ]]; then
     flags+=" --register-with-taints=${NODE_TAINTS}"
-  fi  
+  fi
   if [[ -n "${EVICTION_HARD:-}" ]]; then
     flags+=" --eviction-hard=${EVICTION_HARD}"
   fi
@@ -1875,7 +1880,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
   create-master-etcd-auth
 else
   create-node-pki
-  create-kubelet-kubeconfig
+  create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
   create-kubeproxy-kubeconfig
   if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
     create-node-problem-detector-kubeconfig

cluster/get-kube-local.sh

@@ -21,6 +21,7 @@ set -o nounset
 set -o pipefail
 
 KUBE_HOST=${KUBE_HOST:-localhost}
+KUBELET_KUBECONFIG=${KUBELET_KUBECONFIG:-"/var/run/kubernetes/kubelet.kubeconfig"}
 
 declare -r RED="\033[0;31m"
 declare -r GREEN="\033[0;32m"
@@ -53,9 +54,38 @@ function run {
   fi
 }
 
+# Creates a kubeconfig file for the kubelet.
+# Args: destination file path
+function create-kubelet-kubeconfig() {
+  local destination="${2}"
+  if [[ -z "${destination}" ]]; then
+    echo "Must provide destination path to create Kubelet kubeconfig file!"
+    exit 1
+  fi
+  echo "Creating Kubelet kubeconfig file"
+  local dest_dir="$(dirname "${destination}")"
+  mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}"
+  sudo=$(test -w "${dest_dir}" || echo "sudo -E")
+  cat <<EOF | ${sudo} tee "${destination}" > /dev/null
+apiVersion: v1
+kind: Config
+clusters:
+  - cluster:
+      server: http://localhost:8080
+    name: local
+contexts:
+  - context:
+      cluster: local
+    name: local
+current-context: local
+EOF
+}
+
+
 function create_cluster {
   echo "Creating a local cluster:"
   echo -e -n "\tStarting kubelet..."
+  create-kubelet-kubeconfig "${KUBELET_KUBECONFIG}"
   run "docker run \
   --volume=/:/rootfs:ro \
   --volume=/sys:/sys:ro \
@@ -72,7 +102,7 @@ function create_cluster {
       --containerized \
       --hostname-override="127.0.0.1" \
       --address="0.0.0.0" \
-      --api-servers=http://localhost:8080 \
+      --kubeconfig=${KUBELET_KUBECONFIG}/kubelet.kubeconfig \
       --pod-manifest-path=/etc/kubernetes/manifests \
       --allow-privileged=true \
       --cluster-dns=10.0.0.10 \

cluster/libvirt-coreos/user_data_minion.yml

@@ -17,7 +17,7 @@ coreos:
         --address=0.0.0.0 \
         --hostname-override=${NODE_IPS[$i]} \
         --cluster-domain=cluster.local \
-        --api-servers=http://${MASTER_IP}:8080 \
+        --kubeconfig=/opt/kubernetes/kubeconfig/kubelet.kubeconfig \
         --tls-cert-file=/opt/kubernetes/certs/${NODE_NAMES[$i]}-node.pem \ \
         --tls-private-key-file=/opt/kubernetes/certs/${NODE_NAMES[$i]}-node-key.pem \
         $( [[ "$ENABLE_CLUSTER_DNS" == "true" ]] && echo "--cluster-dns=${DNS_SERVER_IP}" ) \

cluster/libvirt-coreos/util.sh

@@ -33,6 +33,38 @@ readonly POOL_PATH=/var/lib/libvirt/images/kubernetes
 
 [ ! -d "${POOL_PATH}" ] && (echo "$POOL_PATH" does not exist ; exit 1 )
 
+# Creates a kubeconfig file for the kubelet.
+# Args: address (e.g. "http://localhost:8080"), destination file path
+function create-kubelet-kubeconfig() {
+  local apiserver_address="${1}"
+  local destination="${2}"
+  if [[ -z "${apiserver_address}" ]]; then
+    echo "Must provide API server address to create Kubelet kubeconfig file!"
+    exit 1
+  fi
+  if [[ -z "${destination}" ]]; then
+    echo "Must provide destination path to create Kubelet kubeconfig file!"
+    exit 1
+  fi
+  echo "Creating Kubelet kubeconfig file"
+  local dest_dir="$(dirname "${destination}")"
+  mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}"
+  sudo=$(test -w "${dest_dir}" || echo "sudo -E")
+  cat <<EOF | ${sudo} tee "${destination}" > /dev/null
+apiVersion: v1
+kind: Config
+clusters:
+  - cluster:
+      server: ${apiserver_address}
+    name: local
+contexts:
+  - context:
+      cluster: local
+    name: local
+current-context: local
+EOF
+}
+
 # join <delim> <list...>
 # Concatenates the list elements with the delimiter passed as first parameter
 #
@@ -279,6 +311,7 @@ function kube-up {
   export KUBE_SERVER="http://192.168.10.1:8080"
   export CONTEXT="libvirt-coreos"
   create-kubeconfig
+  create-kubelet-kubeconfig "http://${MASTER_IP}:8080" "${POOL_PATH}/kubernetes/kubeconfig/kubelet.kubeconfig"
 
   wait-cluster-readiness
 

cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml

@@ -17,6 +17,7 @@ write_files:
         network_mode: openvswitch
         networkInterfaceName: eth0
         api_servers: $MASTER_IP
+        kubelet_kubeconfig: /srv/salt-overlay/salt/kubelet/kubeconfig
         cloud: openstack
         cloud_config: /srv/kubernetes/openstack.conf
         roles:

cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-master.yaml

@@ -34,6 +34,7 @@ write_files:
       clusters:
         - name: local
           cluster:
+            server: https://$MASTER_IP
             insecure-skip-tls-verify: true
       contexts:
         - context:

cluster/openstack-heat/kubernetes-heat/fragments/deploy-kube-auth-files-node.yaml

@@ -16,6 +16,7 @@ write_files:
       clusters:
         - name: local
           cluster:
+            server: https://$MASTER_IP
             insecure-skip-tls-verify: true
       contexts:
         - context:

cluster/photon-controller/templates/create-dynamic-salt-files.sh

@@ -39,6 +39,7 @@ apiVersion: v1
 kind: Config
 clusters:
 - cluster:
+    server: https://${KUBE_MASTER_IP}
     insecure-skip-tls-verify: true
   name: local
 contexts: