Configure the kubelet to use HTTPS (take 2)

cluster/addons/cluster-monitoring/heapster-controller.yaml

@@ -1,9 +1,9 @@
 apiVersion: v1beta1
 id: monitoring-heapster-controller
 kind: ReplicationController
-desiredState: 
+desiredState:
   replicas: 1
-  replicaSelector:    
+  replicaSelector:
     name: heapster
   podTemplate:
     desiredState:
@@ -13,11 +13,13 @@ desiredState:
         containers:
           - name: heapster
             image: gcr.io/google_containers/heapster:v0.10.0
-            env: 
+            env:
               - name: "INFLUXDB_HOST"
                 value: "monitoring-influxdb"
               - name: "SINK"
                 value: "influxdb"
+              - name: "FLAGS"
+                value: "--kubelet_port=10255"
             volumeMounts:
               - name: ssl-certs
                 mountPath: /etc/ssl/certs
@@ -27,10 +29,10 @@ desiredState:
             source:
               hostDir:
                 path: /etc/ssl/certs
-    labels: 
+    labels:
       name: heapster
       uses: monitoring-influxdb
       kubernetes.io/cluster-service: "true"
-labels: 
+labels:
   name: heapster
   kubernetes.io/cluster-service: "true"

cmd/kube-apiserver/app/server.go

@@ -100,7 +100,7 @@ func NewAPIServer() *APIServer {
 		RuntimeConfig: make(util.ConfigurationMap),
 		KubeletConfig: client.KubeletConfig{
 			Port:        10250,
-			EnableHttps: false,
+			EnableHttps: true,
 			HTTPTimeout: time.Duration(5) * time.Second,
 		},
 	}

cmd/kube-controller-manager/app/controllermanager.go

@@ -83,7 +83,7 @@ func NewCMServer() *CMServer {
 		SyncNodeStatus:          false,
 		KubeletConfig: client.KubeletConfig{
 			Port:        ports.KubeletPort,
-			EnableHttps: false,
+			EnableHttps: true,
 			HTTPTimeout: time.Duration(5) * time.Second,
 		},
 	}

cmd/kubelet/app/server.go

@@ -18,10 +18,12 @@ limitations under the License.
 package app
 
 import (
+	"crypto/tls"
 	"fmt"
 	"math/rand"
 	"net"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -59,6 +61,7 @@ type KubeletServer struct {
 	EnableServer                   bool
 	Address                        util.IP
 	Port                           uint
+	ReadOnlyPort                   uint
 	HostnameOverride               string
 	PodInfraContainerImage         string
 	DockerEndpoint                 string
@@ -88,17 +91,21 @@ type KubeletServer struct {
 	NetworkPluginName              string
 	CloudProvider                  string
 	CloudConfigFile                string
+	TLSCertFile                    string
+	TLSPrivateKeyFile              string
+	CertDirectory                  string
 }
 
 // NewKubeletServer will create a new KubeletServer with default values.
 func NewKubeletServer() *KubeletServer {
 	return &KubeletServer{
-		SyncFrequency:      10 * time.Second,
-		FileCheckFrequency: 20 * time.Second,
-		HTTPCheckFrequency: 20 * time.Second,
-		EnableServer:       true,
-		Address:            util.IP(net.ParseIP("0.0.0.0")),
-		Port:               ports.KubeletPort,
+		SyncFrequency:               10 * time.Second,
+		FileCheckFrequency:          20 * time.Second,
+		HTTPCheckFrequency:          20 * time.Second,
+		EnableServer:                true,
+		Address:                     util.IP(net.ParseIP("0.0.0.0")),
+		Port:                        ports.KubeletPort,
+		ReadOnlyPort:                ports.KubeletReadOnlyPort,
 		PodInfraContainerImage:      kubelet.PodInfraContainerImage,
 		RootDirectory:               defaultRootDir,
 		RegistryBurst:               10,
@@ -116,6 +123,7 @@ func NewKubeletServer() *KubeletServer {
 		ImageGCLowThresholdPercent:  80,
 		NetworkPluginName:           "",
 		HostNetworkSources:          kubelet.FileSource,
+		CertDirectory:               "/var/run/kubernetes",
 	}
 }
 
@@ -129,6 +137,13 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.EnableServer, "enable_server", s.EnableServer, "Enable the info server")
 	fs.Var(&s.Address, "address", "The IP address for the info server to serve on (set to 0.0.0.0 for all interfaces)")
 	fs.UintVar(&s.Port, "port", s.Port, "The port for the info server to serve on")
+	fs.UintVar(&s.ReadOnlyPort, "read_only_port", s.ReadOnlyPort, "The read-only port for the info server to serve on (set to 0 to disable)")
+	fs.StringVar(&s.TLSCertFile, "tls_cert_file", s.TLSCertFile, ""+
+		"File containing x509 Certificate for HTTPS.  (CA cert, if any, concatenated after server cert). "+
+		"If --tls_cert_file and --tls_private_key_file are not provided, a self-signed certificate and key "+
+		"are generated for the public address and saved to the directory passed to --cert_dir.")
+	fs.StringVar(&s.TLSPrivateKeyFile, "tls_private_key_file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls_cert_file.")
+	fs.StringVar(&s.CertDirectory, "cert_dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes)")
 	fs.StringVar(&s.HostnameOverride, "hostname_override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
 	fs.StringVar(&s.PodInfraContainerImage, "pod_infra_container_image", s.PodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.")
 	fs.StringVar(&s.DockerEndpoint, "docker_endpoint", s.DockerEndpoint, "If non-empty, use this for the docker endpoint to communicate with")
@@ -195,6 +210,26 @@ func (s *KubeletServer) Run(_ []string) error {
 	if err != nil {
 		return err
 	}
+
+	if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
+		s.TLSCertFile = path.Join(s.CertDirectory, "kubelet.crt")
+		s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "kubelet.key")
+		if err := util.GenerateSelfSignedCert(util.GetHostname(s.HostnameOverride), s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
+			glog.Fatalf("Unable to generate self signed cert: %v", err)
+		}
+		glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
+	}
+	tlsOptions := &kubelet.TLSOptions{
+		Config: &tls.Config{
+			// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability).
+			MinVersion: tls.VersionTLS10,
+			// Populate PeerCertificates in requests, but don't yet reject connections without certificates.
+			ClientAuth: tls.RequestClientCert,
+		},
+		CertFile: s.TLSCertFile,
+		KeyFile:  s.TLSPrivateKeyFile,
+	}
+
 	kcfg := KubeletConfig{
 		Address:                        s.Address,
 		AllowPrivileged:                s.AllowPrivileged,
@@ -216,6 +251,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		ClusterDNS:                     s.ClusterDNS,
 		Runonce:                        s.RunOnce,
 		Port:                           s.Port,
+		ReadOnlyPort:                   s.ReadOnlyPort,
 		CadvisorInterface:              cadvisorInterface,
 		EnableServer:                   s.EnableServer,
 		EnableDebuggingHandlers:        s.EnableDebuggingHandlers,
@@ -226,6 +262,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		NetworkPlugins:                 ProbeNetworkPlugins(),
 		NetworkPluginName:              s.NetworkPluginName,
 		StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout,
+		TLSOptions:                     tlsOptions,
 		ImageGCPolicy:                  imageGCPolicy,
 		Cloud:                          cloud,
 	}
@@ -381,6 +418,11 @@ func startKubelet(k *kubelet.Kubelet, podCfg *config.PodConfig, kc *KubeletConfi
 			kubelet.ListenAndServeKubeletServer(k, net.IP(kc.Address), kc.Port, kc.TLSOptions, kc.EnableDebuggingHandlers)
 		}, 0)
 	}
+	if kc.ReadOnlyPort > 0 {
+		go util.Forever(func() {
+			kubelet.ListenAndServeKubeletReadOnlyServer(k, net.IP(kc.Address), kc.ReadOnlyPort)
+		}, 0)
+	}
 }
 
 func makePodSourceConfig(kc *KubeletConfig) *config.PodConfig {
@@ -433,6 +475,7 @@ type KubeletConfig struct {
 	EnableServer                   bool
 	EnableDebuggingHandlers        bool
 	Port                           uint
+	ReadOnlyPort                   uint
 	Runonce                        bool
 	MasterServiceNamespace         string
 	VolumePlugins                  []volume.VolumePlugin

hack/test-cmd.sh

@@ -48,6 +48,7 @@ ETCD_PORT=${ETCD_PORT:-4001}
 API_PORT=${API_PORT:-8080}
 API_HOST=${API_HOST:-127.0.0.1}
 KUBELET_PORT=${KUBELET_PORT:-10250}
+KUBELET_HEALTHZ_PORT=${KUBELET_HEALTHZ_PORT:-10248}
 CTLRMGR_PORT=${CTLRMGR_PORT:-10252}
 
 # Check kubectl
@@ -58,27 +59,31 @@ kube::log::status "Starting kubelet in masterless mode"
 "${KUBE_OUTPUT_HOSTBIN}/kubelet" \
   --really_crash_for_testing=true \
   --root_dir=/tmp/kubelet.$$ \
+  --cert_dir="${TMPDIR:-/tmp/}" \
   --docker_endpoint="fake://" \
   --hostname_override="127.0.0.1" \
   --address="127.0.0.1" \
-  --port="$KUBELET_PORT" 1>&2 &
+  --port="$KUBELET_PORT" \
+  --healthz_port="${KUBELET_HEALTHZ_PORT}" 1>&2 &
 KUBELET_PID=$!
-kube::util::wait_for_url "http://127.0.0.1:${KUBELET_PORT}/healthz" "kubelet: " 0.2 25
+kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet: " 0.2 25
 kill ${KUBELET_PID} 1>&2 2>/dev/null
 
 kube::log::status "Starting kubelet in masterful mode"
 "${KUBE_OUTPUT_HOSTBIN}/kubelet" \
   --really_crash_for_testing=true \
   --root_dir=/tmp/kubelet.$$ \
+  --cert_dir="${TMPDIR:-/tmp/}" \
   --docker_endpoint="fake://" \
   --hostname_override="127.0.0.1" \
   --address="127.0.0.1" \
   --api_servers="${API_HOST}:${API_PORT}" \
   --auth_path="${KUBE_ROOT}/hack/.test-cmd-auth" \
-  --port="$KUBELET_PORT" 1>&2 &
+  --port="$KUBELET_PORT" \
+  --healthz_port="${KUBELET_HEALTHZ_PORT}" 1>&2 &
 KUBELET_PID=$!
 
-kube::util::wait_for_url "http://127.0.0.1:${KUBELET_PORT}/healthz" "kubelet: " 0.2 25
+kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet: " 0.2 25
 
 # Start kube-apiserver
 kube::log::status "Starting kube-apiserver"

pkg/apiserver/apiserver.go

@@ -139,14 +139,7 @@ func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
 
 // TODO: Convert to go-restful
 func InstallValidator(mux Mux, servers func() map[string]Server) {
-	validator, err := NewValidator(servers)
-	if err != nil {
-		glog.Errorf("failed to set up validator: %v", err)
-		return
-	}
-	if validator != nil {
-		mux.Handle("/validate", validator)
-	}
+	mux.Handle("/validate", NewValidator(servers))
 }
 
 // TODO: document all handlers

pkg/apiserver/validator.go

@@ -17,6 +17,7 @@ limitations under the License.
 package apiserver
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -34,21 +35,26 @@ type httpGet interface {
 }
 
 type Server struct {
-	Addr string
-	Port int
-	Path string
+	Addr        string
+	Port        int
+	Path        string
+	EnableHTTPS bool
 }
 
 // validator is responsible for validating the cluster and serving
 type validator struct {
 	// a list of servers to health check
 	servers func() map[string]Server
-	client  httpGet
+	rt      http.RoundTripper
 }
 
 // TODO: can this use pkg/probe/http
 func (s *Server) check(client httpGet) (probe.Result, string, error) {
-	resp, err := client.Get("http://" + net.JoinHostPort(s.Addr, strconv.Itoa(s.Port)) + s.Path)
+	scheme := "http://"
+	if s.EnableHTTPS {
+		scheme = "https://"
+	}
+	resp, err := client.Get(scheme + net.JoinHostPort(s.Addr, strconv.Itoa(s.Port)) + s.Path)
 	if err != nil {
 		return probe.Unknown, "", err
 	}
@@ -81,7 +87,22 @@ func (v *validator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	reply := []ServerStatus{}
 	for name, server := range v.servers() {
-		status, msg, err := server.check(v.client)
+		transport := v.rt
+		if server.EnableHTTPS {
+			// TODO(roberthbailey): The servers that use HTTPS are currently the
+			// kubelets, and we should be using a standard kubelet client library
+			// to talk to them rather than a separate http client.
+			transport = &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+				Dial: (&net.Dialer{
+					Timeout:   30 * time.Second,
+					KeepAlive: 30 * time.Second,
+				}).Dial,
+				TLSHandshakeTimeout: 10 * time.Second,
+				TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
+			}
+		}
+		status, msg, err := server.check(&http.Client{Transport: transport})
 		var errorMsg string
 		if err != nil {
 			errorMsg = err.Error()
@@ -103,30 +124,6 @@ func (v *validator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 // NewValidator creates a validator for a set of servers.
-func NewValidator(servers func() map[string]Server) (http.Handler, error) {
-	return &validator{
-		servers: servers,
-		client:  &http.Client{},
-	}, nil
-}
-
-func makeTestValidator(servers map[string]string, get httpGet) (http.Handler, error) {
-	result := map[string]Server{}
-	for name, value := range servers {
-		host, port, err := net.SplitHostPort(value)
-		if err != nil {
-			return nil, fmt.Errorf("invalid server spec: %s (%v)", value, err)
-		}
-		val, err := strconv.Atoi(port)
-		if err != nil {
-			return nil, fmt.Errorf("invalid server spec: %s (%v)", port, err)
-		}
-		result[name] = Server{Addr: host, Port: val, Path: "/healthz"}
-	}
-
-	v, e := NewValidator(func() map[string]Server { return result })
-	if e == nil {
-		v.(*validator).client = get
-	}
-	return v, e
+func NewValidator(servers func() map[string]Server) http.Handler {
+	return &validator{servers: servers, rt: http.DefaultTransport}
 }