Etcd TLS Options

The current implementation does not put any transport security on the etcd cluster. The PR provides and optional flag to enable TLS the etcd cluster

- cleaned up and fixed any formatting issues on the journey
- added two new certificates (server/client) for etcd peers and a client certificate for kubeapi and others perhaps (perhaps calico?)
- disabled the protokube service for nodes completely is not required; note this was first raised in #3091, but figured it would be easier to place in here given the relation
- updated protokube codebase to reflect the changes, removing the master option as its no longer required
- added additional integretion tests for the protokube manifests;
- note, still need to add documentation, but opening the PR to get feedback
- one outstanding issue is the migration from http -> https for preexisting clusters, i'm gonna hit the coreos board to ask for the best options

nodeup/pkg/model/context.go

@@ -19,52 +19,49 @@ package model
 import (
 	"fmt"
 
-	"github.com/blang/semver"
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/upup/pkg/fi"
+
+	"github.com/blang/semver"
 )
 
+// NodeupModelContext is the context supplied the nodeup tasks
 type NodeupModelContext struct {
-	NodeupConfig *nodeup.Config
-
-	Cluster       *kops.Cluster
-	InstanceGroup *kops.InstanceGroup
-	Architecture  Architecture
-	Distribution  distros.Distribution
-
-	IsMaster bool
-
-	Assets      *fi.AssetStore
-	KeyStore    fi.CAStore
-	SecretStore fi.SecretStore
-
+	Architecture      Architecture
+	Assets            *fi.AssetStore
+	Cluster           *kops.Cluster
+	Distribution      distros.Distribution
+	InstanceGroup     *kops.InstanceGroup
+	IsMaster          bool
+	KeyStore          fi.CAStore
 	KubernetesVersion semver.Version
+	NodeupConfig      *nodeup.Config
+	SecretStore       fi.SecretStore
 }
 
+// SSLHostPaths returns the TLS paths for the distribution
 func (c *NodeupModelContext) SSLHostPaths() []string {
 	paths := []string{"/etc/ssl", "/etc/pki/tls", "/etc/pki/ca-trust"}
 
 	switch c.Distribution {
 	case distros.DistributionCoreOS:
 		// Because /usr is read-only on CoreOS, we can't have any new directories; docker will try (and fail) to create them
 		// TODO: Just check if the directories exist?
-
 		paths = append(paths, "/usr/share/ca-certificates")
-
 	case distros.DistributionContainerOS:
 		paths = append(paths, "/usr/share/ca-certificates")
-
 	default:
 		paths = append(paths, "/usr/share/ssl", "/usr/ssl", "/usr/lib/ssl", "/usr/local/openssl", "/var/ssl", "/etc/openssl")
 	}
 
 	return paths
 }
 
+// PathSrvKubernetes returns the path for the kubernetes service files
 func (c *NodeupModelContext) PathSrvKubernetes() string {
 	switch c.Distribution {
 	case distros.DistributionContainerOS:
@@ -74,6 +71,7 @@ func (c *NodeupModelContext) PathSrvKubernetes() string {
 	}
 }
 
+// PathSrvSshproxy returns the path for the SSL proxy
 func (c *NodeupModelContext) PathSrvSshproxy() string {
 	switch c.Distribution {
 	case distros.DistributionContainerOS:
@@ -83,6 +81,7 @@ func (c *NodeupModelContext) PathSrvSshproxy() string {
 	}
 }
 
+// CNIBinDir returns the path for the CNI binaries
 func (c *NodeupModelContext) CNIBinDir() string {
 	switch c.Distribution {
 	case distros.DistributionContainerOS:
@@ -92,10 +91,12 @@ func (c *NodeupModelContext) CNIBinDir() string {
 	}
 }
 
+// CNIConfDir returns the CNI directory
 func (c *NodeupModelContext) CNIConfDir() string {
 	return "/etc/cni/net.d/"
 }
 
+// buildPKIKubeconfig generates a kubeconfig
 func (c *NodeupModelContext) buildPKIKubeconfig(id string) (string, error) {
 	caCertificate, err := c.KeyStore.Cert(fi.CertificateId_CA)
 	if err != nil {
@@ -172,10 +173,12 @@ func (c *NodeupModelContext) buildPKIKubeconfig(id string) (string, error) {
 	return string(yaml), nil
 }
 
+// IsKubernetesGTE checks if the version is greater-than-or-equal
 func (c *NodeupModelContext) IsKubernetesGTE(version string) bool {
 	return util.IsKubernetesGTE(version, c.KubernetesVersion)
 }
 
+// UsesCNI checks if the cluster has CNI configured
 func (c *NodeupModelContext) UsesCNI() bool {
 	networking := c.Cluster.Spec.Networking
 	if networking == nil || networking.Classic != nil {

nodeup/pkg/model/convenience.go

@@ -27,3 +27,8 @@ func s(v string) *string {
 func i64(v int64) *int64 {
 	return fi.Int64(v)
 }
+
+// b returns a pointer to a boolean
+func b(v bool) *bool {
+	return fi.Bool(v)
+}

nodeup/pkg/model/etcd.go

@@ -17,10 +17,11 @@ limitations under the License.
 package model
 
 import (
-	"github.com/golang/glog"
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+
+	"github.com/golang/glog"
 )
 
 // EtcdBuilder installs etcd
@@ -30,6 +31,7 @@ type EtcdBuilder struct {
 
 var _ fi.ModelBuilder = &EtcdBuilder{}
 
+// Build is responsible for creating the etcd user
 func (b *EtcdBuilder) Build(c *fi.ModelBuilderContext) error {
 	if !b.IsMaster {
 		return nil

nodeup/pkg/model/kubeapiserver.go

@@ -41,6 +41,7 @@ type KubeAPIServerBuilder struct {
 
 var _ fi.ModelBuilder = &KubeAPIServerBuilder{}
 
+// Build is responsible for generating the kubernetes api manifest
 func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 	if !b.IsMaster {
 		return nil
@@ -127,22 +128,29 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte
 			Type:     nodetasks.FileType_File,
 		}
 		c.AddTask(t)
+
 		return nil
-	} else {
-		return fmt.Errorf("Unrecognized authentication config %v", b.Cluster.Spec.Authentication)
 	}
+
+	return fmt.Errorf("Unrecognized authentication config %v", b.Cluster.Spec.Authentication)
 }
 
 func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 	kubeAPIServer := b.Cluster.Spec.KubeAPIServer
-
 	kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
 	kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.cert")
 	kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key")
-
 	kubeAPIServer.BasicAuthFile = filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv")
 	kubeAPIServer.TokenAuthFile = filepath.Join(b.PathSrvKubernetes(), "known_tokens.csv")
 
+	if b.Cluster.Spec.EnableEtcdTLS {
+		kubeAPIServer.EtcdCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
+		kubeAPIServer.EtcdCertFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client.pem")
+		kubeAPIServer.EtcdKeyFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client-key.pem")
+		kubeAPIServer.EtcdServers = []string{"https://127.0.0.1:4001"}
+		kubeAPIServer.EtcdServersOverrides = []string{"/events#https://127.0.0.1:4002"}
+	}
+
 	flags, err := flagbuilder.BuildFlags(b.Cluster.Spec.KubeAPIServer)
 	if err != nil {
 		return nil, fmt.Errorf("error building kube-apiserver flags: %v", err)

nodeup/pkg/model/kubectl.go

@@ -18,10 +18,12 @@ package model
 
 import (
 	"fmt"
-	"github.com/golang/glog"
+
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+
+	"github.com/golang/glog"
 )
 
 // KubectlBuilder install kubectl
@@ -31,13 +33,12 @@ type KubectlBuilder struct {
 
 var _ fi.ModelBuilder = &KubectlBuilder{}
 
+// Build is responsible for mananging the kubectl on the nodes
 func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error {
 	if !b.IsMaster {
-		// We don't have the configuration on the machines, so it only works on the master anyway
 		return nil
 	}
 
-	// Add kubectl file as an asset
 	{
 		// TODO: Extract to common function?
 		assetName := "kubectl"
@@ -59,7 +60,6 @@ func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(t)
 	}
 
-	// Add kubeconfig
 	{
 		kubeconfig, err := b.buildPKIKubeconfig("kubecfg")
 		if err != nil {

nodeup/pkg/model/kubelet.go

@@ -18,8 +18,7 @@ package model
 
 import (
 	"fmt"
-	"github.com/blang/semver"
-	"github.com/golang/glog"
+
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/pkg/apis/kops"
@@ -29,6 +28,9 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/upup/pkg/fi/utils"
+
+	"github.com/blang/semver"
+	"github.com/golang/glog"
 )
 
 // KubeletBuilder install kubelet
@@ -38,6 +40,7 @@ type KubeletBuilder struct {
 
 var _ fi.ModelBuilder = &KubeletBuilder{}
 
+// Build is responsible for generating the kubelet config
 func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
 	kubeletConfig, err := b.buildKubeletConfig()
 	if err != nil {
@@ -52,12 +55,11 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(t)
 	}
 
-	// Add kubelet file itself (as an asset)
 	{
-		// TODO: Extract to common function?
+		// @TODO Extract to common function?
 		assetName := "kubelet"
 		assetPath := ""
-		// TODO make Find call to an interface, we cannot mock out this function because it finds a file on disk
+		// @TODO make Find call to an interface, we cannot mock out this function because it finds a file on disk
 		asset, err := b.Assets.Find(assetName, assetPath)
 		if err != nil {
 			return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
@@ -75,10 +77,8 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(t)
 	}
 
-	// Add kubeconfig
 	{
-		// TODO: Change kubeconfig to be https
-
+		// @TODO Change kubeconfig to be https
 		kubeconfig, err := b.buildPKIKubeconfig("kubelet")
 		if err != nil {
 			return err
@@ -109,6 +109,7 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
 	return nil
 }
 
+// kubeletPath returns the path of the kubelet based on distro
 func (b *KubeletBuilder) kubeletPath() string {
 	kubeletCommand := "/usr/local/bin/kubelet"
 	if b.Distribution == distros.DistributionCoreOS {
@@ -120,6 +121,7 @@ func (b *KubeletBuilder) kubeletPath() string {
 	return kubeletCommand
 }
 
+// buildSystemdEnvironmentFile renders the environment file for the kubelet
 func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) {
 	// TODO: Dump the separate file for flags - just complexity!
 	flags, err := flagbuilder.BuildFlags(kubeletConfig)
@@ -155,6 +157,7 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet
 	return t, nil
 }
 
+// buildSystemdService is responsible for generating the kubelet systemd unit
 func (b *KubeletBuilder) buildSystemdService() *nodetasks.Service {
 	kubeletCommand := b.kubeletPath()
 
@@ -174,17 +177,19 @@ func (b *KubeletBuilder) buildSystemdService() *nodetasks.Service {
 	manifest.Set("Service", "RestartSec", "2s")
 	manifest.Set("Service", "StartLimitInterval", "0")
 	manifest.Set("Service", "KillMode", "process")
-
 	manifestString := manifest.Render()
+
 	glog.V(8).Infof("Built service manifest %q\n%s", "kubelet", manifestString)
 
 	service := &nodetasks.Service{
 		Name:       "kubelet.service",
 		Definition: s(manifestString),
 	}
 
-	// To avoid going in to backoff, we wait for protokube to start us
-	service.Running = fi.Bool(false)
+	// @check if we are a master allow protokube to start kubelet
+	if b.IsMaster {
+		service.Running = fi.Bool(false)
+	}
 
 	service.InitDefaults()
 
@@ -199,9 +204,9 @@ func (b *KubeletBuilder) buildKubeletConfig() (*kops.KubeletConfigSpec, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error building kubelet config: %v", err)
 	}
+
 	// TODO: Memoize if we reuse this
 	return kubeletConfigSpec, nil
-
 }
 
 func (b *KubeletBuilder) addStaticUtils(c *fi.ModelBuilderContext) error {