apiserver.go

/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package components

import (
	"fmt"
	"strings"

	"k8s.io/api/core/v1"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/loader"

	"github.com/blang/semver"
)

// KubeAPIServerOptionsBuilder adds options for the apiserver to the model
type KubeAPIServerOptionsBuilder struct {
	*OptionsContext
}

var _ loader.OptionsBuilder = &KubeAPIServerOptionsBuilder{}

// BuildOptions is resposible for filling in the default settings for the kube apiserver
func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error {
	clusterSpec := o.(*kops.ClusterSpec)
	if clusterSpec.KubeAPIServer == nil {
		clusterSpec.KubeAPIServer = &kops.KubeAPIServerConfig{}
	}
	c := clusterSpec.KubeAPIServer

	if c.APIServerCount == nil {
		count := b.buildAPIServerCount(clusterSpec)
		if count == 0 {
			return fmt.Errorf("no instance groups found")
		}
		c.APIServerCount = fi.Int32(int32(count))
	}

	// @question: should the question every be able to set this?
	if c.StorageBackend == nil {
		// @note: we can use the first version as we enforce both running the same versions.
		// albeit feels a little weird to do this
		sem, err := semver.Parse(strings.TrimPrefix(clusterSpec.EtcdClusters[0].Version, "v"))
		if err != nil {
			return err
		}
		c.StorageBackend = fi.String(fmt.Sprintf("etcd%d", sem.Major))
	}

	if c.KubeletPreferredAddressTypes == nil {
		if b.IsKubernetesGTE("1.5") {
			// We prioritize the internal IP above the hostname
			c.KubeletPreferredAddressTypes = []string{
				string(v1.NodeInternalIP),
				string(v1.NodeHostName),
				string(v1.NodeExternalIP),
			}

			if b.IsKubernetesLT("1.7") {
				// NodeLegacyHostIP was removed in 1.7; we add it to prior versions with lowest precedence
				c.KubeletPreferredAddressTypes = append(c.KubeletPreferredAddressTypes, "LegacyHostIP")
			}
		}
	}

	if clusterSpec.Authentication != nil {
		if clusterSpec.Authentication.Kopeio != nil {
			c.AuthenticationTokenWebhookConfigFile = fi.String("/etc/kubernetes/authn.config")
		}
	}

	if clusterSpec.Authorization == nil || clusterSpec.Authorization.IsEmpty() {
		// Do nothing - use the default as defined by the apiserver
		// (this won't happen anyway because of our default logic)
	} else if clusterSpec.Authorization.AlwaysAllow != nil {
		clusterSpec.KubeAPIServer.AuthorizationMode = fi.String("AlwaysAllow")
	} else if clusterSpec.Authorization.RBAC != nil {
		var modes []string

		if b.IsKubernetesGTE("1.10") {
			if fi.BoolValue(clusterSpec.KubeAPIServer.EnableBootstrapAuthToken) {
				// Enable the Node authorizer, used for special per-node RBAC policies
				modes = append(modes, "Node")
			}
		}
		modes = append(modes, "RBAC")

		clusterSpec.KubeAPIServer.AuthorizationMode = fi.String(strings.Join(modes, ","))
	}

	if clusterSpec.KubeAPIServer.EtcdQuorumRead == nil {
		if b.IsKubernetesGTE("1.9") {
			// 1.9 changed etcd-quorum-reads default to true
			// There's a balance between some bugs which are attributed to not having etcd-quorum-reads,
			// and the poor implementation of quorum-reads in etcd2.

			etcdHA := false
			etcdV2 := true
			for _, c := range clusterSpec.EtcdClusters {
				if len(c.Members) > 1 {
					etcdHA = true
				}
				if c.Version != "" && !strings.HasPrefix(c.Version, "2.") {
					etcdV2 = false
				}
			}

			if !etcdV2 {
				// etcd3 quorum reads are cheap.  Stick with default (which is to enable quorum reads)
				clusterSpec.KubeAPIServer.EtcdQuorumRead = nil
			} else {
				// etcd2 quorum reads go through raft => write to disk => expensive
				if !etcdHA {
					// Turn off quorum reads - they still go through raft, but don't serve any purpose in non-HA clusters.
					clusterSpec.KubeAPIServer.EtcdQuorumRead = fi.Bool(false)
				} else {
					// The problematic case.  We risk exposing more bugs, but against that we have to balance performance.
					// For now we turn off quorum reads - it's a bad enough performance regression
					// We'll likely make this default to true once we can set IOPS on the etcd volume and can easily upgrade to etcd3
					clusterSpec.KubeAPIServer.EtcdQuorumRead = fi.Bool(false)
				}
			}
		}
	}

	if err := b.configureAggregation(clusterSpec); err != nil {
		return nil
	}

	image, err := Image("kube-apiserver", clusterSpec, b.AssetBuilder)
	if err != nil {
		return err
	}
	c.Image = image

	switch kops.CloudProviderID(clusterSpec.CloudProvider) {
	case kops.CloudProviderAWS:
		c.CloudProvider = "aws"
	case kops.CloudProviderGCE:
		c.CloudProvider = "gce"
	case kops.CloudProviderDO:
		c.CloudProvider = "external"
	case kops.CloudProviderVSphere:
		c.CloudProvider = "vsphere"
	case kops.CloudProviderBareMetal:
		// for baremetal, we don't specify a cloudprovider to apiserver
	case kops.CloudProviderOpenstack:
		c.CloudProvider = "openstack"
	default:
		return fmt.Errorf("unknown cloudprovider %q", clusterSpec.CloudProvider)
	}

	if clusterSpec.ExternalCloudControllerManager != nil {
		c.CloudProvider = "external"
	}

	c.LogLevel = 2
	c.SecurePort = 443

	if b.IsKubernetesGTE("1.10") {
		c.BindAddress = "0.0.0.0"
		c.InsecureBindAddress = "127.0.0.1"
	} else {
		c.Address = "127.0.0.1"
	}

	c.AllowPrivileged = fi.Bool(true)
	c.ServiceClusterIPRange = clusterSpec.ServiceClusterIPRange
	c.EtcdServers = []string{"http://127.0.0.1:4001"}
	c.EtcdServersOverrides = []string{"/events#http://127.0.0.1:4002"}

	// TODO: We can probably rewrite these more clearly in descending order
	if b.IsKubernetesGTE("1.3") && b.IsKubernetesLT("1.4") {
		c.AdmissionControl = []string{
			"NamespaceLifecycle",
			"LimitRanger",
			"ServiceAccount",
			"PersistentVolumeLabel",
			"ResourceQuota",
		}
	}
	if b.IsKubernetesGTE("1.4") && b.IsKubernetesLT("1.5") {
		c.AdmissionControl = []string{
			"NamespaceLifecycle",
			"LimitRanger",
			"ServiceAccount",
			"PersistentVolumeLabel",
			"DefaultStorageClass",
			"ResourceQuota",
		}
	}
	if b.IsKubernetesGTE("1.5") && b.IsKubernetesLT("1.6") {
		c.AdmissionControl = []string{
			"NamespaceLifecycle",
			"LimitRanger",
			"ServiceAccount",
			"PersistentVolumeLabel",
			"DefaultStorageClass",
			"ResourceQuota",
		}
	}
	if b.IsKubernetesGTE("1.6") && b.IsKubernetesLT("1.7") {
		c.AdmissionControl = []string{
			"NamespaceLifecycle",
			"LimitRanger",
			"ServiceAccount",
			"PersistentVolumeLabel",
			"DefaultStorageClass",
			"DefaultTolerationSeconds",
			"ResourceQuota",
		}
	}
	if b.IsKubernetesGTE("1.7") && b.IsKubernetesLT("1.9") {
		c.AdmissionControl = []string{
			"Initializers",
			"NamespaceLifecycle",
			"LimitRanger",
			"ServiceAccount",
			"PersistentVolumeLabel",
			"DefaultStorageClass",
			"DefaultTolerationSeconds",
			"NodeRestriction",
			"ResourceQuota",
		}
	}
	if b.IsKubernetesGTE("1.9") && b.IsKubernetesLT("1.10") {
		c.AdmissionControl = []string{
			"Initializers",
			"NamespaceLifecycle",
			"LimitRanger",
			"ServiceAccount",
			"PersistentVolumeLabel",
			"DefaultStorageClass",
			"DefaultTolerationSeconds",
			"MutatingAdmissionWebhook",
			"ValidatingAdmissionWebhook",
			"NodeRestriction",
			"ResourceQuota",
		}
	}
	// Based on recommendations from:
	// https://kubernetes.io/docs/admin/admission-controllers/#is-there-a-recommended-set-of-admission-controllers-to-use
	if b.IsKubernetesGTE("1.10") {
		c.EnableAdmissionPlugins = []string{
			"Initializers",
			"NamespaceLifecycle",
			"LimitRanger",
			"ServiceAccount",
			"PersistentVolumeLabel",
			"DefaultStorageClass",
			"DefaultTolerationSeconds",
			"MutatingAdmissionWebhook",
			"ValidatingAdmissionWebhook",
			"NodeRestriction",
			"ResourceQuota",
		}
	}

	// We make sure to disable AnonymousAuth from when it was introduced
	if b.IsKubernetesGTE("1.5") {
		c.AnonymousAuth = fi.Bool(false)
	}

	// FIXME : Disable the insecure port when kubernetes issue #43784 is fixed
	c.InsecurePort = 8080

	return nil
}

// buildAPIServerCount calculates the count of the api servers, essentuially the number of node marked as Master role
func (b *KubeAPIServerOptionsBuilder) buildAPIServerCount(clusterSpec *kops.ClusterSpec) int {
	// The --apiserver-count flag is (generally agreed) to be something we need to get rid of in k8s

	// We should do something like this:

	//count := 0
	//for _, ig := range b.InstanceGroups {
	//	if !ig.IsMaster() {
	//		continue
	//	}
	//	size := fi.IntValue(ig.Spec.MaxSize)
	//	if size == 0 {
	//		size = fi.IntValue(ig.Spec.MinSize)
	//	}
	//	count += size
	//}

	// But if we do, we end up with a weird dependency on InstanceGroups.  We actually could tolerate
	// that in kops, but we don't really want to.

	// So instead, we assume that the etcd cluster size is the API Server Count.
	// We can re-examine this when we allow separate etcd clusters - at which time hopefully
	// the flag won't exist

	counts := make(map[string]int)
	for _, etcdCluster := range clusterSpec.EtcdClusters {
		counts[etcdCluster.Name] = len(etcdCluster.Members)
	}

	count := counts["main"]

	return count
}

// configureAggregation sets up the aggregation options
func (b *KubeAPIServerOptionsBuilder) configureAggregation(clusterSpec *kops.ClusterSpec) error {
	if b.IsKubernetesGTE("1.7") {
		clusterSpec.KubeAPIServer.RequestheaderAllowedNames = []string{"aggregator"}
		clusterSpec.KubeAPIServer.RequestheaderExtraHeaderPrefixes = []string{"X-Remote-Extra-"}
		clusterSpec.KubeAPIServer.RequestheaderGroupHeaders = []string{"X-Remote-Group"}
		clusterSpec.KubeAPIServer.RequestheaderUsernameHeaders = []string{"X-Remote-User"}
	}

	return nil
}