Add Enable Pod Security Option (#2048)

* Add PodSecurityPolicy

* use helpers.IsTrueBoolPointer, delete EnablePodSecurityPolicy function and update defaultAPIServerConfig

parts/k8s/kubernetesmastercustomscript.sh

@@ -607,6 +607,16 @@ function ensureEtcdDataDir() {
    exit 4
 }
 
+function ensurePodSecurityPolicy(){
+    if $REBOOTREQUIRED; then
+        return
+    fi
+    POD_SECURITY_POLICY_FILE="/etc/kubernetes/manifests/pod-security-policy.yaml"
+    if [ -f $POD_SECURITY_POLICY_FILE ]; then
+        kubectl create -f $POD_SECURITY_POLICY_FILE
+    fi
+}
+
 function writeKubeConfig() {
     KUBECONFIGDIR=/home/$ADMINUSER/.kube
     KUBECONFIGFILE=$KUBECONFIGDIR/config
@@ -695,6 +705,7 @@ if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then
     ensureEtcdDataDir
     ensureEtcd
     ensureApiserver
+    ensurePodSecurityPolicy
 fi
 
 if [[ $OS == $UBUNTU_OS_NAME ]]; then

parts/k8s/manifests/kubernetesmaster-pod-security-policy.yaml

@@ -0,0 +1,128 @@
+apiVersion: extensions/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: privileged
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: "*"
+spec:
+  privileged: true
+  allowPrivilegeEscalation: true
+  allowedCapabilities:
+  - "*"
+  volumes:
+  - "*"
+  hostNetwork: true
+  hostPorts:
+  - min: 0
+    max: 65535
+  hostIPC: true
+  hostPID: true
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+---
+apiVersion: extensions/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: restricted
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'docker/default'
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
+spec:
+  privileged: false
+  # Required to prevent escalations to root.
+  allowPrivilegeEscalation: false
+  # This is redundant with non-root + disallow privilege escalation,
+  # but we can provide it for defense in depth.
+  requiredDropCapabilities:
+    - ALL
+  # Allow core volume types.
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    # Assume that persistentVolumes set up by the cluster admin are safe to use.
+    - 'persistentVolumeClaim'
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    # Require the container to run without root privileges.
+    rule: 'MustRunAsNonRoot'
+  seLinux:
+    # This policy assumes the nodes are using AppArmor rather than SELinux.
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      # Forbid adding the root group.
+      - min: 1
+        max: 65535
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      # Forbid adding the root group.
+      - min: 1
+        max: 65535
+  readOnlyRootFilesystem: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: psp:privileged
+rules:
+- apiGroups: ['extensions']
+  resources: ['podsecuritypolicies']
+  verbs:     ['use']
+  resourceNames:
+  - privileged
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: psp:restricted
+rules:
+- apiGroups: ['extensions']
+  resources: ['podsecuritypolicies']
+  verbs:     ['use']
+  resourceNames:
+  - restricted
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: default:restricted
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:restricted
+subjects:
+- kind: Group
+  name: system:authenticated
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: default:privileged
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:privileged
+subjects:
+- kind: Group
+  name: system:masters
+  apiGroup: rbac.authorization.k8s.io
+- kind: Group
+  name: system:serviceaccounts:kube-system
+  apiGroup: rbac.authorization.k8s.io

pkg/acsengine/addons.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/Azure/acs-engine/pkg/api"
+	"github.com/Azure/acs-engine/pkg/helpers"
 )
 
 type kubernetesFeatureSetting struct {
@@ -85,6 +86,11 @@ func kubernetesManifestSettingsInit(profile *api.Properties) []kubernetesFeature
 			"cloud-controller-manager.yaml",
 			profile.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager != nil && *profile.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager,
 		},
+		{
+			"kubernetesmaster-pod-security-policy.yaml",
+			"pod-security-policy.yaml",
+			helpers.IsTrueBoolPointer(profile.OrchestratorProfile.KubernetesConfig.EnablePodSecurityPolicy),
+		},
 		{
 			"kubernetesmaster-kube-apiserver.yaml",
 			"kube-apiserver.yaml",

pkg/acsengine/const.go

@@ -58,6 +58,8 @@ const (
 	DefaultKubernetesCtrlMgrRouteReconciliationPeriod = "10s"
 	// DefaultKubernetesCtrlMgrTerminatedPodGcThreshold is set to 5000, see --terminated-pod-gc-threshold at https://kubernetes.io/docs/admin/kube-controller-manager/ and https://github.com/kubernetes/kubernetes/issues/22680
 	DefaultKubernetesCtrlMgrTerminatedPodGcThreshold = "5000"
+	// DefaultKubernetesCtrlMgrUseSvcAccountCreds is "true", see --use-service-account-credentials at https://kubernetes.io/docs/admin/kube-controller-manager/
+	DefaultKubernetesCtrlMgrUseSvcAccountCreds = "true"
 	// DefaultKubernetesCloudProviderBackoff is false to disable cloudprovider backoff implementation for API calls
 	DefaultKubernetesCloudProviderBackoff = false
 	// DefaultKubernetesCloudProviderBackoffRetries is 6, takes effect if DefaultKubernetesCloudProviderBackoff is true

pkg/acsengine/defaults-apiserver.go

@@ -99,6 +99,11 @@ func setAPIServerConfig(cs *api.ContainerService) {
 		}
 	}
 
+	// Pod Security Policy configuration
+	if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnablePodSecurityPolicy) {
+		defaultAPIServerConfig["--admission-control"] = defaultAPIServerConfig["--admission-control"] + ",PodSecurityPolicy"
+	}
+
 	// If no user-configurable apiserver config values exists, use the defaults
 	if o.KubernetesConfig.APIServerConfig == nil {
 		o.KubernetesConfig.APIServerConfig = defaultAPIServerConfig

pkg/acsengine/defaults-controller-manager.go

@@ -44,10 +44,11 @@ func setControllerManagerConfig(cs *api.ContainerService) {
 
 	// Default controller-manager config
 	defaultControllerManagerConfig := map[string]string{
-		"--node-monitor-grace-period":   DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
-		"--pod-eviction-timeout":        DefaultKubernetesCtrlMgrPodEvictionTimeout,
-		"--route-reconciliation-period": DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
-		"--terminated-pod-gc-threshold": DefaultKubernetesCtrlMgrTerminatedPodGcThreshold,
+		"--node-monitor-grace-period":       DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
+		"--pod-eviction-timeout":            DefaultKubernetesCtrlMgrPodEvictionTimeout,
+		"--route-reconciliation-period":     DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
+		"--terminated-pod-gc-threshold":     DefaultKubernetesCtrlMgrTerminatedPodGcThreshold,
+		"--use-service-account-credentials": DefaultKubernetesCtrlMgrUseSvcAccountCreds,
 	}
 
 	// If no user-configurable controller-manager config values exists, use the defaults

pkg/api/converterfromapi.go

@@ -669,6 +669,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
 	vlabs.EnableSecureKubelet = api.EnableSecureKubelet
 	vlabs.EnableAggregatedAPIs = api.EnableAggregatedAPIs
 	vlabs.EnableDataEncryptionAtRest = api.EnableDataEncryptionAtRest
+	vlabs.EnablePodSecurityPolicy = api.EnablePodSecurityPolicy
 	vlabs.GCHighThreshold = api.GCHighThreshold
 	vlabs.GCLowThreshold = api.GCLowThreshold
 	vlabs.EtcdVersion = api.EtcdVersion

pkg/api/convertertoapi.go

@@ -614,6 +614,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	api.EnableSecureKubelet = vlabs.EnableSecureKubelet
 	api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs
 	api.EnableDataEncryptionAtRest = vlabs.EnableDataEncryptionAtRest
+	api.EnablePodSecurityPolicy = vlabs.EnablePodSecurityPolicy
 	api.GCHighThreshold = vlabs.GCHighThreshold
 	api.GCLowThreshold = vlabs.GCLowThreshold
 	api.EtcdVersion = vlabs.EtcdVersion

pkg/api/types.go

@@ -243,6 +243,7 @@ type KubernetesConfig struct {
 	EtcdVersion                  string            `json:"etcdVersion,omitempty"`
 	EtcdDiskSizeGB               string            `json:"etcdDiskSizeGB,omitempty"`
 	EnableDataEncryptionAtRest   *bool             `json:"enableDataEncryptionAtRest,omitempty"`
+	EnablePodSecurityPolicy      *bool             `json:"enablePodSecurityPolicy,omitempty"`
 	Addons                       []KubernetesAddon `json:"addons,omitempty"`
 	KubeletConfig                map[string]string `json:"kubeletConfig,omitempty"`
 	ControllerManagerConfig      map[string]string `json:"controllerManagerConfig,omitempty"`

pkg/api/vlabs/types.go

@@ -251,6 +251,7 @@ type KubernetesConfig struct {
 	EtcdVersion                  string            `json:"etcdVersion,omitempty"`
 	EtcdDiskSizeGB               string            `json:"etcdDiskSizeGB,omitempty"`
 	EnableDataEncryptionAtRest   *bool             `json:"enableDataEncryptionAtRest,omitempty"`
+	EnablePodSecurityPolicy      *bool             `json:"enablePodSecurityPolicy,omitempty"`
 	Addons                       []KubernetesAddon `json:"addons,omitempty"`
 	KubeletConfig                map[string]string `json:"kubeletConfig,omitempty"`
 	ControllerManagerConfig      map[string]string `json:"controllerManagerConfig,omitempty"`

pkg/api/vlabs/validate.go

@@ -111,6 +111,10 @@ func (o *OrchestratorProfile) Validate(isUpdate bool) error {
 						}
 					}
 				}
+				if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnablePodSecurityPolicy) &&
+					!helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableRbac) {
+					return fmt.Errorf("enablePodSecurityPolicy requires the enableRbac feature as a prerequisite")
+				}
 			}
 
 		default: