diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 90fa2c30ad..2cd8ca9d89 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -209,7 +209,7 @@ Below is a list of controller-manager options that acs-engine will configure by |"--route-reconciliation-period"|"10s"| -Below is a list of kubelet options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces kubelet configuration, or because a static configuration is required to build a functional cluster: +Below is a list of controller-manager options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces controller-manager configuration, or because a static configuration is required to build a functional cluster: |controller-manager option|default value| |---|---| @@ -228,7 +228,60 @@ Below is a list of kubelet options that are *not* currently user-configurable, e |"--profiling"|"false"| |"--use-service-account-credentials"|"false" ("true" if kubernetesConfig.enableRbac is true)| -We consider `kubeletConfig` and `controllerManagerConfig` to be generic conveniences that add power/flexibility to cluster deployments. Their usage comes with no operational guarantees! They are manual tuning features that enable low-level configuration of a kubernetes cluster. +#### apiServerConfig + +`apiServerConfig` declares runtime configuration for the kube-apiserver daemon running on all master nodes. Like `kubeletConfig` and `controllerManagerConfig` it is a generic key/value object, and a child property of `kubernetesConfig`. An example custom apiserver config: + +``` +"kubernetesConfig": { + "apiServerConfig": { + "--request-timeout": "30s" + } +} +``` + +See [here](https://kubernetes.io/docs/reference/generated/kube-apiserver/) for a reference of supported apiserver options. + +Below is a list of apiserver options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces kubelet configuration, or because a static configuration is required to build a functional cluster: + +|apiserver option|default value| +|---|---| +|"--admission-control"|"NamespaceLifecycle, LimitRanger, ServiceAccount, DefaultStorageClass, ResourceQuota, DenyEscalatingExec"| +|"--address"|"0.0.0.0"| +|"--advertise-address"|*calculated value that represents listening URI for API server*| +|"--allow-privileged"|"true"| +|"--insecure-port"|"8080"| +|"--secure-port"|"443"| +|"--etcd-cafile"|"/etc/kubernetes/certs/ca.crt"| +|"--etcd-certfile"|"/etc/kubernetes/certs/etcdclient.crt"| +|"--etcd-keyfile"|"/etc/kubernetes/certs/etcdclient.key"| +|"--etcd-servers"|*calculated value that represents etcd servers*| +|"--etcd-quorum-read"|"true"| +|"--tls-cert-file"|"/etc/kubernetes/certs/apiserver.crt"| +|"--tls-private-key-file"|"/etc/kubernetes/certs/apiserver.key"| +|"--client-ca-file"|"/etc/kubernetes/certs/ca.crt"| +|"--service-account-key-file"|"/etc/kubernetes/certs/apiserver.key"| +|"--kubelet-client-certificate"|"/etc/kubernetes/certs/client.crt"| +|"--kubelet-client-key"|"/etc/kubernetes/certs/client.key"| +|"--service-cluster-ip-range"|*see serviceCIDR*| +|"--storage-backend"|*calculated value that represents etcd version*| +|"--v"|"4"| +|"--authorization-mode"|"RBAC" (*if enabledRbac is true*)| +|"--experimental-encryption-provider-config"|"/etc/kubernetes/encryption-config.yaml" (*if enableDataEncryptionAtRest is true*)| +|"--requestheader-client-ca-file"|"/etc/kubernetes/certs/proxy-ca.crt" (*if enableAggregatedAPIs is true*)| +|"--proxy-client-cert-file"|"/etc/kubernetes/certs/proxy.crt" (*if enableAggregatedAPIs is true*)| +|"--proxy-client-key-file"|"/etc/kubernetes/certs/proxy.key" (*if enableAggregatedAPIs is true*)| +|"--requestheader-allowed-names"|"" (*if enableAggregatedAPIs is true*)| +|"--requestheader-extra-headers-prefix"|"X-Remote-Extra-" (*if enableAggregatedAPIs is true*)| +|"--requestheader-group-headers"|"X-Remote-Group" (*if enableAggregatedAPIs is true*)| +|"--requestheader-username-headers"|"X-Remote-User" (*if enableAggregatedAPIs is true*)| +|"--cloud-provider"|"azure" (*unless useCloudControllerManager is true*)| +|"--cloud-config"|"/etc/kubernetes/azure.json" (*unless useCloudControllerManager is true*)| +|"--oidc-username-claim"|"oid" (*if has AADProfile*)| +|"--oidc-client-id"|*calculated value that represents OID client ID* (*if has AADProfile*)| +|"--oidc-issuer-url"|*calculated value that represents OID issuer URL* (*if has AADProfile*)| + +We consider `kubeletConfig`, `controllerManagerConfig`, and `apiServerConfig` to be generic conveniences that add power/flexibility to cluster deployments. Their usage comes with no operational guarantees! They are manual tuning features that enable low-level configuration of a kubernetes cluster. ### masterProfile `masterProfile` describes the settings for master configuration. diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index ef59258d1f..76917a7922 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -180,25 +180,8 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER # SNAT outbound traffic from pods to destinations outside of VNET. iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsVariable "vnetCidr"}} -j MASQUERADE {{end}} - -{{ if .HasAadProfile }} - OIDC_CLIENT_ID=spn:{{WrapAsVariable "aadServerAppId"}} - VAR_AAD_TENANT_ID={{WrapAsVariable "aadTenantId"}} - VAR_TENANT_ID={{WrapAsVariable "tenantId"}} - VAR_TARGET_ENV={{WrapAsVariable "targetEnvironment"}} - AAD_TENANT_ID=${VAR_AAD_TENANT_ID:-$VAR_TENANT_ID} - AAD_ISSUER_HOST="sts.windows.net" - if [ "$VAR_TARGET_ENV" = "AzureChinaCloud" ]; then - AAD_ISSUER_HOST="sts.chinacloudapi.cn" - fi - - OIDC_ISSUER_URL="https://$AAD_ISSUER_HOST/$AAD_TENANT_ID/" - perl -pi -e "s|--oidc-client-id=\K(?=\")|$OIDC_CLIENT_ID| || s|--oidc-issuer-url=\K(?=\")|$OIDC_ISSUER_URL|" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{else}} - sed -i "/--oidc-client-id\|--oidc-issuer-url\|--oidc-username-claim/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{end}} sed -i "s||{{WrapAsVariable "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml" - sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s||{{WrapAsVariable "kubeServiceCidr"}}|g; s||{{WrapAsVariable "masterEtcdClientPort"}}|g; s||{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" + sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s||{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml" @@ -245,19 +228,9 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsVariable "kubernetesReschedulerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" {{end}} -{{if .OrchestratorProfile.KubernetesConfig.EnableRbac }} - # If RBAC enabled then add parameters to API server and Controller manager configuration - sed -i "s||--authorization-mode=RBAC|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{else}} - sed -i "//d" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{end}} - {{if EnableDataEncryptionAtRest }} ETCD_ENCRYPTION_SECRET="$(head -c 32 /dev/urandom | base64)" sed -i "s||$ETCD_ENCRYPTION_SECRET|g" "/etc/kubernetes/encryption-config.yaml" - sed -i "s||--experimental-encryption-provider-config=/etc/kubernetes/encryption-config.yaml|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{else}} - sed -i "//d" "/etc/kubernetes/manifests/kube-apiserver.yaml" {{end}} {{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}} @@ -265,27 +238,19 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml" {{end}} -{{if not .OrchestratorProfile.KubernetesConfig.EnableAggregatedAPIs}} - sed -i "/requestheader-client-ca-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/proxy-client-cert-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/proxy-client-key-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/requestheader-allowed-names/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/requestheader-extra-headers-prefix/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/requestheader-group-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/requestheader-username-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{end}} - sed -i "s||{{ .OrchestratorProfile.GetAPIServerEtcdAPIVersion }}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" - {{if UseCloudControllerManager }} sed -i "s||{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s||{{WrapAsVariable "masterFqdnPrefix"}}|g; s||{{WrapAsVariable "allocateNodeCidrs"}}|g; s||{{WrapAsVariable "kubeClusterCidr"}}|g; s||{{GetCloudControllerManagerRouteReconciliationPeriod .OrchestratorProfile.KubernetesConfig}}|g" \ /etc/kubernetes/manifests/cloud-controller-manager.yaml - - sed -i "/--\(cloud-config\|cloud-provider\|route-reconciliation-period\)=/d" \ - /etc/kubernetes/manifests/kube-controller-manager.yaml - sed -i "/--\(cloud-config\|cloud-provider\)=/d" \ - /etc/kubernetes/manifests/kube-apiserver.yaml {{end}} sed -i "s||{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" + sed -i "s||{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" + sed -i "s||{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" +{{ if .HasAadProfile }} + VAR_AAD_TENANT_ID={{WrapAsVariable "aadTenantId"}} + VAR_TENANT_ID={{WrapAsVariable "tenantId"}} + AAD_TENANT_ID=${VAR_AAD_TENANT_ID:-$VAR_TENANT_ID} + sed -i "/--oidc-issuer-url/s/$/$AAD_TENANT_ID/" "/etc/kubernetes/manifests/kube-apiserver.yaml" +{{end}} - path: "/opt/azure/containers/provision.sh" permissions: "0744" diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 1260fb151d..bed18bb84b 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -109,7 +109,6 @@ {{end}} "sshPublicKeyData": "[parameters('sshRSAPublicKey')]", {{if .HasAadProfile}} - "aadServerAppId": "[parameters('aadServerAppId')]", "aadTenantId": "[parameters('aadTenantId')]", {{end}} {{if not IsHostedMaster}} @@ -267,8 +266,8 @@ "[concat(variables('masterFirstAddrPrefix'), add(3, int(variables('masterFirstAddrOctet4'))))]", "[concat(variables('masterFirstAddrPrefix'), add(4, int(variables('masterFirstAddrOctet4'))))]" ], - "masterEtcdServerPort": 2380, - "masterEtcdClientPort": 2379, + "masterEtcdServerPort": {{GetMasterEtcdServerPort}}, + "masterEtcdClientPort": {{GetMasterEtcdClientPort}}, "masterEtcdPeerURLs":[ "[concat('https://', variables('masterPrivateIpAddrs')[0], ':', variables('masterEtcdServerPort'))]", "[concat('https://', variables('masterPrivateIpAddrs')[1], ':', variables('masterEtcdServerPort'))]", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 389ce1028f..fe693177d1 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -1,10 +1,4 @@ {{if .HasAadProfile}} - "aadServerAppId": { - "metadata": { - "description": "The server AAD application ID" - }, - "type": "string" - }, "aadTenantId": { "defaultValue": "", "metadata": { diff --git a/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml b/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml index 28a0a0833c..bef8e55df9 100644 --- a/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml +++ b/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml @@ -21,7 +21,6 @@ spec: - "--cloud-config=/etc/kubernetes/azure.json" - "--leader-elect=true" # TODO: RBAC support -# - "" - "--route-reconciliation-period=" - "--v=2" volumeMounts: diff --git a/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml b/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml index fe1763e794..8c48f7bd95 100644 --- a/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml +++ b/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml @@ -11,43 +11,8 @@ spec: containers: - name: "kube-apiserver" image: "" - command: - - "/hyperkube" - - "apiserver" - - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec" - - "--address=0.0.0.0" - - "--allow-privileged" - - "--insecure-port=8080" - - "--secure-port=443" - - "--cloud-provider=azure" - - "--cloud-config=/etc/kubernetes/azure.json" - - "--service-cluster-ip-range=" - - "--etcd-cafile=/etc/kubernetes/certs/ca.crt" - - "--etcd-certfile=/etc/kubernetes/certs/etcdclient.crt" - - "--etcd-keyfile=/etc/kubernetes/certs/etcdclient.key" - - "--etcd-servers=https://127.0.0.1:" - - "--etcd-quorum-read=true" - - "--advertise-address=" - - "--tls-cert-file=/etc/kubernetes/certs/apiserver.crt" - - "--tls-private-key-file=/etc/kubernetes/certs/apiserver.key" - - "--client-ca-file=/etc/kubernetes/certs/ca.crt" - - "--requestheader-client-ca-file=/etc/kubernetes/certs/proxy-ca.crt" - - "--proxy-client-cert-file=/etc/kubernetes/certs/proxy.crt" - - "--proxy-client-key-file=/etc/kubernetes/certs/proxy.key" - - "--service-account-key-file=/etc/kubernetes/certs/apiserver.key" - - "--kubelet-client-certificate=/etc/kubernetes/certs/client.crt" - - "--kubelet-client-key=/etc/kubernetes/certs/client.key" - - "--oidc-client-id=" - - "--oidc-issuer-url=" - - "--oidc-username-claim=oid" - - "--storage-backend=" - - "--v=4" - - "" - - "" - - "--requestheader-allowed-names=" - - "--requestheader-extra-headers-prefix=X-Remote-Extra-" - - "--requestheader-group-headers=X-Remote-Group" - - "--requestheader-username-headers=X-Remote-User" + command: ["/hyperkube", "apiserver"] + args: [] volumeMounts: - name: "etc-kubernetes" mountPath: "/etc/kubernetes" diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index fff9f4b97e..f9a7be62f6 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -105,6 +105,10 @@ const ( DefaultReschedulerAddonName = "rescheduler" // DefaultKubernetesKubeletMaxPods is the max pods per kubelet DefaultKubernetesKubeletMaxPods = 110 + // DefaultMasterEtcdServerPort is the default etcd server port for Kubernetes master nodes + DefaultMasterEtcdServerPort = 2380 + // DefaultMasterEtcdClientPort is the default etcd client port for Kubernetes master nodes + DefaultMasterEtcdClientPort = 2379 ) const ( diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go new file mode 100644 index 0000000000..f1d98b5579 --- /dev/null +++ b/pkg/acsengine/defaults-apiserver.go @@ -0,0 +1,107 @@ +package acsengine + +import ( + "strconv" + + "github.com/Azure/acs-engine/pkg/api" + "github.com/Azure/acs-engine/pkg/helpers" +) + +func setAPIServerConfig(cs *api.ContainerService) { + o := cs.Properties.OrchestratorProfile + staticLinuxAPIServerConfig := map[string]string{ + "--admission-control": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec", + "--address": "0.0.0.0", + "--advertise-address": "", + "--allow-privileged": "true", + "--insecure-port": "8080", + "--secure-port": "443", + "--etcd-cafile": "/etc/kubernetes/certs/ca.crt", + "--etcd-certfile": "/etc/kubernetes/certs/etcdclient.crt", + "--etcd-keyfile": "/etc/kubernetes/certs/etcdclient.key", + "--etcd-servers": "https://127.0.0.1:" + strconv.Itoa(DefaultMasterEtcdClientPort), + "--etcd-quorum-read": "true", + "--tls-cert-file": "/etc/kubernetes/certs/apiserver.crt", + "--tls-private-key-file": "/etc/kubernetes/certs/apiserver.key", + "--client-ca-file": "/etc/kubernetes/certs/ca.crt", + "--service-account-key-file": "/etc/kubernetes/certs/apiserver.key", + "--kubelet-client-certificate": "/etc/kubernetes/certs/client.crt", + "--kubelet-client-key": "/etc/kubernetes/certs/client.key", + "--service-cluster-ip-range": o.KubernetesConfig.ServiceCIDR, + "--storage-backend": o.GetAPIServerEtcdAPIVersion(), + "--v": "4", + } + + // RBAC configuration + if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableRbac) { + staticLinuxAPIServerConfig["--authorization-mode"] = "RBAC" + } + + // Data Encryption at REST configuration + if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableDataEncryptionAtRest) { + staticLinuxAPIServerConfig["--experimental-encryption-provider-config"] = "/etc/kubernetes/encryption-config.yaml" + } + + // Aggregated API configuration + if o.KubernetesConfig.EnableAggregatedAPIs { + staticLinuxAPIServerConfig["--requestheader-client-ca-file"] = "/etc/kubernetes/certs/proxy-ca.crt" + staticLinuxAPIServerConfig["--proxy-client-cert-file"] = "/etc/kubernetes/certs/proxy.crt" + staticLinuxAPIServerConfig["--proxy-client-key-file"] = "/etc/kubernetes/certs/proxy.key" + staticLinuxAPIServerConfig["--requestheader-allowed-names"] = "" + staticLinuxAPIServerConfig["--requestheader-extra-headers-prefix"] = "X-Remote-Extra-" + staticLinuxAPIServerConfig["--requestheader-group-headers"] = "X-Remote-Group" + staticLinuxAPIServerConfig["--requestheader-username-headers"] = "X-Remote-User" + } + + // Enable cloudprovider if we're not using cloud controller manager + if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { + staticLinuxAPIServerConfig["--cloud-provider"] = "azure" + staticLinuxAPIServerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" + } + + // AAD configuration + if cs.Properties.HasAadProfile() { + staticLinuxAPIServerConfig["--oidc-username-claim"] = "oid" + staticLinuxAPIServerConfig["--oidc-client-id"] = "spn:" + cs.Properties.AADProfile.ServerAppID + issuerHost := "sts.windows.net" + if GetCloudTargetEnv(cs.Location) == "AzureChinaCloud" { + issuerHost = "sts.chinacloudapi.cn" + } + staticLinuxAPIServerConfig["--oidc-issuer-url"] = "https://" + issuerHost + "/" + } + + staticWindowsAPIServerConfig := make(map[string]string) + for key, val := range staticLinuxAPIServerConfig { + staticWindowsAPIServerConfig[key] = val + } + // Windows apiserver config overrides + // TODO placeholder for specific config overrides for Windows clusters + + // Default apiserver config + defaultAPIServerConfig := map[string]string{} + + // If no user-configurable apiserver config values exists, use the defaults + if o.KubernetesConfig.APIServerConfig == nil { + o.KubernetesConfig.APIServerConfig = defaultAPIServerConfig + } else { + for key, val := range defaultAPIServerConfig { + // If we don't have a user-configurable apiserver config for each option + if _, ok := o.KubernetesConfig.APIServerConfig[key]; !ok { + // then assign the default value + o.KubernetesConfig.APIServerConfig[key] = val + } + } + } + + // We don't support user-configurable values for the following, + // so any of the value assignments below will override user-provided values + var overrideAPIServerConfig map[string]string + if cs.Properties.HasWindows() { + overrideAPIServerConfig = staticWindowsAPIServerConfig + } else { + overrideAPIServerConfig = staticLinuxAPIServerConfig + } + for key, val := range overrideAPIServerConfig { + o.KubernetesConfig.APIServerConfig[key] = val + } +} diff --git a/pkg/acsengine/defaults-controller-manager.go b/pkg/acsengine/defaults-controller-manager.go index 7b731ad070..e16d682676 100644 --- a/pkg/acsengine/defaults-controller-manager.go +++ b/pkg/acsengine/defaults-controller-manager.go @@ -4,6 +4,7 @@ import ( "strconv" "github.com/Azure/acs-engine/pkg/api" + "github.com/Azure/acs-engine/pkg/helpers" ) func setControllerManagerConfig(cs *api.ContainerService) { @@ -12,8 +13,6 @@ func setControllerManagerConfig(cs *api.ContainerService) { "--kubeconfig": "/var/lib/kubelet/kubeconfig", "--allocate-node-cidrs": strconv.FormatBool(!o.IsAzureCNI()), "--cluster-cidr": o.KubernetesConfig.ClusterSubnet, - "--cloud-provider": "azure", - "--cloud-config": "/etc/kubernetes/azure.json", "--root-ca-file": "/etc/kubernetes/certs/ca.crt", "--cluster-signing-cert-file": "/etc/kubernetes/certs/ca.crt", "--cluster-signing-key-file": "/etc/kubernetes/certs/ca.key", @@ -30,6 +29,12 @@ func setControllerManagerConfig(cs *api.ContainerService) { staticLinuxControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix } + // Enable cloudprovider if we're not using cloud controller manager + if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { + staticLinuxControllerManagerConfig["--cloud-provider"] = "azure" + staticLinuxControllerManagerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" + } + staticWindowsControllerManagerConfig := make(map[string]string) for key, val := range staticLinuxControllerManagerConfig { staticWindowsControllerManagerConfig[key] = val diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 27a1b0846a..c06b7ea13b 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -403,6 +403,8 @@ func setOrchestratorDefaults(cs *api.ContainerService) { setKubeletConfig(cs) // Configure controller-manager setControllerManagerConfig(cs) + // Configure apiserver + setAPIServerConfig(cs) } else if o.OrchestratorType == api.DCOS { if o.DcosConfig == nil { diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 3dfd763626..af24f30573 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -638,7 +638,6 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s if properties.AADProfile != nil { addValue(parametersMap, "aadTenantId", properties.AADProfile.TenantID) - addValue(parametersMap, "aadServerAppId", properties.AADProfile.ServerAppID) } } @@ -873,6 +872,20 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat } return strings.TrimSuffix(buf.String(), ", ") }, + "GetAPIServerConfigKeyVals": func(kc *api.KubernetesConfig) string { + apiServerConfig := kc.APIServerConfig + // Order by key for consistency + keys := []string{} + for key := range apiServerConfig { + keys = append(keys, key) + } + sort.Strings(keys) + var buf bytes.Buffer + for _, key := range keys { + buf.WriteString(fmt.Sprintf("\\\"%s=%s\\\", ", key, apiServerConfig[key])) + } + return strings.TrimSuffix(buf.String(), ", ") + }, // temporary until we genericise cloud controller manager config "GetCloudControllerManagerRouteReconciliationPeriod": func(kc *api.KubernetesConfig) string { controllerManagerConfig := cs.Properties.OrchestratorProfile.KubernetesConfig.ControllerManagerConfig @@ -1222,6 +1235,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat cloudSpecConfig := GetCloudSpecConfig(cs.Location) return fmt.Sprintf("\"%s\"", cloudSpecConfig.OSImageConfig[profile.Distro].ImageVersion) }, + "GetMasterEtcdServerPort": func() int { + return DefaultMasterEtcdServerPort + }, + "GetMasterEtcdClientPort": func() int { + return DefaultMasterEtcdClientPort + }, "PopulateClassicModeDefaultValue": func(attr string) string { var val string if !t.ClassicMode { diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 1f4a4fc7f9..b5c9d1a3a0 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -674,6 +674,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet convertAddonsToVlabs(api, vlabs) convertKubeletConfigToVlabs(api, vlabs) convertControllerManagerConfigToVlabs(api, vlabs) + convertAPIServerConfigToVlabs(api, vlabs) } func convertKubeletConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) { @@ -690,6 +691,13 @@ func convertControllerManagerConfigToVlabs(a *KubernetesConfig, v *vlabs.Kuberne } } +func convertAPIServerConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) { + v.APIServerConfig = map[string]string{} + for key, val := range a.APIServerConfig { + v.APIServerConfig[key] = val + } +} + func convertAddonsToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) { v.Addons = []vlabs.KubernetesAddon{} for i := range a.Addons { diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 8870524085..7418c627db 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -618,6 +618,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes convertAddonsToAPI(vlabs, api) convertKubeletConfigToAPI(vlabs, api) convertControllerManagerConfigToAPI(vlabs, api) + convertAPIServerConfigToAPI(vlabs, api) } func setVlabsKubernetesDefaults(vp *vlabs.Properties, api *OrchestratorProfile) { @@ -674,6 +675,13 @@ func convertControllerManagerConfigToAPI(v *vlabs.KubernetesConfig, a *Kubernete } } +func convertAPIServerConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) { + a.APIServerConfig = map[string]string{} + for key, val := range v.APIServerConfig { + a.APIServerConfig[key] = val + } +} + func convertV20160930MasterProfile(v20160930 *v20160930.MasterProfile, api *MasterProfile) { api.Count = v20160930.Count api.DNSPrefix = v20160930.DNSPrefix diff --git a/pkg/api/types.go b/pkg/api/types.go index 229e9f545e..ad6e7be9ac 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -228,6 +228,7 @@ type KubernetesConfig struct { Addons []KubernetesAddon `json:"addons,omitempty"` KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` } // DcosConfig Configuration for DC/OS diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index efceb8c6ef..fa9aae6194 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -246,6 +246,7 @@ type KubernetesConfig struct { Addons []KubernetesAddon `json:"addons,omitempty"` KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` } // DcosConfig Configuration for DC/OS