From e19a6f1ea398e362fd7cd3901b12b4ac5147585e Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Jan 2018 12:57:48 -0800 Subject: [PATCH 01/10] generic apiserver config --- docs/clusterdefinition.md | 57 +++++++++- parts/k8s/kubernetesmastercustomdata.yml | 40 +------ parts/k8s/kubernetesmastervars.t | 4 +- ...rnetesmaster-cloud-controller-manager.yaml | 1 - .../kubernetesmaster-kube-apiserver.yaml | 39 +------ pkg/acsengine/const.go | 4 + pkg/acsengine/defaults-apiserver.go | 102 ++++++++++++++++++ pkg/acsengine/defaults.go | 2 + pkg/acsengine/engine.go | 16 +++ pkg/api/converterfromapi.go | 8 ++ pkg/api/convertertoapi.go | 8 ++ pkg/api/types.go | 1 + pkg/api/vlabs/types.go | 1 + 13 files changed, 204 insertions(+), 79 deletions(-) create mode 100644 pkg/acsengine/defaults-apiserver.go diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 90fa2c30ad..5b4704076e 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -209,7 +209,7 @@ Below is a list of controller-manager options that acs-engine will configure by |"--route-reconciliation-period"|"10s"| -Below is a list of kubelet options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces kubelet configuration, or because a static configuration is required to build a functional cluster: +Below is a list of controller-manager options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces controller-manager configuration, or because a static configuration is required to build a functional cluster: |controller-manager option|default value| |---|---| @@ -228,7 +228,60 @@ Below is a list of kubelet options that are *not* currently user-configurable, e |"--profiling"|"false"| |"--use-service-account-credentials"|"false" ("true" if kubernetesConfig.enableRbac is true)| -We consider `kubeletConfig` and `controllerManagerConfig` to be generic conveniences that add power/flexibility to cluster deployments. Their usage comes with no operational guarantees! They are manual tuning features that enable low-level configuration of a kubernetes cluster. +#### apiServerConfig + +`apiServerConfig` declares runtime configuration for the kube-apiserver daemon running on all master nodes. Like `kubeletConfig` and `controllerManagerConfig` it is a generic key/value object, and a child property of `kubernetesConfig`. An example custom apiserver config: + +``` +"kubernetesConfig": { + "apiServerConfig": { + "--request-timeout": "30s" + } +} +``` + +See [here](https://kubernetes.io/docs/reference/generated/kube-apiserver/) for a reference of supported apiserver options. + +Below is a list of apiserver options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces kubelet configuration, or because a static configuration is required to build a functional cluster: + +|apiserver option|default value| +|---|---| +|"--admission-control"|"NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec"| +|"--address"|"0.0.0.0"| +|"--advertise-address"|*calculated value that represents listening URI for API server*| +|"--allow-privileged"|"true"| +|"--insecure-port"|"8080"| +|"--secure-port"|"443"| +|"--etcd-cafile"|"/etc/kubernetes/certs/ca.crt"| +|"--etcd-certfile"|"/etc/kubernetes/certs/etcdclient.crt"| +|"--etcd-keyfile"|"/etc/kubernetes/certs/etcdclient.key"| +|"--etcd-servers"|*calculated value that represents etcd servers*| +|"--etcd-quorum-read"|"true"| +|"--tls-cert-file"|"/etc/kubernetes/certs/apiserver.crt"| +|"--tls-private-key-file"|"/etc/kubernetes/certs/apiserver.key"| +|"--client-ca-file"|"/etc/kubernetes/certs/ca.crt"| +|"--service-account-key-file"|"/etc/kubernetes/certs/apiserver.key"| +|"--kubelet-client-certificate"|"/etc/kubernetes/certs/client.crt"| +|"--kubelet-client-key"|"/etc/kubernetes/certs/client.key"| +|"--service-cluster-ip-range"|*see serviceCIDR*| +|"--storage-backend"|*calculated value that represents etcd version*| +|"--v"|"4"| +|"--authorization-mode"|"RBAC" (*if enabledRbac is true*)| +|"--experimental-encryption-provider-config"|"/etc/kubernetes/encryption-config.yaml" (*if enableDataEncryptionAtRest is true*)| +|"--requestheader-client-ca-file"|"/etc/kubernetes/certs/proxy-ca.crt" (*if enableAggregatedAPIs is true*)| +|"--proxy-client-cert-file"|"/etc/kubernetes/certs/proxy.crt" (*if enableAggregatedAPIs is true*)| +|"--proxy-client-key-file"|"/etc/kubernetes/certs/proxy.key" (*if enableAggregatedAPIs is true*)| +|"--requestheader-allowed-names"|"" (*if enableAggregatedAPIs is true*)| +|"--requestheader-extra-headers-prefix"|"X-Remote-Extra-" (*if enableAggregatedAPIs is true*)| +|"--requestheader-group-headers"|"X-Remote-Group" (*if enableAggregatedAPIs is true*)| +|"--requestheader-username-headers"|"X-Remote-User" (*if enableAggregatedAPIs is true*)| +|"--cloud-provider"|"azure" (*unless useCloudControllerManager is true*)| +|"--cloud-config"|"/etc/kubernetes/azure.json" (*unless useCloudControllerManager is true*)| +|"--oidc-username-claim"|"oid" (*if has AADProfile*)| +|"--oidc-client-id"|*calculated value that represents OID client ID* (*if has AADProfile*)| +|"--oidc-issuer-url"|*calculated value that represents OID issuer URL* (*if has AADProfile*)| + +We consider `kubeletConfig`, `controllerManagerConfig`, and `apiServerConfig` to be generic conveniences that add power/flexibility to cluster deployments. Their usage comes with no operational guarantees! They are manual tuning features that enable low-level configuration of a kubernetes cluster. ### masterProfile `masterProfile` describes the settings for master configuration. diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index ef59258d1f..ec41b2e6d5 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -182,23 +182,11 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{end}} {{ if .HasAadProfile }} - OIDC_CLIENT_ID=spn:{{WrapAsVariable "aadServerAppId"}} - VAR_AAD_TENANT_ID={{WrapAsVariable "aadTenantId"}} - VAR_TENANT_ID={{WrapAsVariable "tenantId"}} - VAR_TARGET_ENV={{WrapAsVariable "targetEnvironment"}} AAD_TENANT_ID=${VAR_AAD_TENANT_ID:-$VAR_TENANT_ID} - AAD_ISSUER_HOST="sts.windows.net" - if [ "$VAR_TARGET_ENV" = "AzureChinaCloud" ]; then - AAD_ISSUER_HOST="sts.chinacloudapi.cn" - fi - - OIDC_ISSUER_URL="https://$AAD_ISSUER_HOST/$AAD_TENANT_ID/" - perl -pi -e "s|--oidc-client-id=\K(?=\")|$OIDC_CLIENT_ID| || s|--oidc-issuer-url=\K(?=\")|$OIDC_ISSUER_URL|" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{else}} - sed -i "/--oidc-client-id\|--oidc-issuer-url\|--oidc-username-claim/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" + sed -i "/--oidc-issuer-url/s/$/$AAD_TENANT_ID/" "/etc/kubernetes/manifests/kube-apiserver.yaml" {{end}} sed -i "s||{{WrapAsVariable "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml" - sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s||{{WrapAsVariable "kubeServiceCidr"}}|g; s||{{WrapAsVariable "masterEtcdClientPort"}}|g; s||{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" + sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s||{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s||{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml" @@ -245,19 +233,9 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsVariable "kubernetesReschedulerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" {{end}} -{{if .OrchestratorProfile.KubernetesConfig.EnableRbac }} - # If RBAC enabled then add parameters to API server and Controller manager configuration - sed -i "s||--authorization-mode=RBAC|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{else}} - sed -i "//d" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{end}} - {{if EnableDataEncryptionAtRest }} ETCD_ENCRYPTION_SECRET="$(head -c 32 /dev/urandom | base64)" sed -i "s||$ETCD_ENCRYPTION_SECRET|g" "/etc/kubernetes/encryption-config.yaml" - sed -i "s||--experimental-encryption-provider-config=/etc/kubernetes/encryption-config.yaml|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{else}} - sed -i "//d" "/etc/kubernetes/manifests/kube-apiserver.yaml" {{end}} {{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}} @@ -265,27 +243,15 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml" {{end}} -{{if not .OrchestratorProfile.KubernetesConfig.EnableAggregatedAPIs}} - sed -i "/requestheader-client-ca-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/proxy-client-cert-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/proxy-client-key-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/requestheader-allowed-names/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/requestheader-extra-headers-prefix/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/requestheader-group-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "/requestheader-username-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{end}} - sed -i "s||{{ .OrchestratorProfile.GetAPIServerEtcdAPIVersion }}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" - {{if UseCloudControllerManager }} sed -i "s||{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s||{{WrapAsVariable "masterFqdnPrefix"}}|g; s||{{WrapAsVariable "allocateNodeCidrs"}}|g; s||{{WrapAsVariable "kubeClusterCidr"}}|g; s||{{GetCloudControllerManagerRouteReconciliationPeriod .OrchestratorProfile.KubernetesConfig}}|g" \ /etc/kubernetes/manifests/cloud-controller-manager.yaml sed -i "/--\(cloud-config\|cloud-provider\|route-reconciliation-period\)=/d" \ /etc/kubernetes/manifests/kube-controller-manager.yaml - sed -i "/--\(cloud-config\|cloud-provider\)=/d" \ - /etc/kubernetes/manifests/kube-apiserver.yaml {{end}} sed -i "s||{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" + sed -i "s||{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" - path: "/opt/azure/containers/provision.sh" permissions: "0744" diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 1260fb151d..4d05e472de 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -267,8 +267,8 @@ "[concat(variables('masterFirstAddrPrefix'), add(3, int(variables('masterFirstAddrOctet4'))))]", "[concat(variables('masterFirstAddrPrefix'), add(4, int(variables('masterFirstAddrOctet4'))))]" ], - "masterEtcdServerPort": 2380, - "masterEtcdClientPort": 2379, + "masterEtcdServerPort": "[parameters('masterEtcdServerPort')]", + "masterEtcdClientPort": "[parameters('masterEtcdClientPort')]", "masterEtcdPeerURLs":[ "[concat('https://', variables('masterPrivateIpAddrs')[0], ':', variables('masterEtcdServerPort'))]", "[concat('https://', variables('masterPrivateIpAddrs')[1], ':', variables('masterEtcdServerPort'))]", diff --git a/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml b/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml index 28a0a0833c..bef8e55df9 100644 --- a/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml +++ b/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml @@ -21,7 +21,6 @@ spec: - "--cloud-config=/etc/kubernetes/azure.json" - "--leader-elect=true" # TODO: RBAC support -# - "" - "--route-reconciliation-period=" - "--v=2" volumeMounts: diff --git a/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml b/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml index fe1763e794..8c48f7bd95 100644 --- a/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml +++ b/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml @@ -11,43 +11,8 @@ spec: containers: - name: "kube-apiserver" image: "" - command: - - "/hyperkube" - - "apiserver" - - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec" - - "--address=0.0.0.0" - - "--allow-privileged" - - "--insecure-port=8080" - - "--secure-port=443" - - "--cloud-provider=azure" - - "--cloud-config=/etc/kubernetes/azure.json" - - "--service-cluster-ip-range=" - - "--etcd-cafile=/etc/kubernetes/certs/ca.crt" - - "--etcd-certfile=/etc/kubernetes/certs/etcdclient.crt" - - "--etcd-keyfile=/etc/kubernetes/certs/etcdclient.key" - - "--etcd-servers=https://127.0.0.1:" - - "--etcd-quorum-read=true" - - "--advertise-address=" - - "--tls-cert-file=/etc/kubernetes/certs/apiserver.crt" - - "--tls-private-key-file=/etc/kubernetes/certs/apiserver.key" - - "--client-ca-file=/etc/kubernetes/certs/ca.crt" - - "--requestheader-client-ca-file=/etc/kubernetes/certs/proxy-ca.crt" - - "--proxy-client-cert-file=/etc/kubernetes/certs/proxy.crt" - - "--proxy-client-key-file=/etc/kubernetes/certs/proxy.key" - - "--service-account-key-file=/etc/kubernetes/certs/apiserver.key" - - "--kubelet-client-certificate=/etc/kubernetes/certs/client.crt" - - "--kubelet-client-key=/etc/kubernetes/certs/client.key" - - "--oidc-client-id=" - - "--oidc-issuer-url=" - - "--oidc-username-claim=oid" - - "--storage-backend=" - - "--v=4" - - "" - - "" - - "--requestheader-allowed-names=" - - "--requestheader-extra-headers-prefix=X-Remote-Extra-" - - "--requestheader-group-headers=X-Remote-Group" - - "--requestheader-username-headers=X-Remote-User" + command: ["/hyperkube", "apiserver"] + args: [] volumeMounts: - name: "etc-kubernetes" mountPath: "/etc/kubernetes" diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index fff9f4b97e..5c8d569332 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -105,6 +105,10 @@ const ( DefaultReschedulerAddonName = "rescheduler" // DefaultKubernetesKubeletMaxPods is the max pods per kubelet DefaultKubernetesKubeletMaxPods = 110 + // DefaultMasterEtcdClientPort is the etcd client port for master nodes + DefaultMasterEtcdClientPort = 2379 + // DefaultMasterEtcdServerPort is the etcd server port for master nodes + DefaultMasterEtcdServerPort = 2380 ) const ( diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go new file mode 100644 index 0000000000..b53b077111 --- /dev/null +++ b/pkg/acsengine/defaults-apiserver.go @@ -0,0 +1,102 @@ +package acsengine + +import ( + "strconv" + + "github.com/Azure/acs-engine/pkg/api" + "github.com/Azure/acs-engine/pkg/helpers" +) + +func setAPIServerConfig(cs *api.ContainerService) { + o := cs.Properties.OrchestratorProfile + staticLinuxAPIServerConfig := map[string]string{ + "--admission-control": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec", + "--address": "0.0.0.0", + "--advertise-address": "", + "--allow-privileged": "true", + "--insecure-port": "8080", + "--secure-port": "443", + "--etcd-cafile": "/etc/kubernetes/certs/ca.crt", + "--etcd-certfile": "/etc/kubernetes/certs/etcdclient.crt", + "--etcd-keyfile": "/etc/kubernetes/certs/etcdclient.key", + "--etcd-servers": "https://127.0.0.1:" + strconv.Itoa(DefaultMasterEtcdClientPort), + "--etcd-quorum-read": "true", + "--tls-cert-file": "/etc/kubernetes/certs/apiserver.crt", + "--tls-private-key-file": "/etc/kubernetes/certs/apiserver.key", + "--client-ca-file": "/etc/kubernetes/certs/ca.crt", + "--service-account-key-file": "/etc/kubernetes/certs/apiserver.key", + "--kubelet-client-certificate": "/etc/kubernetes/certs/client.crt", + "--kubelet-client-key": "/etc/kubernetes/certs/client.key", + "--service-cluster-ip-range": o.KubernetesConfig.ServiceCIDR, + "--storage-backend": o.GetAPIServerEtcdAPIVersion(), + "--v": "4", + } + + if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableRbac) { + staticLinuxAPIServerConfig["--authorization-mode"] = "RBAC" + } + + if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableDataEncryptionAtRest) { + staticLinuxAPIServerConfig["--experimental-encryption-provider-config"] = "/etc/kubernetes/encryption-config.yaml" + } + + if o.KubernetesConfig.EnableAggregatedAPIs { + staticLinuxAPIServerConfig["--requestheader-client-ca-file"] = "/etc/kubernetes/certs/proxy-ca.crt" + staticLinuxAPIServerConfig["--proxy-client-cert-file"] = "/etc/kubernetes/certs/proxy.crt" + staticLinuxAPIServerConfig["--proxy-client-key-file"] = "/etc/kubernetes/certs/proxy.key" + staticLinuxAPIServerConfig["--requestheader-allowed-names"] = "" + staticLinuxAPIServerConfig["--requestheader-extra-headers-prefix"] = "X-Remote-Extra-" + staticLinuxAPIServerConfig["--requestheader-group-headers"] = "X-Remote-Group" + staticLinuxAPIServerConfig["--requestheader-username-headers"] = "X-Remote-User" + } + + if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { + staticLinuxAPIServerConfig["--cloud-provider"] = "azure" + staticLinuxAPIServerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" + } + + if cs.Properties.HasAadProfile() { + staticLinuxAPIServerConfig["--oidc-username-claim"] = "oid" + staticLinuxAPIServerConfig["--oidc-client-id"] = "spn:" + cs.Properties.AADProfile.ServerAppID + issuerHost := "sts.windows.net" + if GetCloudTargetEnv(cs.Location) == "AzureChinaCloud" { + issuerHost = "sts.chinacloudapi.cn" + } + staticLinuxAPIServerConfig["--oidc-issuer-url"] = "https://" + issuerHost + "/" + cs.Properties.AADProfile.TenantID + } + + staticWindowsAPIServerConfig := make(map[string]string) + for key, val := range staticLinuxAPIServerConfig { + staticWindowsAPIServerConfig[key] = val + } + // Windows apiserver config overrides + // TODO placeholder for specific config overrides for Windows clusters + + // Default apiserver config + defaultAPIServerConfig := map[string]string{} + + // If no user-configurable apiserver config values exists, use the defaults + if o.KubernetesConfig.APIServerConfig == nil { + o.KubernetesConfig.APIServerConfig = defaultAPIServerConfig + } else { + for key, val := range defaultAPIServerConfig { + // If we don't have a user-configurable apiserver config for each option + if _, ok := o.KubernetesConfig.APIServerConfig[key]; !ok { + // then assign the default value + o.KubernetesConfig.APIServerConfig[key] = val + } + } + } + + // We don't support user-configurable values for the following, + // so any of the value assignments below will override user-provided values + var overrideAPIServerConfig map[string]string + if cs.Properties.HasWindows() { + overrideAPIServerConfig = staticWindowsAPIServerConfig + } else { + overrideAPIServerConfig = staticLinuxAPIServerConfig + } + for key, val := range overrideAPIServerConfig { + o.KubernetesConfig.APIServerConfig[key] = val + } +} diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 27a1b0846a..c06b7ea13b 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -403,6 +403,8 @@ func setOrchestratorDefaults(cs *api.ContainerService) { setKubeletConfig(cs) // Configure controller-manager setControllerManagerConfig(cs) + // Configure apiserver + setAPIServerConfig(cs) } else if o.OrchestratorType == api.DCOS { if o.DcosConfig == nil { diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 3dfd763626..e384aeb7bf 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -621,6 +621,8 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "etcdDownloadURLBase", cloudSpecConfig.KubernetesSpecConfig.EtcdDownloadURLBase) addValue(parametersMap, "etcdVersion", cs.Properties.OrchestratorProfile.KubernetesConfig.EtcdVersion) addValue(parametersMap, "etcdDiskSizeGB", cs.Properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB) + addValue(parametersMap, "masterEtcdServerPort", DefaultMasterEtcdServerPort) + addValue(parametersMap, "masterEtcdClientPort", DefaultMasterEtcdClientPort) if properties.OrchestratorProfile.KubernetesConfig == nil || !properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity { @@ -873,6 +875,20 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat } return strings.TrimSuffix(buf.String(), ", ") }, + "GetAPIServerConfigKeyVals": func(kc *api.KubernetesConfig) string { + apiServerConfig := kc.APIServerConfig + // Order by key for consistency + keys := []string{} + for key := range apiServerConfig { + keys = append(keys, key) + } + sort.Strings(keys) + var buf bytes.Buffer + for _, key := range keys { + buf.WriteString(fmt.Sprintf("\"%s=%s\", ", key, apiServerConfig[key])) + } + return strings.TrimSuffix(buf.String(), ", ") + }, // temporary until we genericise cloud controller manager config "GetCloudControllerManagerRouteReconciliationPeriod": func(kc *api.KubernetesConfig) string { controllerManagerConfig := cs.Properties.OrchestratorProfile.KubernetesConfig.ControllerManagerConfig diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 1f4a4fc7f9..b5c9d1a3a0 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -674,6 +674,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet convertAddonsToVlabs(api, vlabs) convertKubeletConfigToVlabs(api, vlabs) convertControllerManagerConfigToVlabs(api, vlabs) + convertAPIServerConfigToVlabs(api, vlabs) } func convertKubeletConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) { @@ -690,6 +691,13 @@ func convertControllerManagerConfigToVlabs(a *KubernetesConfig, v *vlabs.Kuberne } } +func convertAPIServerConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) { + v.APIServerConfig = map[string]string{} + for key, val := range a.APIServerConfig { + v.APIServerConfig[key] = val + } +} + func convertAddonsToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) { v.Addons = []vlabs.KubernetesAddon{} for i := range a.Addons { diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 8870524085..7418c627db 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -618,6 +618,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes convertAddonsToAPI(vlabs, api) convertKubeletConfigToAPI(vlabs, api) convertControllerManagerConfigToAPI(vlabs, api) + convertAPIServerConfigToAPI(vlabs, api) } func setVlabsKubernetesDefaults(vp *vlabs.Properties, api *OrchestratorProfile) { @@ -674,6 +675,13 @@ func convertControllerManagerConfigToAPI(v *vlabs.KubernetesConfig, a *Kubernete } } +func convertAPIServerConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) { + a.APIServerConfig = map[string]string{} + for key, val := range v.APIServerConfig { + a.APIServerConfig[key] = val + } +} + func convertV20160930MasterProfile(v20160930 *v20160930.MasterProfile, api *MasterProfile) { api.Count = v20160930.Count api.DNSPrefix = v20160930.DNSPrefix diff --git a/pkg/api/types.go b/pkg/api/types.go index 229e9f545e..ad6e7be9ac 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -228,6 +228,7 @@ type KubernetesConfig struct { Addons []KubernetesAddon `json:"addons,omitempty"` KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` } // DcosConfig Configuration for DC/OS diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index efceb8c6ef..fa9aae6194 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -246,6 +246,7 @@ type KubernetesConfig struct { Addons []KubernetesAddon `json:"addons,omitempty"` KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` } // DcosConfig Configuration for DC/OS From 704e9dc154811f248176ccca7a4a9fecc442a3ae Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Jan 2018 13:09:27 -0800 Subject: [PATCH 02/10] add spaces for visual improvement --- docs/clusterdefinition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 5b4704076e..2cd8ca9d89 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -246,7 +246,7 @@ Below is a list of apiserver options that are *not* currently user-configurable, |apiserver option|default value| |---|---| -|"--admission-control"|"NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec"| +|"--admission-control"|"NamespaceLifecycle, LimitRanger, ServiceAccount, DefaultStorageClass, ResourceQuota, DenyEscalatingExec"| |"--address"|"0.0.0.0"| |"--advertise-address"|*calculated value that represents listening URI for API server*| |"--allow-privileged"|"true"| From 01422ac5472ae21d8100c54d35db6f95d3a78996 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Jan 2018 13:18:09 -0800 Subject: [PATCH 03/10] rationalize aad vars --- parts/k8s/kubernetesmastercustomdata.yml | 2 ++ parts/k8s/kubernetesmastervars.t | 1 - parts/k8s/kubernetesparams.t | 6 ------ pkg/acsengine/defaults-apiserver.go | 2 +- pkg/acsengine/engine.go | 1 - 5 files changed, 3 insertions(+), 9 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index ec41b2e6d5..f44927a745 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -182,6 +182,8 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{end}} {{ if .HasAadProfile }} + VAR_AAD_TENANT_ID={{WrapAsVariable "aadTenantId"}} + VAR_TENANT_ID={{WrapAsVariable "tenantId"}} AAD_TENANT_ID=${VAR_AAD_TENANT_ID:-$VAR_TENANT_ID} sed -i "/--oidc-issuer-url/s/$/$AAD_TENANT_ID/" "/etc/kubernetes/manifests/kube-apiserver.yaml" {{end}} diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 4d05e472de..4211307d48 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -109,7 +109,6 @@ {{end}} "sshPublicKeyData": "[parameters('sshRSAPublicKey')]", {{if .HasAadProfile}} - "aadServerAppId": "[parameters('aadServerAppId')]", "aadTenantId": "[parameters('aadTenantId')]", {{end}} {{if not IsHostedMaster}} diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 389ce1028f..fe693177d1 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -1,10 +1,4 @@ {{if .HasAadProfile}} - "aadServerAppId": { - "metadata": { - "description": "The server AAD application ID" - }, - "type": "string" - }, "aadTenantId": { "defaultValue": "", "metadata": { diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go index b53b077111..dacfec8284 100644 --- a/pkg/acsengine/defaults-apiserver.go +++ b/pkg/acsengine/defaults-apiserver.go @@ -62,7 +62,7 @@ func setAPIServerConfig(cs *api.ContainerService) { if GetCloudTargetEnv(cs.Location) == "AzureChinaCloud" { issuerHost = "sts.chinacloudapi.cn" } - staticLinuxAPIServerConfig["--oidc-issuer-url"] = "https://" + issuerHost + "/" + cs.Properties.AADProfile.TenantID + staticLinuxAPIServerConfig["--oidc-issuer-url"] = "https://" + issuerHost + "/" } staticWindowsAPIServerConfig := make(map[string]string) diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index e384aeb7bf..8cc3cf5116 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -640,7 +640,6 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s if properties.AADProfile != nil { addValue(parametersMap, "aadTenantId", properties.AADProfile.TenantID) - addValue(parametersMap, "aadServerAppId", properties.AADProfile.ServerAppID) } } From 6f052c5ebaa16a17f59de9dc6174b5716dd5b18e Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Jan 2018 13:30:59 -0800 Subject: [PATCH 04/10] same ccm config approach for both apiserver and controller-manager --- parts/k8s/kubernetesmastercustomdata.yml | 3 --- pkg/acsengine/defaults-apiserver.go | 5 +++++ pkg/acsengine/defaults-controller-manager.go | 9 +++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index f44927a745..407a17321b 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -248,9 +248,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{if UseCloudControllerManager }} sed -i "s||{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s||{{WrapAsVariable "masterFqdnPrefix"}}|g; s||{{WrapAsVariable "allocateNodeCidrs"}}|g; s||{{WrapAsVariable "kubeClusterCidr"}}|g; s||{{GetCloudControllerManagerRouteReconciliationPeriod .OrchestratorProfile.KubernetesConfig}}|g" \ /etc/kubernetes/manifests/cloud-controller-manager.yaml - - sed -i "/--\(cloud-config\|cloud-provider\|route-reconciliation-period\)=/d" \ - /etc/kubernetes/manifests/kube-controller-manager.yaml {{end}} sed -i "s||{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" sed -i "s||{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go index dacfec8284..f1d98b5579 100644 --- a/pkg/acsengine/defaults-apiserver.go +++ b/pkg/acsengine/defaults-apiserver.go @@ -32,14 +32,17 @@ func setAPIServerConfig(cs *api.ContainerService) { "--v": "4", } + // RBAC configuration if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableRbac) { staticLinuxAPIServerConfig["--authorization-mode"] = "RBAC" } + // Data Encryption at REST configuration if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableDataEncryptionAtRest) { staticLinuxAPIServerConfig["--experimental-encryption-provider-config"] = "/etc/kubernetes/encryption-config.yaml" } + // Aggregated API configuration if o.KubernetesConfig.EnableAggregatedAPIs { staticLinuxAPIServerConfig["--requestheader-client-ca-file"] = "/etc/kubernetes/certs/proxy-ca.crt" staticLinuxAPIServerConfig["--proxy-client-cert-file"] = "/etc/kubernetes/certs/proxy.crt" @@ -50,11 +53,13 @@ func setAPIServerConfig(cs *api.ContainerService) { staticLinuxAPIServerConfig["--requestheader-username-headers"] = "X-Remote-User" } + // Enable cloudprovider if we're not using cloud controller manager if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { staticLinuxAPIServerConfig["--cloud-provider"] = "azure" staticLinuxAPIServerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" } + // AAD configuration if cs.Properties.HasAadProfile() { staticLinuxAPIServerConfig["--oidc-username-claim"] = "oid" staticLinuxAPIServerConfig["--oidc-client-id"] = "spn:" + cs.Properties.AADProfile.ServerAppID diff --git a/pkg/acsengine/defaults-controller-manager.go b/pkg/acsengine/defaults-controller-manager.go index 7b731ad070..e16d682676 100644 --- a/pkg/acsengine/defaults-controller-manager.go +++ b/pkg/acsengine/defaults-controller-manager.go @@ -4,6 +4,7 @@ import ( "strconv" "github.com/Azure/acs-engine/pkg/api" + "github.com/Azure/acs-engine/pkg/helpers" ) func setControllerManagerConfig(cs *api.ContainerService) { @@ -12,8 +13,6 @@ func setControllerManagerConfig(cs *api.ContainerService) { "--kubeconfig": "/var/lib/kubelet/kubeconfig", "--allocate-node-cidrs": strconv.FormatBool(!o.IsAzureCNI()), "--cluster-cidr": o.KubernetesConfig.ClusterSubnet, - "--cloud-provider": "azure", - "--cloud-config": "/etc/kubernetes/azure.json", "--root-ca-file": "/etc/kubernetes/certs/ca.crt", "--cluster-signing-cert-file": "/etc/kubernetes/certs/ca.crt", "--cluster-signing-key-file": "/etc/kubernetes/certs/ca.key", @@ -30,6 +29,12 @@ func setControllerManagerConfig(cs *api.ContainerService) { staticLinuxControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix } + // Enable cloudprovider if we're not using cloud controller manager + if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { + staticLinuxControllerManagerConfig["--cloud-provider"] = "azure" + staticLinuxControllerManagerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" + } + staticWindowsControllerManagerConfig := make(map[string]string) for key, val := range staticLinuxControllerManagerConfig { staticWindowsControllerManagerConfig[key] = val From 33d9b33ac83ea6dcb2e4c3cd69d66d5d2c549537 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Jan 2018 13:52:50 -0800 Subject: [PATCH 05/10] reverting changes to masterEtcdServerPort and masterEtcdClientPort --- parts/k8s/kubernetesmastervars.t | 4 ++-- pkg/acsengine/const.go | 4 ---- pkg/acsengine/engine.go | 2 -- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 4211307d48..550e3817a8 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -266,8 +266,8 @@ "[concat(variables('masterFirstAddrPrefix'), add(3, int(variables('masterFirstAddrOctet4'))))]", "[concat(variables('masterFirstAddrPrefix'), add(4, int(variables('masterFirstAddrOctet4'))))]" ], - "masterEtcdServerPort": "[parameters('masterEtcdServerPort')]", - "masterEtcdClientPort": "[parameters('masterEtcdClientPort')]", + "masterEtcdServerPort": 2380, + "masterEtcdClientPort": 2379, "masterEtcdPeerURLs":[ "[concat('https://', variables('masterPrivateIpAddrs')[0], ':', variables('masterEtcdServerPort'))]", "[concat('https://', variables('masterPrivateIpAddrs')[1], ':', variables('masterEtcdServerPort'))]", diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 5c8d569332..fff9f4b97e 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -105,10 +105,6 @@ const ( DefaultReschedulerAddonName = "rescheduler" // DefaultKubernetesKubeletMaxPods is the max pods per kubelet DefaultKubernetesKubeletMaxPods = 110 - // DefaultMasterEtcdClientPort is the etcd client port for master nodes - DefaultMasterEtcdClientPort = 2379 - // DefaultMasterEtcdServerPort is the etcd server port for master nodes - DefaultMasterEtcdServerPort = 2380 ) const ( diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 8cc3cf5116..af9834ece3 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -621,8 +621,6 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "etcdDownloadURLBase", cloudSpecConfig.KubernetesSpecConfig.EtcdDownloadURLBase) addValue(parametersMap, "etcdVersion", cs.Properties.OrchestratorProfile.KubernetesConfig.EtcdVersion) addValue(parametersMap, "etcdDiskSizeGB", cs.Properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB) - addValue(parametersMap, "masterEtcdServerPort", DefaultMasterEtcdServerPort) - addValue(parametersMap, "masterEtcdClientPort", DefaultMasterEtcdClientPort) if properties.OrchestratorProfile.KubernetesConfig == nil || !properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity { From f0817ac2ce39e671fda77d84c74cac6d45f98562 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Jan 2018 14:01:23 -0800 Subject: [PATCH 06/10] etcd client port is static 2397 --- pkg/acsengine/defaults-apiserver.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go index f1d98b5579..603440abf8 100644 --- a/pkg/acsengine/defaults-apiserver.go +++ b/pkg/acsengine/defaults-apiserver.go @@ -1,8 +1,6 @@ package acsengine import ( - "strconv" - "github.com/Azure/acs-engine/pkg/api" "github.com/Azure/acs-engine/pkg/helpers" ) @@ -19,7 +17,7 @@ func setAPIServerConfig(cs *api.ContainerService) { "--etcd-cafile": "/etc/kubernetes/certs/ca.crt", "--etcd-certfile": "/etc/kubernetes/certs/etcdclient.crt", "--etcd-keyfile": "/etc/kubernetes/certs/etcdclient.key", - "--etcd-servers": "https://127.0.0.1:" + strconv.Itoa(DefaultMasterEtcdClientPort), + "--etcd-servers": "https://127.0.0.1:2397", "--etcd-quorum-read": "true", "--tls-cert-file": "/etc/kubernetes/certs/apiserver.crt", "--tls-private-key-file": "/etc/kubernetes/certs/apiserver.key", From 72e149d3600a499fc1252c8f3398565e634d3638 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Jan 2018 14:59:11 -0800 Subject: [PATCH 07/10] moving sed bits around --- parts/k8s/kubernetesmastercustomdata.yml | 16 ++++++++-------- pkg/acsengine/defaults-apiserver.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 407a17321b..7139c1a8d9 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -180,15 +180,8 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER # SNAT outbound traffic from pods to destinations outside of VNET. iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsVariable "vnetCidr"}} -j MASQUERADE {{end}} - -{{ if .HasAadProfile }} - VAR_AAD_TENANT_ID={{WrapAsVariable "aadTenantId"}} - VAR_TENANT_ID={{WrapAsVariable "tenantId"}} - AAD_TENANT_ID=${VAR_AAD_TENANT_ID:-$VAR_TENANT_ID} - sed -i "/--oidc-issuer-url/s/$/$AAD_TENANT_ID/" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{end}} sed -i "s||{{WrapAsVariable "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml" - sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s||{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" + sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s||{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml" @@ -251,6 +244,13 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{end}} sed -i "s||{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" sed -i "s||{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" + sed -i "s|kubernetesAPIServerIP|{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" +{{ if .HasAadProfile }} + VAR_AAD_TENANT_ID={{WrapAsVariable "aadTenantId"}} + VAR_TENANT_ID={{WrapAsVariable "tenantId"}} + AAD_TENANT_ID=${VAR_AAD_TENANT_ID:-$VAR_TENANT_ID} + sed -i "/--oidc-issuer-url/s/$/$AAD_TENANT_ID/" "/etc/kubernetes/manifests/kube-apiserver.yaml" +{{end}} - path: "/opt/azure/containers/provision.sh" permissions: "0744" diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go index 603440abf8..77d2b8a08a 100644 --- a/pkg/acsengine/defaults-apiserver.go +++ b/pkg/acsengine/defaults-apiserver.go @@ -10,7 +10,7 @@ func setAPIServerConfig(cs *api.ContainerService) { staticLinuxAPIServerConfig := map[string]string{ "--admission-control": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec", "--address": "0.0.0.0", - "--advertise-address": "", + "--advertise-address": "kubernetesAPIServerIP", "--allow-privileged": "true", "--insecure-port": "8080", "--secure-port": "443", From 5b49dae372d5c3ef51b98d528a6fdb63f1adde5b Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Jan 2018 16:50:37 -0800 Subject: [PATCH 08/10] fun w/ sed --- parts/k8s/kubernetesmastercustomdata.yml | 2 +- pkg/acsengine/defaults-apiserver.go | 2 +- pkg/acsengine/engine.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 7139c1a8d9..76917a7922 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -244,7 +244,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{end}} sed -i "s||{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" sed -i "s||{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "s|kubernetesAPIServerIP|{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" + sed -i "s||{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" {{ if .HasAadProfile }} VAR_AAD_TENANT_ID={{WrapAsVariable "aadTenantId"}} VAR_TENANT_ID={{WrapAsVariable "tenantId"}} diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go index 77d2b8a08a..603440abf8 100644 --- a/pkg/acsengine/defaults-apiserver.go +++ b/pkg/acsengine/defaults-apiserver.go @@ -10,7 +10,7 @@ func setAPIServerConfig(cs *api.ContainerService) { staticLinuxAPIServerConfig := map[string]string{ "--admission-control": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec", "--address": "0.0.0.0", - "--advertise-address": "kubernetesAPIServerIP", + "--advertise-address": "", "--allow-privileged": "true", "--insecure-port": "8080", "--secure-port": "443", diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index af9834ece3..e5856f9687 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -882,7 +882,7 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat sort.Strings(keys) var buf bytes.Buffer for _, key := range keys { - buf.WriteString(fmt.Sprintf("\"%s=%s\", ", key, apiServerConfig[key])) + buf.WriteString(fmt.Sprintf("\\\"%s=%s\\\", ", key, apiServerConfig[key])) } return strings.TrimSuffix(buf.String(), ", ") }, From 74e9471ebee75393ee84e71f2279daf8482a8dd6 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Jan 2018 17:06:42 -0800 Subject: [PATCH 09/10] humans are the best --- pkg/acsengine/defaults-apiserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go index 603440abf8..ae0727ac6a 100644 --- a/pkg/acsengine/defaults-apiserver.go +++ b/pkg/acsengine/defaults-apiserver.go @@ -17,7 +17,7 @@ func setAPIServerConfig(cs *api.ContainerService) { "--etcd-cafile": "/etc/kubernetes/certs/ca.crt", "--etcd-certfile": "/etc/kubernetes/certs/etcdclient.crt", "--etcd-keyfile": "/etc/kubernetes/certs/etcdclient.key", - "--etcd-servers": "https://127.0.0.1:2397", + "--etcd-servers": "https://127.0.0.1:2379", "--etcd-quorum-read": "true", "--tls-cert-file": "/etc/kubernetes/certs/apiserver.crt", "--tls-private-key-file": "/etc/kubernetes/certs/apiserver.key", From f40d7c0e09c97064e05d6f82ff49517896676b11 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Tue, 9 Jan 2018 09:00:34 -0800 Subject: [PATCH 10/10] using vars for etcd master ports --- parts/k8s/kubernetesmastervars.t | 4 ++-- pkg/acsengine/const.go | 4 ++++ pkg/acsengine/defaults-apiserver.go | 4 +++- pkg/acsengine/engine.go | 6 ++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 550e3817a8..bed18bb84b 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -266,8 +266,8 @@ "[concat(variables('masterFirstAddrPrefix'), add(3, int(variables('masterFirstAddrOctet4'))))]", "[concat(variables('masterFirstAddrPrefix'), add(4, int(variables('masterFirstAddrOctet4'))))]" ], - "masterEtcdServerPort": 2380, - "masterEtcdClientPort": 2379, + "masterEtcdServerPort": {{GetMasterEtcdServerPort}}, + "masterEtcdClientPort": {{GetMasterEtcdClientPort}}, "masterEtcdPeerURLs":[ "[concat('https://', variables('masterPrivateIpAddrs')[0], ':', variables('masterEtcdServerPort'))]", "[concat('https://', variables('masterPrivateIpAddrs')[1], ':', variables('masterEtcdServerPort'))]", diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index fff9f4b97e..f9a7be62f6 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -105,6 +105,10 @@ const ( DefaultReschedulerAddonName = "rescheduler" // DefaultKubernetesKubeletMaxPods is the max pods per kubelet DefaultKubernetesKubeletMaxPods = 110 + // DefaultMasterEtcdServerPort is the default etcd server port for Kubernetes master nodes + DefaultMasterEtcdServerPort = 2380 + // DefaultMasterEtcdClientPort is the default etcd client port for Kubernetes master nodes + DefaultMasterEtcdClientPort = 2379 ) const ( diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go index ae0727ac6a..f1d98b5579 100644 --- a/pkg/acsengine/defaults-apiserver.go +++ b/pkg/acsengine/defaults-apiserver.go @@ -1,6 +1,8 @@ package acsengine import ( + "strconv" + "github.com/Azure/acs-engine/pkg/api" "github.com/Azure/acs-engine/pkg/helpers" ) @@ -17,7 +19,7 @@ func setAPIServerConfig(cs *api.ContainerService) { "--etcd-cafile": "/etc/kubernetes/certs/ca.crt", "--etcd-certfile": "/etc/kubernetes/certs/etcdclient.crt", "--etcd-keyfile": "/etc/kubernetes/certs/etcdclient.key", - "--etcd-servers": "https://127.0.0.1:2379", + "--etcd-servers": "https://127.0.0.1:" + strconv.Itoa(DefaultMasterEtcdClientPort), "--etcd-quorum-read": "true", "--tls-cert-file": "/etc/kubernetes/certs/apiserver.crt", "--tls-private-key-file": "/etc/kubernetes/certs/apiserver.key", diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index e5856f9687..af24f30573 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -1235,6 +1235,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat cloudSpecConfig := GetCloudSpecConfig(cs.Location) return fmt.Sprintf("\"%s\"", cloudSpecConfig.OSImageConfig[profile.Distro].ImageVersion) }, + "GetMasterEtcdServerPort": func() int { + return DefaultMasterEtcdServerPort + }, + "GetMasterEtcdClientPort": func() int { + return DefaultMasterEtcdClientPort + }, "PopulateClassicModeDefaultValue": func(attr string) string { var val string if !t.ClassicMode {