This repository has been archived by the owner on Jan 11, 2023. It is now read-only.
generic Kubernetes API server config interface #2012
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -180,25 +180,8 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER | |
| # SNAT outbound traffic from pods to destinations outside of VNET. | ||
| iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsVariable "vnetCidr"}} -j MASQUERADE | ||
| {{end}} | ||
| sed -i "s|<kubernetesAddonManagerSpec>|{{WrapAsVariable "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml" | ||
| sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
| sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" | ||
| sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml" | ||
| sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml" | ||
|
|
@@ -245,47 +228,29 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER | |
| sed -i "s|<kubernetesReschedulerMemoryLimit>|{{WrapAsVariable "kubernetesReschedulerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" | ||
| {{end}} | ||
|
|
||
| {{if EnableDataEncryptionAtRest }} | ||
| ETCD_ENCRYPTION_SECRET="$(head -c 32 /dev/urandom | base64)" | ||
| sed -i "s|<etcdEncryptionSecret>|$ETCD_ENCRYPTION_SECRET|g" "/etc/kubernetes/encryption-config.yaml" | ||
| {{end}} | ||
|
|
||
| {{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}} | ||
| # If Calico Policy enabled then update Cluster Cidr | ||
| sed -i "s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml" | ||
| {{end}} | ||
|
|
||
| {{if UseCloudControllerManager }} | ||
| sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s|<masterFqdnPrefix>|{{WrapAsVariable "masterFqdnPrefix"}}|g; s|<allocateNodeCidrs>|{{WrapAsVariable "allocateNodeCidrs"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g; s|<kubernetesCtrlMgrRouteReconciliationPeriod>|{{GetCloudControllerManagerRouteReconciliationPeriod .OrchestratorProfile.KubernetesConfig}}|g" \ | ||
| /etc/kubernetes/manifests/cloud-controller-manager.yaml | ||
|
There was a problem hiding this comment. This should have been removed in #1960, doing so now. |
||
| {{end}} | ||
| sed -i "s|<kubernetesControllerManagerConfig>|{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" | ||
| sed -i "s|<kubernetesAPIServerConfig>|{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
|
There was a problem hiding this comment. It is probably better to take APIServerConfig (instead of KubernetesConfig) as argument for GetAPIServerConfigKeyVals. The original code logic is pretty tangled here and I have not dig very deep, I assume GetAPIServerConfigKeyVals will return the already processed argument map. Please just confirm it. There was a problem hiding this comment. This is good feedback. Let's do this: after all of these config refactors are done, we can optimize the number of key/val getter functions in |
||
| sed -i "s|<kubernetesAPIServerIP>|{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
| {{ if .HasAadProfile }} | ||
| VAR_AAD_TENANT_ID={{WrapAsVariable "aadTenantId"}} | ||
| VAR_TENANT_ID={{WrapAsVariable "tenantId"}} | ||
| AAD_TENANT_ID=${VAR_AAD_TENANT_ID:-$VAR_TENANT_ID} | ||
| sed -i "/--oidc-issuer-url/s/$/$AAD_TENANT_ID/" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
|
There was a problem hiding this comment. If the above GetAPIServerConfigKeyVals returns already processed map, why are we overriding "--oidc-issuer-url" here? There was a problem hiding this comment. Good question. It's because the only way (that I know of) to assign the right tenantId value is in the flow of this template: we depend upon |
||
| {{end}} | ||
|
|
||
| - path: "/opt/azure/containers/provision.sh" | ||
| permissions: "0744" | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,4 @@ | ||
| {{if .HasAadProfile}} | ||
| "aadTenantId": { | ||
| "defaultValue": "", | ||
| "metadata": { | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| package acsengine | ||
|
|
||
| import ( | ||
| "github.com/Azure/acs-engine/pkg/api" | ||
| "github.com/Azure/acs-engine/pkg/helpers" | ||
| ) | ||
|
|
||
| func setAPIServerConfig(cs *api.ContainerService) { | ||
| o := cs.Properties.OrchestratorProfile | ||
| staticLinuxAPIServerConfig := map[string]string{ | ||
| "--admission-control": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec", | ||
| "--address": "0.0.0.0", | ||
| "--advertise-address": "<kubernetesAPIServerIP>", | ||
| "--allow-privileged": "true", | ||
| "--insecure-port": "8080", | ||
| "--secure-port": "443", | ||
| "--etcd-cafile": "/etc/kubernetes/certs/ca.crt", | ||
| "--etcd-certfile": "/etc/kubernetes/certs/etcdclient.crt", | ||
| "--etcd-keyfile": "/etc/kubernetes/certs/etcdclient.key", | ||
| "--etcd-servers": "https://127.0.0.1:2379", | ||
| "--etcd-quorum-read": "true", | ||
| "--tls-cert-file": "/etc/kubernetes/certs/apiserver.crt", | ||
| "--tls-private-key-file": "/etc/kubernetes/certs/apiserver.key", | ||
| "--client-ca-file": "/etc/kubernetes/certs/ca.crt", | ||
| "--service-account-key-file": "/etc/kubernetes/certs/apiserver.key", | ||
| "--kubelet-client-certificate": "/etc/kubernetes/certs/client.crt", | ||
| "--kubelet-client-key": "/etc/kubernetes/certs/client.key", | ||
| "--service-cluster-ip-range": o.KubernetesConfig.ServiceCIDR, | ||
| "--storage-backend": o.GetAPIServerEtcdAPIVersion(), | ||
| "--v": "4", | ||
| } | ||
|
|
||
| // RBAC configuration | ||
| if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableRbac) { | ||
| staticLinuxAPIServerConfig["--authorization-mode"] = "RBAC" | ||
| } | ||
|
|
||
| // Data Encryption at REST configuration | ||
| if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableDataEncryptionAtRest) { | ||
| staticLinuxAPIServerConfig["--experimental-encryption-provider-config"] = "/etc/kubernetes/encryption-config.yaml" | ||
| } | ||
|
|
||
| // Aggregated API configuration | ||
| if o.KubernetesConfig.EnableAggregatedAPIs { | ||
| staticLinuxAPIServerConfig["--requestheader-client-ca-file"] = "/etc/kubernetes/certs/proxy-ca.crt" | ||
| staticLinuxAPIServerConfig["--proxy-client-cert-file"] = "/etc/kubernetes/certs/proxy.crt" | ||
| staticLinuxAPIServerConfig["--proxy-client-key-file"] = "/etc/kubernetes/certs/proxy.key" | ||
| staticLinuxAPIServerConfig["--requestheader-allowed-names"] = "" | ||
| staticLinuxAPIServerConfig["--requestheader-extra-headers-prefix"] = "X-Remote-Extra-" | ||
| staticLinuxAPIServerConfig["--requestheader-group-headers"] = "X-Remote-Group" | ||
| staticLinuxAPIServerConfig["--requestheader-username-headers"] = "X-Remote-User" | ||
| } | ||
|
|
||
| // Enable cloudprovider if we're not using cloud controller manager | ||
| if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { | ||
| staticLinuxAPIServerConfig["--cloud-provider"] = "azure" | ||
| staticLinuxAPIServerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" | ||
| } | ||
|
|
||
| // AAD configuration | ||
| if cs.Properties.HasAadProfile() { | ||
| staticLinuxAPIServerConfig["--oidc-username-claim"] = "oid" | ||
| staticLinuxAPIServerConfig["--oidc-client-id"] = "spn:" + cs.Properties.AADProfile.ServerAppID | ||
| issuerHost := "sts.windows.net" | ||
| if GetCloudTargetEnv(cs.Location) == "AzureChinaCloud" { | ||
| issuerHost = "sts.chinacloudapi.cn" | ||
| } | ||
| staticLinuxAPIServerConfig["--oidc-issuer-url"] = "https://" + issuerHost + "/" | ||
| } | ||
|
|
||
| staticWindowsAPIServerConfig := make(map[string]string) | ||
| for key, val := range staticLinuxAPIServerConfig { | ||
| staticWindowsAPIServerConfig[key] = val | ||
| } | ||
| // Windows apiserver config overrides | ||
| // TODO placeholder for specific config overrides for Windows clusters | ||
|
|
||
| // Default apiserver config | ||
| defaultAPIServerConfig := map[string]string{} | ||
|
|
||
| // If no user-configurable apiserver config values exists, use the defaults | ||
| if o.KubernetesConfig.APIServerConfig == nil { | ||
| o.KubernetesConfig.APIServerConfig = defaultAPIServerConfig | ||
| } else { | ||
| for key, val := range defaultAPIServerConfig { | ||
| // If we don't have a user-configurable apiserver config for each option | ||
| if _, ok := o.KubernetesConfig.APIServerConfig[key]; !ok { | ||
| // then assign the default value | ||
| o.KubernetesConfig.APIServerConfig[key] = val | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // We don't support user-configurable values for the following, | ||
| // so any of the value assignments below will override user-provided values | ||
| var overrideAPIServerConfig map[string]string | ||
| if cs.Properties.HasWindows() { | ||
| overrideAPIServerConfig = staticWindowsAPIServerConfig | ||
| } else { | ||
| overrideAPIServerConfig = staticLinuxAPIServerConfig | ||
| } | ||
| for key, val := range overrideAPIServerConfig { | ||
| o.KubernetesConfig.APIServerConfig[key] = val | ||
|
There was a problem hiding this comment. Some argument like "--admission-control" and "--authorization-mode" takes a list of features, and probably should allow feature addition (as long as static config is a subset of the user input config), instead of override. It can be done in a later change if needed. There was a problem hiding this comment. I agree this would be a useful addition to this surface area. Feature gates also are special, in that they essentially contain key/vals themselves. Yeah, I'd prefer a follow-up PR for that. |
||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ import ( | |
| "strconv" | ||
|
|
||
| "github.com/Azure/acs-engine/pkg/api" | ||
| "github.com/Azure/acs-engine/pkg/helpers" | ||
| ) | ||
|
|
||
| func setControllerManagerConfig(cs *api.ContainerService) { | ||
|
|
@@ -12,8 +13,6 @@ func setControllerManagerConfig(cs *api.ContainerService) { | |
| "--kubeconfig": "/var/lib/kubelet/kubeconfig", | ||
| "--allocate-node-cidrs": strconv.FormatBool(!o.IsAzureCNI()), | ||
| "--cluster-cidr": o.KubernetesConfig.ClusterSubnet, | ||
|
There was a problem hiding this comment. This should have been done in #1960, doing so now. |
||
| "--root-ca-file": "/etc/kubernetes/certs/ca.crt", | ||
| "--cluster-signing-cert-file": "/etc/kubernetes/certs/ca.crt", | ||
| "--cluster-signing-key-file": "/etc/kubernetes/certs/ca.key", | ||
|
|
@@ -30,6 +29,12 @@ func setControllerManagerConfig(cs *api.ContainerService) { | |
| staticLinuxControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix | ||
| } | ||
|
|
||
| // Enable cloudprovider if we're not using cloud controller manager | ||
|
There was a problem hiding this comment. This should have been done in #1960, doing so now. |
||
| if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { | ||
| staticLinuxControllerManagerConfig["--cloud-provider"] = "azure" | ||
| staticLinuxControllerManagerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" | ||
| } | ||
|
|
||
| staticWindowsControllerManagerConfig := make(map[string]string) | ||
| for key, val := range staticLinuxControllerManagerConfig { | ||
| staticWindowsControllerManagerConfig[key] = val | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Correcting copy/paste errors from a prior, related PR.