generic Kubernetes API server config interface #2012
Changes from 3 commits
e19a6f1
704e9dc
01422ac
6f052c5
33d9b33
f0817ac
72e149d
5b49dae
74e9471
f40d7c0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -182,23 +182,13 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER | |
{{end}} | ||
|
||
{{ if .HasAadProfile }} | ||
OIDC_CLIENT_ID=spn:{{WrapAsVariable "aadServerAppId"}} | ||
VAR_AAD_TENANT_ID={{WrapAsVariable "aadTenantId"}} | ||
VAR_TENANT_ID={{WrapAsVariable "tenantId"}} | ||
VAR_TARGET_ENV={{WrapAsVariable "targetEnvironment"}} | ||
AAD_TENANT_ID=${VAR_AAD_TENANT_ID:-$VAR_TENANT_ID} | ||
AAD_ISSUER_HOST="sts.windows.net" | ||
if [ "$VAR_TARGET_ENV" = "AzureChinaCloud" ]; then | ||
AAD_ISSUER_HOST="sts.chinacloudapi.cn" | ||
fi | ||
|
||
OIDC_ISSUER_URL="https://$AAD_ISSUER_HOST/$AAD_TENANT_ID/" | ||
perl -pi -e "s|--oidc-client-id=\K(?=\")|$OIDC_CLIENT_ID| || s|--oidc-issuer-url=\K(?=\")|$OIDC_ISSUER_URL|" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
{{else}} | ||
sed -i "/--oidc-client-id\|--oidc-issuer-url\|--oidc-username-claim/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
sed -i "/--oidc-issuer-url/s/$/$AAD_TENANT_ID/" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We still need to do this post-deployment transformation because I'm not aware of any other way to implement the fallback tenantID assignment. The one here uses this
Which gives us our fallback |
||
{{end}} | ||
sed -i "s|<kubernetesAddonManagerSpec>|{{WrapAsVariable "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml" | ||
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s|<kubeServiceCidr>|{{WrapAsVariable "kubeServiceCidr"}}|g; s|<masterEtcdClientPort>|{{WrapAsVariable "masterEtcdClientPort"}}|g; s|<kubernetesAPIServerIP>|{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s|<kubernetesAPIServerIP>|{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" | ||
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml" | ||
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml" | ||
|
@@ -245,47 +235,25 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER | |
sed -i "s|<kubernetesReschedulerMemoryLimit>|{{WrapAsVariable "kubernetesReschedulerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" | ||
{{end}} | ||
|
||
{{if .OrchestratorProfile.KubernetesConfig.EnableRbac }} | ||
# If RBAC enabled then add parameters to API server and Controller manager configuration | ||
sed -i "s|<kubernetesEnableRbac>|--authorization-mode=RBAC|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
{{else}} | ||
sed -i "/<kubernetesEnableRbac>/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
{{end}} | ||
|
||
{{if EnableDataEncryptionAtRest }} | ||
ETCD_ENCRYPTION_SECRET="$(head -c 32 /dev/urandom | base64)" | ||
sed -i "s|<etcdEncryptionSecret>|$ETCD_ENCRYPTION_SECRET|g" "/etc/kubernetes/encryption-config.yaml" | ||
sed -i "s|<kubernetesEnableEtcdEncryption>|--experimental-encryption-provider-config=/etc/kubernetes/encryption-config.yaml|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
{{else}} | ||
sed -i "/<kubernetesEnableEtcdEncryption>/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
{{end}} | ||
|
||
{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}} | ||
# If Calico Policy enabled then update Cluster Cidr | ||
sed -i "s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml" | ||
{{end}} | ||
|
||
{{if not .OrchestratorProfile.KubernetesConfig.EnableAggregatedAPIs}} | ||
sed -i "/requestheader-client-ca-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
sed -i "/proxy-client-cert-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
sed -i "/proxy-client-key-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
sed -i "/requestheader-allowed-names/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
sed -i "/requestheader-extra-headers-prefix/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
sed -i "/requestheader-group-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
sed -i "/requestheader-username-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
{{end}} | ||
sed -i "s|<etcdApiVersion>|{{ .OrchestratorProfile.GetAPIServerEtcdAPIVersion }}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
|
||
{{if UseCloudControllerManager }} | ||
sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s|<masterFqdnPrefix>|{{WrapAsVariable "masterFqdnPrefix"}}|g; s|<allocateNodeCidrs>|{{WrapAsVariable "allocateNodeCidrs"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g; s|<kubernetesCtrlMgrRouteReconciliationPeriod>|{{GetCloudControllerManagerRouteReconciliationPeriod .OrchestratorProfile.KubernetesConfig}}|g" \ | ||
/etc/kubernetes/manifests/cloud-controller-manager.yaml | ||
|
||
sed -i "/--\(cloud-config\|cloud-provider\|route-reconciliation-period\)=/d" \ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This should have been removed in #1960, doing so now. |
||
/etc/kubernetes/manifests/kube-controller-manager.yaml | ||
sed -i "/--\(cloud-config\|cloud-provider\)=/d" \ | ||
/etc/kubernetes/manifests/kube-apiserver.yaml | ||
{{end}} | ||
sed -i "s|<kubernetesControllerManagerConfig>|{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" | ||
sed -i "s|<kubernetesAPIServerConfig>|{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is probably better to take APIServerConfig (instead of KubernetesConfig) as argument for GetAPIServerConfigKeyVals. The original code logic is pretty tangled here and I have not dig very deep, I assume GetAPIServerConfigKeyVals will return the already processed argument map. Please just confirm it. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is good feedback. Let's do this: after all of these config refactors are done, we can optimize the number of key/val getter functions in |
||
|
||
- path: "/opt/azure/containers/provision.sh" | ||
permissions: "0744" | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -109,7 +109,6 @@ | |
{{end}} | ||
"sshPublicKeyData": "[parameters('sshRSAPublicKey')]", | ||
{{if .HasAadProfile}} | ||
"aadServerAppId": "[parameters('aadServerAppId')]", | ||
"aadTenantId": "[parameters('aadTenantId')]", | ||
{{end}} | ||
{{if not IsHostedMaster}} | ||
|
@@ -267,8 +266,8 @@ | |
"[concat(variables('masterFirstAddrPrefix'), add(3, int(variables('masterFirstAddrOctet4'))))]", | ||
"[concat(variables('masterFirstAddrPrefix'), add(4, int(variables('masterFirstAddrOctet4'))))]" | ||
], | ||
"masterEtcdServerPort": 2380, | ||
"masterEtcdClientPort": 2379, | ||
"masterEtcdServerPort": "[parameters('masterEtcdServerPort')]", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Moved these vals to |
||
"masterEtcdClientPort": "[parameters('masterEtcdClientPort')]", | ||
"masterEtcdPeerURLs":[ | ||
"[concat('https://', variables('masterPrivateIpAddrs')[0], ':', variables('masterEtcdServerPort'))]", | ||
"[concat('https://', variables('masterPrivateIpAddrs')[1], ':', variables('masterEtcdServerPort'))]", | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -21,7 +21,6 @@ spec: | |
- "--cloud-config=/etc/kubernetes/azure.json" | ||
- "--leader-elect=true" | ||
# TODO: RBAC support | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do you want to remove RBAC placeholder here? Seems not related to this PR. If so, remove the above TODO comment too? |
||
# - "<kubernetesEnableRbac>" | ||
- "--route-reconciliation-period=<kubernetesCtrlMgrRouteReconciliationPeriod>" | ||
- "--v=2" | ||
volumeMounts: | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
package acsengine | ||
|
||
import ( | ||
"strconv" | ||
|
||
"github.com/Azure/acs-engine/pkg/api" | ||
"github.com/Azure/acs-engine/pkg/helpers" | ||
) | ||
|
||
func setAPIServerConfig(cs *api.ContainerService) { | ||
o := cs.Properties.OrchestratorProfile | ||
staticLinuxAPIServerConfig := map[string]string{ | ||
"--admission-control": "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DenyEscalatingExec", | ||
"--address": "0.0.0.0", | ||
"--advertise-address": "<kubernetesAPIServerIP>", | ||
"--allow-privileged": "true", | ||
"--insecure-port": "8080", | ||
"--secure-port": "443", | ||
"--etcd-cafile": "/etc/kubernetes/certs/ca.crt", | ||
"--etcd-certfile": "/etc/kubernetes/certs/etcdclient.crt", | ||
"--etcd-keyfile": "/etc/kubernetes/certs/etcdclient.key", | ||
"--etcd-servers": "https://127.0.0.1:" + strconv.Itoa(DefaultMasterEtcdClientPort), | ||
"--etcd-quorum-read": "true", | ||
"--tls-cert-file": "/etc/kubernetes/certs/apiserver.crt", | ||
"--tls-private-key-file": "/etc/kubernetes/certs/apiserver.key", | ||
"--client-ca-file": "/etc/kubernetes/certs/ca.crt", | ||
"--service-account-key-file": "/etc/kubernetes/certs/apiserver.key", | ||
"--kubelet-client-certificate": "/etc/kubernetes/certs/client.crt", | ||
"--kubelet-client-key": "/etc/kubernetes/certs/client.key", | ||
"--service-cluster-ip-range": o.KubernetesConfig.ServiceCIDR, | ||
"--storage-backend": o.GetAPIServerEtcdAPIVersion(), | ||
"--v": "4", | ||
} | ||
|
||
if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableRbac) { | ||
staticLinuxAPIServerConfig["--authorization-mode"] = "RBAC" | ||
} | ||
|
||
if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableDataEncryptionAtRest) { | ||
staticLinuxAPIServerConfig["--experimental-encryption-provider-config"] = "/etc/kubernetes/encryption-config.yaml" | ||
} | ||
|
||
if o.KubernetesConfig.EnableAggregatedAPIs { | ||
staticLinuxAPIServerConfig["--requestheader-client-ca-file"] = "/etc/kubernetes/certs/proxy-ca.crt" | ||
staticLinuxAPIServerConfig["--proxy-client-cert-file"] = "/etc/kubernetes/certs/proxy.crt" | ||
staticLinuxAPIServerConfig["--proxy-client-key-file"] = "/etc/kubernetes/certs/proxy.key" | ||
staticLinuxAPIServerConfig["--requestheader-allowed-names"] = "" | ||
staticLinuxAPIServerConfig["--requestheader-extra-headers-prefix"] = "X-Remote-Extra-" | ||
staticLinuxAPIServerConfig["--requestheader-group-headers"] = "X-Remote-Group" | ||
staticLinuxAPIServerConfig["--requestheader-username-headers"] = "X-Remote-User" | ||
} | ||
|
||
if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { | ||
staticLinuxAPIServerConfig["--cloud-provider"] = "azure" | ||
staticLinuxAPIServerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" | ||
} | ||
|
||
if cs.Properties.HasAadProfile() { | ||
staticLinuxAPIServerConfig["--oidc-username-claim"] = "oid" | ||
staticLinuxAPIServerConfig["--oidc-client-id"] = "spn:" + cs.Properties.AADProfile.ServerAppID | ||
issuerHost := "sts.windows.net" | ||
if GetCloudTargetEnv(cs.Location) == "AzureChinaCloud" { | ||
issuerHost = "sts.chinacloudapi.cn" | ||
} | ||
staticLinuxAPIServerConfig["--oidc-issuer-url"] = "https://" + issuerHost + "/" | ||
} | ||
|
||
staticWindowsAPIServerConfig := make(map[string]string) | ||
for key, val := range staticLinuxAPIServerConfig { | ||
staticWindowsAPIServerConfig[key] = val | ||
} | ||
// Windows apiserver config overrides | ||
// TODO placeholder for specific config overrides for Windows clusters | ||
|
||
// Default apiserver config | ||
defaultAPIServerConfig := map[string]string{} | ||
|
||
// If no user-configurable apiserver config values exists, use the defaults | ||
if o.KubernetesConfig.APIServerConfig == nil { | ||
o.KubernetesConfig.APIServerConfig = defaultAPIServerConfig | ||
} else { | ||
for key, val := range defaultAPIServerConfig { | ||
// If we don't have a user-configurable apiserver config for each option | ||
if _, ok := o.KubernetesConfig.APIServerConfig[key]; !ok { | ||
// then assign the default value | ||
o.KubernetesConfig.APIServerConfig[key] = val | ||
} | ||
} | ||
} | ||
|
||
// We don't support user-configurable values for the following, | ||
// so any of the value assignments below will override user-provided values | ||
var overrideAPIServerConfig map[string]string | ||
if cs.Properties.HasWindows() { | ||
overrideAPIServerConfig = staticWindowsAPIServerConfig | ||
} else { | ||
overrideAPIServerConfig = staticLinuxAPIServerConfig | ||
} | ||
for key, val := range overrideAPIServerConfig { | ||
o.KubernetesConfig.APIServerConfig[key] = val | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Some argument like "--admission-control" and "--authorization-mode" takes a list of features, and probably should allow feature addition (as long as static config is a subset of the user input config), instead of override. It can be done in a later change if needed. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I agree this would be a useful addition to this surface area. Feature gates also are special, in that they essentially contain key/vals themselves. Yeah, I'd prefer a follow-up PR for that. |
||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Correcting copy/paste errors from a prior, related PR.