Skip to content
This repository was archived by the owner on Jan 11, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions docs/clusterdefinition.md
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,40 @@ Below is a list of controller-manager options that are *not* currently user-conf
|"--profiling"|"false"|
|"--use-service-account-credentials"|"false" ("true" if kubernetesConfig.enableRbac is true)|

#### cloudControllerManagerConfig

`cloudControllerManagerConfig` declares runtime configuration for the cloud-controller-manager daemon running on all master nodes in a Cloud Controller Manager configuration. Like `kubeletConfig` it is a generic key/value object, and a child property of `kubernetesConfig`. An example custom cloud-controller-manager config:

```
"kubernetesConfig": {
"cloudControllerManagerConfig": {
"--route-reconciliation-period": "1m"
}
}
```

See [here](https://kubernetes.io/docs/reference/generated/cloud-controller-manager/) for a reference of supported controller-manager options.

Below is a list of cloud-controller-manager options that acs-engine will configure by default:

|controller-manager option|default value|
|---|---|
|"--route-reconciliation-period"|"10s"|


Below is a list of cloud-controller-manager options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces controller-manager configuration, or because a static configuration is required to build a functional cluster:

|controller-manager option|default value|
|---|---|
|"--kubeconfig"|"/var/lib/kubelet/kubeconfig"|
|"--allocate-node-cidrs"|"false"|
|"--cluster-cidr"|"10.240.0.0/12"|
|"--cluster-name"|<auto-generated using api model properties>|
|"--cloud-provider"|"azure"|
|"--cloud-config"|"/etc/kubernetes/azure.json"|
|"--leader-elect"|"true"|
|"--v"|"2"|

#### apiServerConfig

`apiServerConfig` declares runtime configuration for the kube-apiserver daemon running on all master nodes. Like `kubeletConfig` and `controllerManagerConfig` it is a generic key/value object, and a child property of `kubernetesConfig`. An example custom apiserver config:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.8",
"kubernetesConfig": {
"useCloudControllerManager": true
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be beneficial to put a CloudControllerManagerConfig field here for reference

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure I want to encourage someone to use this yet until we spend some time testing various custom configurations that actually work! :)

}
},
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v2"
},
"agentPoolProfiles": [
{
"name": "agentpool1",
"count": 3,
"vmSize": "Standard_D2_v2",
"availabilityProfile": "AvailabilitySet"
}
],
"linuxProfile": {
"adminUsername": "azureUser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
},
"servicePrincipalProfile": {
"clientId": "",
"secret": ""
}
}
}
5 changes: 2 additions & 3 deletions parts/k8s/kubernetesmastercustomdata.yml
Original file line number Diff line number Diff line change
Expand Up @@ -237,10 +237,9 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
# If Calico Policy enabled then update Cluster Cidr
sed -i "s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml"
{{end}}

{{if UseCloudControllerManager }}
sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s|<masterFqdnPrefix>|{{WrapAsVariable "masterFqdnPrefix"}}|g; s|<allocateNodeCidrs>|{{WrapAsVariable "allocateNodeCidrs"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g; s|<kubernetesCtrlMgrRouteReconciliationPeriod>|{{GetCloudControllerManagerRouteReconciliationPeriod .OrchestratorProfile.KubernetesConfig}}|g" \
/etc/kubernetes/manifests/cloud-controller-manager.yaml
sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsVariable "kubernetesCcmImageSpec"}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml"
sed -i "s|<kubernetesCloudControllerManagerConfig>|{{GetCloudControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml"
{{end}}
sed -i "s|<kubernetesControllerManagerConfig>|{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml"
sed -i "s|<kubernetesAPIServerConfig>|{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
Expand Down
14 changes: 2 additions & 12 deletions parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,8 @@ spec:
containers:
- name: "cloud-controller-manager"
image: "<kubernetesCcmImageSpec>"
command:
- "cloud-controller-manager"
- "--kubeconfig=/var/lib/kubelet/kubeconfig"
- "--allocate-node-cidrs=<allocateNodeCidrs>"
- "--cluster-cidr=<kubeClusterCidr>"
- "--cluster-name=<masterFqdnPrefix>"
- "--cloud-provider=azure"
- "--cloud-config=/etc/kubernetes/azure.json"
- "--leader-elect=true"
# TODO: RBAC support
- "--route-reconciliation-period=<kubernetesCtrlMgrRouteReconciliationPeriod>"
- "--v=2"
command: ["cloud-controller-manager"]
args: [<kubernetesCloudControllerManagerConfig>]
volumeMounts:
- name: "etc-kubernetes"
mountPath: "/etc/kubernetes"
Expand Down
69 changes: 69 additions & 0 deletions pkg/acsengine/defaults-cloud-controller-manager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package acsengine

import (
"strconv"

"github.com/Azure/acs-engine/pkg/api"
)

func setCloudControllerManagerConfig(cs *api.ContainerService) {
o := cs.Properties.OrchestratorProfile
staticLinuxCloudControllerManagerConfig := map[string]string{
"--allocate-node-cidrs": strconv.FormatBool(!o.IsAzureCNI()),
"--cloud-provider": "azure",
"--cloud-config": "/etc/kubernetes/azure.json",
"--cluster-cidr": o.KubernetesConfig.ClusterSubnet,
"--kubeconfig": "/var/lib/kubelet/kubeconfig",
"--leader-elect": "true",
"--v": "2",
}

// Set --cluster-name based on appropriate DNS prefix
if cs.Properties.MasterProfile != nil {
staticLinuxCloudControllerManagerConfig["--cluster-name"] = cs.Properties.MasterProfile.DNSPrefix
} else if cs.Properties.HostedMasterProfile != nil {
staticLinuxCloudControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix
}

staticWindowsCloudControllerManagerConfig := make(map[string]string)
for key, val := range staticLinuxCloudControllerManagerConfig {
staticWindowsCloudControllerManagerConfig[key] = val
}
// Windows cloud-controller-manager config overrides
// TODO placeholder for specific config overrides for Windows clusters

// Default cloud-controller-manager config
defaultCloudControllerManagerConfig := map[string]string{
"--node-monitor-grace-period": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From the md file, the default is "--route-reconciliation-period", a typo?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch! Actually, the doc is correct, it's an error in the defaultCloudControllerManagerConfig object! (see the changes to parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml). I'll fix.

}

// If no user-configurable cloud-controller-manager config values exists, use the defaults
if o.KubernetesConfig.CloudControllerManagerConfig == nil {
o.KubernetesConfig.CloudControllerManagerConfig = defaultCloudControllerManagerConfig
} else {
for key, val := range defaultCloudControllerManagerConfig {
// If we don't have a user-configurable cloud-controller-manager config for each option
if _, ok := o.KubernetesConfig.CloudControllerManagerConfig[key]; !ok {
// then assign the default value
o.KubernetesConfig.CloudControllerManagerConfig[key] = val
}
}
}

// We don't support user-configurable values for the following,
// so any of the value assignments below will override user-provided values
var overrideCloudControllerManagerConfig map[string]string
if cs.Properties.HasWindows() {
overrideCloudControllerManagerConfig = staticWindowsCloudControllerManagerConfig
} else {
overrideCloudControllerManagerConfig = staticLinuxCloudControllerManagerConfig
}
for key, val := range overrideCloudControllerManagerConfig {
o.KubernetesConfig.CloudControllerManagerConfig[key] = val
}

// TODO add RBAC support
/*if *o.KubernetesConfig.EnableRbac {
o.KubernetesConfig.CloudControllerManagerConfig["--use-service-account-credentials"] = "true"
}*/
}
2 changes: 2 additions & 0 deletions pkg/acsengine/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,8 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
setKubeletConfig(cs)
// Configure controller-manager
setControllerManagerConfig(cs)
// Configure cloud-controller-manager
setCloudControllerManagerConfig(cs)
// Configure apiserver
setAPIServerConfig(cs)

Expand Down
16 changes: 15 additions & 1 deletion pkg/acsengine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s
addValue(parametersMap, "kubernetesEndpoint", properties.HostedMasterProfile.FQDN)
}

if properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager != nil && *properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager {
if helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager) {
kubernetesCcmSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["ccm"]
if properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage != "" {
kubernetesCcmSpec = properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage
Expand Down Expand Up @@ -872,6 +872,20 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
}
return strings.TrimSuffix(buf.String(), ", ")
},
"GetCloudControllerManagerConfigKeyVals": func(kc *api.KubernetesConfig) string {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it necessary to pass in the KubernetesConfig here? For distinct agent/master configs it needs to be passed in but since this looks like it's using the orchestratorProfile's kubernetesConfig here you could also use cs.Properties.OrchestratorProfile.KubernetesConfig since the getTemplateFuncMap gets passed in a ContainerService

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm actually planning to submit a follow-up PR that generalizes the getter function so that it works on map[string]strings generically.

cloudControllerManagerConfig := kc.CloudControllerManagerConfig
// Order by key for consistency
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if this starts getting repeated in more configs (it's already in GetControllerManagerConfigKeyVals, GetKubeletConfigKeyVals and GetApiServerConfigKeyVals) maybe it's time to extract it to a function

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, yes! What I said above.

keys := []string{}
for key := range cloudControllerManagerConfig {
keys = append(keys, key)
}
sort.Strings(keys)
var buf bytes.Buffer
for _, key := range keys {
buf.WriteString(fmt.Sprintf("\\\"%s=%s\\\", ", key, cloudControllerManagerConfig[key]))
}
return strings.TrimSuffix(buf.String(), ", ")
},
"GetAPIServerConfigKeyVals": func(kc *api.KubernetesConfig) string {
apiServerConfig := kc.APIServerConfig
// Order by key for consistency
Expand Down
8 changes: 8 additions & 0 deletions pkg/api/converterfromapi.go
Original file line number Diff line number Diff line change
Expand Up @@ -674,6 +674,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
convertAddonsToVlabs(api, vlabs)
convertKubeletConfigToVlabs(api, vlabs)
convertControllerManagerConfigToVlabs(api, vlabs)
convertCloudControllerManagerConfigToVlabs(api, vlabs)
convertAPIServerConfigToVlabs(api, vlabs)
}

Expand All @@ -691,6 +692,13 @@ func convertControllerManagerConfigToVlabs(a *KubernetesConfig, v *vlabs.Kuberne
}
}

func convertCloudControllerManagerConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) {
v.CloudControllerManagerConfig = map[string]string{}
for key, val := range a.CloudControllerManagerConfig {
v.CloudControllerManagerConfig[key] = val
}
}

func convertAPIServerConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) {
v.APIServerConfig = map[string]string{}
for key, val := range a.APIServerConfig {
Expand Down
8 changes: 8 additions & 0 deletions pkg/api/convertertoapi.go
Original file line number Diff line number Diff line change
Expand Up @@ -618,6 +618,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
convertAddonsToAPI(vlabs, api)
convertKubeletConfigToAPI(vlabs, api)
convertControllerManagerConfigToAPI(vlabs, api)
convertCloudControllerManagerConfigToAPI(vlabs, api)
convertAPIServerConfigToAPI(vlabs, api)
}

Expand Down Expand Up @@ -675,6 +676,13 @@ func convertControllerManagerConfigToAPI(v *vlabs.KubernetesConfig, a *Kubernete
}
}

func convertCloudControllerManagerConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) {
a.CloudControllerManagerConfig = map[string]string{}
for key, val := range v.CloudControllerManagerConfig {
a.CloudControllerManagerConfig[key] = val
}
}

func convertAPIServerConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) {
a.APIServerConfig = map[string]string{}
for key, val := range v.APIServerConfig {
Expand Down
1 change: 1 addition & 0 deletions pkg/api/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ type KubernetesConfig struct {
Addons []KubernetesAddon `json:"addons,omitempty"`
KubeletConfig map[string]string `json:"kubeletConfig,omitempty"`
ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"`
CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"`
APIServerConfig map[string]string `json:"apiServerConfig,omitempty"`
}

Expand Down
1 change: 1 addition & 0 deletions pkg/api/vlabs/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ type KubernetesConfig struct {
Addons []KubernetesAddon `json:"addons,omitempty"`
KubeletConfig map[string]string `json:"kubeletConfig,omitempty"`
ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"`
CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"`
APIServerConfig map[string]string `json:"apiServerConfig,omitempty"`
}

Expand Down