This repository has been archived by the owner on Oct 24, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 523
/
armvariables.go
541 lines (480 loc) · 33.6 KB
/
armvariables.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license.
package engine
import (
"encoding/base64"
"encoding/json"
"fmt"
"strconv"
"github.com/Azure/aks-engine/pkg/api"
"github.com/Azure/aks-engine/pkg/helpers"
"github.com/Azure/go-autorest/autorest/to"
)
func GetKubernetesVariables(cs *api.ContainerService) (map[string]interface{}, error) {
k8sVars := map[string]interface{}{}
profiles := cs.Properties.AgentPoolProfiles
for i := 0; i < len(profiles); i++ {
profile := profiles[i]
k8sVars[fmt.Sprintf("%sIndex", profile.Name)] = i
agentVars := getK8sAgentVars(cs, profile)
for k, v := range agentVars {
k8sVars[k] = v
}
if profile.IsStorageAccount() {
if profile.HasDisks() {
k8sVars[fmt.Sprintf("%sDataAccountName", profile.Name)] = fmt.Sprintf("[concat(variables('storageAccountBaseName'), 'data%d')]", i)
}
k8sVars[fmt.Sprintf("%sAccountName", profile.Name)] = fmt.Sprintf("[concat(variables('storageAccountBaseName'), 'agnt%d')]", i)
}
}
masterVars, err := getK8sMasterVars(cs)
if err != nil {
return k8sVars, err
}
for k, v := range masterVars {
k8sVars[k] = v
}
return k8sVars, nil
}
func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error) {
orchProfile := cs.Properties.OrchestratorProfile
kubernetesConfig := orchProfile.KubernetesConfig
masterProfile := cs.Properties.MasterProfile
profiles := cs.Properties.AgentPoolProfiles
var useManagedIdentity, userAssignedID, userAssignedClientID, enableEncryptionWithExternalKms bool
var excludeMasterFromStandardLB, provisionJumpbox bool
var maxLoadBalancerCount int
var useInstanceMetadata *bool
if kubernetesConfig != nil {
useManagedIdentity = kubernetesConfig.UseManagedIdentity
userAssignedID = useManagedIdentity && kubernetesConfig.UserAssignedID != ""
userAssignedClientID = useManagedIdentity && kubernetesConfig.UserAssignedClientID != ""
enableEncryptionWithExternalKms = to.Bool(kubernetesConfig.EnableEncryptionWithExternalKms)
useInstanceMetadata = kubernetesConfig.UseInstanceMetadata
excludeMasterFromStandardLB = to.Bool(kubernetesConfig.ExcludeMasterFromStandardLB)
maxLoadBalancerCount = kubernetesConfig.MaximumLoadBalancerRuleCount
provisionJumpbox = kubernetesConfig.PrivateJumpboxProvision()
}
isHostedMaster := cs.Properties.IsHostedMasterProfile()
isMasterVMSS := masterProfile != nil && masterProfile.IsVirtualMachineScaleSets()
hasStorageAccountDisks := cs.Properties.HasStorageAccountDisks()
isCustomVnet := cs.Properties.AreAgentProfilesCustomVNET()
hasAgentPool := len(profiles) > 0
hasCosmosEtcd := masterProfile != nil && to.Bool(masterProfile.CosmosEtcd)
kubernetesVersion := orchProfile.OrchestratorVersion
if cs.Properties.IsAzureStackCloud() {
kubernetesVersion = orchProfile.OrchestratorVersion + AzureStackSuffix
}
masterVars := map[string]interface{}{
"maxVMsPerPool": 100,
"useManagedIdentityExtension": strconv.FormatBool(useManagedIdentity),
"userAssignedIDReference": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('userAssignedID'))]",
"useInstanceMetadata": strconv.FormatBool(to.Bool(useInstanceMetadata)),
"loadBalancerSku": kubernetesConfig.LoadBalancerSku,
"excludeMasterFromStandardLB": strconv.FormatBool(excludeMasterFromStandardLB),
"maximumLoadBalancerRuleCount": maxLoadBalancerCount,
"masterFqdnPrefix": "[tolower(parameters('masterEndpointDNSNamePrefix'))]",
"apiVersionCompute": "2018-10-01",
"apiVersionDeployments": "2018-06-01",
"apiVersionStorage": "2018-07-01",
"apiVersionKeyVault": "2018-02-14",
"apiVersionNetwork": "2018-08-01",
"apiVersionManagedIdentity": "2015-08-31-preview",
"apiVersionAuthorizationUser": "2018-09-01-preview",
"apiVersionAuthorizationSystem": "2018-01-01-preview",
"locations": []string{
"[resourceGroup().location]",
"[parameters('location')]",
},
"location": "[variables('locations')[mod(add(2,length(parameters('location'))),add(1,length(parameters('location'))))]]",
"masterAvailabilitySet": "[concat('master-availabilityset-', parameters('nameSuffix'))]",
"resourceGroup": "[resourceGroup().name]",
"truncatedResourceGroup": "[take(replace(replace(resourceGroup().name, '(', '-'), ')', '-'), 63)]",
"labelResourceGroup": "[if(or(or(endsWith(variables('truncatedResourceGroup'), '-'), endsWith(variables('truncatedResourceGroup'), '_')), endsWith(variables('truncatedResourceGroup'), '.')), concat(take(variables('truncatedResourceGroup'), 62), 'z'), variables('truncatedResourceGroup'))]",
"routeTableID": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]",
"sshNatPorts": []int{22, 2201, 2202, 2203, 2204},
"sshKeyPath": "[concat('/home/',parameters('linuxAdminUsername'),'/.ssh/authorized_keys')]",
"provisionScriptParametersCommon": fmt.Sprintf("[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' CONTAINERD_VERSION=',parameters('containerdVersion'),' MOBY_VERSION=',parameters('mobyVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION=%s HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' MAXIMUM_LOADBALANCER_RULE_COUNT=',variables('maximumLoadBalancerRuleCount'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER=%t',' PRIVATE_AZURE_REGISTRY_SERVER=',parameters('privateAzureRegistryServer'),' AUTHENTICATION_METHOD=',variables('customCloudAuthenticationMethod'),' IDENTITY_SYSTEM=',variables('customCloudIdentifySystem'))]",
kubernetesVersion, isHostedMaster),
"orchestratorNameVersionTag": fmt.Sprintf("%s:%s", orchProfile.OrchestratorType, orchProfile.OrchestratorVersion),
"subnetNameResourceSegmentIndex": 10,
"vnetNameResourceSegmentIndex": 8,
"vnetResourceGroupNameResourceSegmentIndex": 4,
}
masterVars["cloudInitFiles"] = map[string]interface{}{
"provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript),
"provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript),
"provisionInstalls": getBase64EncodedGzippedCustomScript(kubernetesCSEInstall),
"provisionConfigs": getBase64EncodedGzippedCustomScript(kubernetesCSEConfig),
"provisionCIS": getBase64EncodedGzippedCustomScript(kubernetesCISScript),
"healthMonitorScript": getBase64EncodedGzippedCustomScript(kubernetesHealthMonitorScript),
"customSearchDomainsScript": getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript),
"generateProxyCertsScript": getBase64EncodedGzippedCustomScript(kubernetesMasterGenerateProxyCertsScript),
"mountEtcdScript": getBase64EncodedGzippedCustomScript(kubernetesMountEtcd),
"kubeletSystemdService": getBase64EncodedGzippedCustomScript(kubeletSystemdService),
"kmsSystemdService": getBase64EncodedGzippedCustomScript(kmsSystemdService),
"kubeletMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesKubeletMonitorSystemdService),
"dockerMonitorSystemdTimer": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdTimer),
"dockerMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdService),
"aptPreferences": getBase64EncodedGzippedCustomScript(aptPreferences),
"dockerClearMountPropagationFlags": getBase64EncodedGzippedCustomScript(dockerClearMountPropagationFlags),
"etcdSystemdService": getBase64EncodedGzippedCustomScript(etcdSystemdService),
}
blockOutboundInternet := cs.Properties.FeatureFlags.IsFeatureEnabled("BlockOutboundInternet")
var cosmosEndPointURI string
if hasCosmosEtcd {
cosmosEndPointURI = fmt.Sprintf("%sk8s.etcd.cosmosdb.azure.com", masterProfile.DNSPrefix)
masterVars["cosmosAccountName"] = fmt.Sprintf(etcdAccountNameFmt, cs.Properties.MasterProfile.DNSPrefix)
masterVars["cosmosDBCertb64"] = base64.StdEncoding.EncodeToString([]byte(cs.Properties.CertificateProfile.EtcdClientCertificate))
masterVars["apiVersionCosmos"] = "2015-04-08"
} else {
cosmosEndPointURI = ""
}
if cs.Properties.IsAzureStackCloud() {
masterVars["apiVersionCompute"] = "2017-03-30"
masterVars["apiVersionStorage"] = "2017-10-01"
masterVars["apiVersionNetwork"] = "2017-10-01"
masterVars["apiVersionKeyVault"] = "2016-10-01"
environmentJSON, err := cs.Properties.GetCustomEnvironmentJSON(false)
if err != nil {
return masterVars, err
}
masterVars["environmentJSON"] = environmentJSON
masterVars["provisionConfigsCustomCloud"] = getBase64EncodedGzippedCustomScript(kubernetesCSECustomCloud)
}
if kubernetesConfig != nil {
if kubernetesConfig.NetworkPlugin == NetworkPluginCilium {
masterVars["systemdBPFMount"] = getBase64EncodedGzippedCustomScript(systemdBPFMount)
}
}
masterVars["customCloudAuthenticationMethod"] = cs.Properties.GetCustomCloudAuthenticationMethod()
masterVars["customCloudIdentifySystem"] = cs.Properties.GetCustomCloudIdentitySystem()
auditDEnabled := "false"
if masterProfile != nil {
auditDEnabled = strconv.FormatBool(to.Bool(masterProfile.AuditDEnabled))
}
if !isHostedMaster {
if isMasterVMSS {
masterVars["provisionScriptParametersMaster"] = fmt.Sprintf("[concat('COSMOS_URI=%s MASTER_NODE=true NO_OUTBOUND=%t AUDITD_ENABLED=%s CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", cosmosEndPointURI, blockOutboundInternet, auditDEnabled)
} else {
masterVars["provisionScriptParametersMaster"] = fmt.Sprintf("[concat('COSMOS_URI=%s MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true NO_OUTBOUND=%t AUDITD_ENABLED=%s CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", cosmosEndPointURI, blockOutboundInternet, auditDEnabled)
}
}
if userAssignedID {
masterVars["userAssignedID"] = kubernetesConfig.UserAssignedID
} else {
masterVars["userAssignedID"] = ""
}
if userAssignedClientID {
masterVars["userAssignedClientID"] = kubernetesConfig.UserAssignedClientID
} else {
masterVars["userAssignedClientID"] = ""
}
if !isHostedMaster {
masterCount := masterProfile.Count
if masterCount == 1 {
masterVars["etcdPeerPrivateKeys"] = []string{"[parameters('etcdPeerPrivateKey0')]"}
masterVars["etcdPeerCertificates"] = []string{"[parameters('etcdPeerCertificate0')]"}
} else if masterCount == 3 {
masterVars["etcdPeerPrivateKeys"] = []string{
"[parameters('etcdPeerPrivateKey0')]",
"[parameters('etcdPeerPrivateKey1')]",
"[parameters('etcdPeerPrivateKey2')]",
}
masterVars["etcdPeerCertificates"] = []string{
"[parameters('etcdPeerCertificate0')]",
"[parameters('etcdPeerCertificate1')]",
"[parameters('etcdPeerCertificate2')]",
}
} else if masterCount == 5 {
masterVars["etcdPeerPrivateKeys"] = []string{
"[parameters('etcdPeerPrivateKey0')]",
"[parameters('etcdPeerPrivateKey1')]",
"[parameters('etcdPeerPrivateKey2')]",
"[parameters('etcdPeerPrivateKey3')]",
"[parameters('etcdPeerPrivateKey4')]",
}
masterVars["etcdPeerCertificates"] = []string{
"[parameters('etcdPeerCertificate0')]",
"[parameters('etcdPeerCertificate1')]",
"[parameters('etcdPeerCertificate2')]",
"[parameters('etcdPeerCertificate3')]",
"[parameters('etcdPeerCertificate4')]",
}
}
masterVars["etcdPeerCertFilepath"] = []string{
"/etc/kubernetes/certs/etcdpeer0.crt",
"/etc/kubernetes/certs/etcdpeer1.crt",
"/etc/kubernetes/certs/etcdpeer2.crt",
"/etc/kubernetes/certs/etcdpeer3.crt",
"/etc/kubernetes/certs/etcdpeer4.crt",
}
masterVars["etcdPeerKeyFilepath"] = []string{
"/etc/kubernetes/certs/etcdpeer0.key",
"/etc/kubernetes/certs/etcdpeer1.key",
"/etc/kubernetes/certs/etcdpeer2.key",
"/etc/kubernetes/certs/etcdpeer3.key",
"/etc/kubernetes/certs/etcdpeer4.key",
}
masterVars["etcdCaFilepath"] = "/etc/kubernetes/certs/ca.crt"
masterVars["etcdClientCertFilepath"] = "/etc/kubernetes/certs/etcdclient.crt"
masterVars["etcdClientKeyFilepath"] = "/etc/kubernetes/certs/etcdclient.key"
masterVars["etcdServerCertFilepath"] = "/etc/kubernetes/certs/etcdserver.crt"
masterVars["etcdServerKeyFilepath"] = "/etc/kubernetes/certs/etcdserver.key"
}
if useManagedIdentity {
masterVars["servicePrincipalClientId"] = "msi"
masterVars["servicePrincipalClientSecret"] = "msi"
} else {
masterVars["servicePrincipalClientId"] = "[parameters('servicePrincipalClientId')]"
masterVars["servicePrincipalClientSecret"] = "[parameters('servicePrincipalClientSecret')]"
}
if !isHostedMaster {
masterVars["masterCount"] = masterProfile.Count
if isMasterVMSS {
masterVars["masterOffset"] = ""
masterVars["masterIpAddressCount"] = masterProfile.IPAddressCount
} else {
masterVars["masterOffset"] = "[parameters('masterOffset')]"
}
if masterProfile.IsCustomVNET() {
if masterProfile.IsVirtualMachineScaleSets() {
masterVars["vnetSubnetID"] = "[parameters('agentVnetSubnetID')]"
masterVars["vnetSubnetIDMaster"] = "[parameters('masterVnetSubnetID')]"
} else {
masterVars["vnetSubnetID"] = "[parameters('masterVnetSubnetID')]"
}
masterVars["subnetName"] = "[split(parameters('masterVnetSubnetID'), '/')[variables('subnetNameResourceSegmentIndex')]]"
masterVars["virtualNetworkName"] = "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetNameResourceSegmentIndex')]]"
masterVars["virtualNetworkResourceGroupName"] = "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetResourceGroupNameResourceSegmentIndex')]]"
} else {
if masterProfile.IsVirtualMachineScaleSets() {
masterVars["subnetName"] = "subnetmaster"
masterVars["vnetSubnetID"] = "[concat(variables('vnetID'),'/subnets/subnetagent')]"
masterVars["vnetSubnetIDMaster"] = "[concat(variables('vnetID'),'/subnets/subnetmaster')]"
} else {
masterVars["subnetName"] = "[concat(parameters('orchestratorName'), '-subnet')]"
masterVars["vnetSubnetID"] = "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]"
}
masterVars["virtualNetworkName"] = "[concat(parameters('orchestratorName'), '-vnet-', parameters('nameSuffix'))]"
masterVars["vnetID"] = "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]"
masterVars["virtualNetworkResourceGroupName"] = "''"
}
masterVars["routeTableName"] = "[concat(variables('masterVMNamePrefix'),'routetable')]"
if masterProfile.IsStorageAccount() {
masterVars["masterStorageAccountName"] = "[concat(variables('storageAccountBaseName'), 'mstr0')]"
}
masterVars["nsgName"] = "[concat(variables('masterVMNamePrefix'), 'nsg')]"
} else {
if isCustomVnet {
if hasAgentPool {
masterVars["vnetSubnetID"] = fmt.Sprintf("[parameters('%sVnetSubnetID')]", profiles[0].Name)
}
masterVars["subnetName"] = "[split(variables('vnetSubnetID'), '/')[variables('subnetNameResourceSegmentIndex')]]"
masterVars["virtualNetworkName"] = "[split(variables('vnetSubnetID'), '/')[variables('vnetNameResourceSegmentIndex')]]"
masterVars["virtualNetworkResourceGroupName"] = "[split(variables('vnetSubnetID'), '/')[variables('vnetResourceGroupNameResourceSegmentIndex')]]"
} else {
masterVars["subnetName"] = "[concat(parameters('orchestratorName'), '-subnet')]"
masterVars["vnetID"] = "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]"
masterVars["vnetSubnetID"] = "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]"
masterVars["virtualNetworkName"] = "[concat(parameters('orchestratorName'), '-vnet-', parameters('nameSuffix'))]"
masterVars["virtualNetworkResourceGroupName"] = ""
}
masterVars["nsgName"] = "[concat(variables('agentNamePrefix'), 'nsg')]"
masterVars["routeTableName"] = "[concat(variables('agentNamePrefix'), 'routetable')]"
}
masterVars["nsgID"] = "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]"
if hasStorageAccountDisks {
masterVars["maxVMsPerStorageAccount"] = 20
masterVars["maxStorageAccountsPerAgent"] = "[div(variables('maxVMsPerPool'),variables('maxVMsPerStorageAccount'))]"
masterVars["dataStorageAccountPrefixSeed"] = 97
masterVars["storageAccountPrefixes"] = []string{"0", "6", "c", "i", "o", "u", "1", "7", "d", "j", "p", "v", "2", "8", "e", "k", "q", "w", "3", "9", "f", "l", "r", "x", "4", "a", "g", "m", "s", "y", "5", "b", "h", "n", "t", "z"}
masterVars["storageAccountPrefixesCount"] = "[length(variables('storageAccountPrefixes'))]"
masterVars["vmsPerStorageAccount"] = 20
masterVars["storageAccountBaseName"] = "[uniqueString(concat(variables('masterFqdnPrefix'),variables('location')))]"
masterVars["vmSizesMap"] = getSizeMap()["vmSizesMap"]
} else {
masterVars["storageAccountPrefixes"] = []interface{}{}
masterVars["storageAccountBaseName"] = ""
}
if cs.Properties.AnyAgentUsesVirtualMachineScaleSets() {
masterVars["primaryAvailabilitySetName"] = ""
masterVars["vmType"] = "vmss"
} else {
if hasAgentPool {
masterVars["primaryAvailabilitySetName"] = fmt.Sprintf("[concat('%s-availabilitySet-',parameters('nameSuffix'))]", profiles[0].Name)
} else {
masterVars["primaryAvailabilitySetName"] = ""
}
masterVars["vmType"] = "standard"
}
masterVars["primaryScaleSetName"] = cs.Properties.GetPrimaryScaleSetName()
if isHostedMaster {
masterVars["kubernetesAPIServerIP"] = "[parameters('kubernetesEndpoint')]"
masterVars["agentNamePrefix"] = "[concat(parameters('orchestratorName'), '-agentpool-', parameters('nameSuffix'), '-')]"
} else {
if cs.Properties.OrchestratorProfile.IsPrivateCluster() {
masterVars["kubeconfigServer"] = "[concat('https://', variables('kubernetesAPIServerIP'), ':443')]"
if provisionJumpbox {
masterVars["jumpboxOSDiskName"] = "[concat(parameters('jumpboxVMName'), '-osdisk')]"
masterVars["jumpboxPublicIpAddressName"] = "[concat(parameters('jumpboxVMName'), '-ip')]"
masterVars["jumpboxNetworkInterfaceName"] = "[concat(parameters('jumpboxVMName'), '-nic')]"
masterVars["jumpboxNetworkSecurityGroupName"] = "[concat(parameters('jumpboxVMName'), '-nsg')]"
kubeConfig, err := GenerateKubeConfig(cs.Properties, cs.Location)
if err != nil {
panic(err)
}
masterVars["kubeconfig"] = escapeSingleLine(kubeConfig)
isJumpboxManagedDisks := kubernetesConfig.PrivateJumpboxProvision() &&
kubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile == api.ManagedDisks
if !isJumpboxManagedDisks {
masterVars["jumpboxStorageAccountName"] = "[concat(variables('storageAccountBaseName'), 'jb')]"
}
if !cs.Properties.HasStorageAccountDisks() {
masterVars["vmSizesMap"] = getSizeMap()["vmSizesMap"]
}
}
} else {
masterVars["masterPublicIPAddressName"] = "[concat(parameters('orchestratorName'), '-master-ip-', variables('masterFqdnPrefix'), '-', parameters('nameSuffix'))]"
masterVars["masterLbID"] = "[resourceId('Microsoft.Network/loadBalancers',variables('masterLbName'))]"
masterVars["masterLbIPConfigID"] = "[concat(variables('masterLbID'),'/frontendIPConfigurations/', variables('masterLbIPConfigName'))]"
masterVars["masterLbIPConfigName"] = "[concat(parameters('orchestratorName'), '-master-lbFrontEnd-', parameters('nameSuffix'))]"
masterVars["masterLbName"] = "[concat(parameters('orchestratorName'), '-master-lb-', parameters('nameSuffix'))]"
masterVars["kubeconfigServer"] = "[concat('https://', variables('masterFqdnPrefix'), '.', variables('location'), '.', parameters('fqdnEndpointSuffix'))]"
}
if masterProfile.Count > 1 {
masterVars["masterInternalLbName"] = "[concat(parameters('orchestratorName'), '-master-internal-lb-', parameters('nameSuffix'))]"
masterVars["masterInternalLbID"] = "[resourceId('Microsoft.Network/loadBalancers',variables('masterInternalLbName'))]"
masterVars["masterInternalLbIPConfigName"] = "[concat(parameters('orchestratorName'), '-master-internal-lbFrontEnd-', parameters('nameSuffix'))]"
masterVars["masterInternalLbIPConfigID"] = "[concat(variables('masterInternalLbID'),'/frontendIPConfigurations/', variables('masterInternalLbIPConfigName'))]"
masterVars["masterInternalLbIPOffset"] = DefaultInternalLbStaticIPOffset
if isMasterVMSS {
masterVars["kubernetesAPIServerIP"] = "[concat(variables('masterFirstAddrOctets')[0],'.',variables('masterFirstAddrOctets')[1],'.255.', variables('masterInternalLbIPOffset'))]"
} else {
masterVars["kubernetesAPIServerIP"] = "[concat(variables('masterFirstAddrPrefix'), add(variables('masterInternalLbIPOffset'), int(variables('masterFirstAddrOctet4'))))]"
}
} else {
masterVars["kubernetesAPIServerIP"] = "[parameters('firstConsecutiveStaticIP')]"
}
masterVars["masterLbBackendPoolName"] = "[concat(parameters('orchestratorName'), '-master-pool-', parameters('nameSuffix'))]"
masterVars["masterFirstAddrComment"] = "these MasterFirstAddrComment are used to place multiple masters consecutively in the address space"
masterVars["masterFirstAddrOctets"] = "[split(parameters('firstConsecutiveStaticIP'),'.')]"
masterVars["masterFirstAddrOctet4"] = "[variables('masterFirstAddrOctets')[3]]"
masterVars["masterFirstAddrPrefix"] = "[concat(variables('masterFirstAddrOctets')[0],'.',variables('masterFirstAddrOctets')[1],'.',variables('masterFirstAddrOctets')[2],'.')]"
masterVars["masterEtcdServerPort"] = DefaultMasterEtcdServerPort
masterVars["masterEtcdClientPort"] = DefaultMasterEtcdClientPort
if isMasterVMSS {
masterVars["masterVMNamePrefix"] = "[concat(parameters('orchestratorName'), '-master-', parameters('nameSuffix'), '-')]"
} else {
masterVars["masterVMNamePrefix"] = cs.Properties.GetMasterVMPrefix()
masterVars["masterVMNames"] = []string{
"[concat(variables('masterVMNamePrefix'), '0')]",
"[concat(variables('masterVMNamePrefix'), '1')]",
"[concat(variables('masterVMNamePrefix'), '2')]",
"[concat(variables('masterVMNamePrefix'), '3')]",
"[concat(variables('masterVMNamePrefix'), '4')]",
}
masterVars["masterPrivateIpAddrs"] = []string{
"[concat(variables('masterFirstAddrPrefix'), add(0, int(variables('masterFirstAddrOctet4'))))]",
"[concat(variables('masterFirstAddrPrefix'), add(1, int(variables('masterFirstAddrOctet4'))))]",
"[concat(variables('masterFirstAddrPrefix'), add(2, int(variables('masterFirstAddrOctet4'))))]",
"[concat(variables('masterFirstAddrPrefix'), add(3, int(variables('masterFirstAddrOctet4'))))]",
"[concat(variables('masterFirstAddrPrefix'), add(4, int(variables('masterFirstAddrOctet4'))))]",
}
masterVars["masterEtcdPeerURLs"] = []string{
"[concat('https://', variables('masterPrivateIpAddrs')[0], ':', variables('masterEtcdServerPort'))]",
"[concat('https://', variables('masterPrivateIpAddrs')[1], ':', variables('masterEtcdServerPort'))]",
"[concat('https://', variables('masterPrivateIpAddrs')[2], ':', variables('masterEtcdServerPort'))]",
"[concat('https://', variables('masterPrivateIpAddrs')[3], ':', variables('masterEtcdServerPort'))]",
"[concat('https://', variables('masterPrivateIpAddrs')[4], ':', variables('masterEtcdServerPort'))]",
}
masterVars["masterEtcdClientURLs"] = []string{
"[concat('https://', variables('masterPrivateIpAddrs')[0], ':', variables('masterEtcdClientPort'))]",
"[concat('https://', variables('masterPrivateIpAddrs')[1], ':', variables('masterEtcdClientPort'))]",
"[concat('https://', variables('masterPrivateIpAddrs')[2], ':', variables('masterEtcdClientPort'))]",
"[concat('https://', variables('masterPrivateIpAddrs')[3], ':', variables('masterEtcdClientPort'))]",
"[concat('https://', variables('masterPrivateIpAddrs')[4], ':', variables('masterEtcdClientPort'))]",
}
masterVars["masterEtcdClusterStates"] = []string{
"[concat(variables('masterVMNames')[0], '=', variables('masterEtcdPeerURLs')[0])]",
"[concat(variables('masterVMNames')[0], '=', variables('masterEtcdPeerURLs')[0], ',', variables('masterVMNames')[1], '=', variables('masterEtcdPeerURLs')[1], ',', variables('masterVMNames')[2], '=', variables('masterEtcdPeerURLs')[2])]",
"[concat(variables('masterVMNames')[0], '=', variables('masterEtcdPeerURLs')[0], ',', variables('masterVMNames')[1], '=', variables('masterEtcdPeerURLs')[1], ',', variables('masterVMNames')[2], '=', variables('masterEtcdPeerURLs')[2], ',', variables('masterVMNames')[3], '=', variables('masterEtcdPeerURLs')[3], ',', variables('masterVMNames')[4], '=', variables('masterEtcdPeerURLs')[4])]",
}
}
}
masterVars["subscriptionId"] = "[subscription().subscriptionId]"
masterVars["contributorRoleDefinitionId"] = "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]"
masterVars["readerRoleDefinitionId"] = "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]"
masterVars["scope"] = "[resourceGroup().id]"
masterVars["tenantId"] = "[subscription().tenantId]"
masterVars["singleQuote"] = "'"
if cs.Properties.HasWindows() {
masterVars["windowsCustomScriptSuffix"] = " $inputFile = '%SYSTEMDRIVE%\\AzureData\\CustomData.bin' ; $outputFile = '%SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.ps1' ; Copy-Item $inputFile $outputFile ; Invoke-Expression('{0} {1}' -f $outputFile, $arguments) ; "
}
if enableEncryptionWithExternalKms {
masterVars["clusterKeyVaultName"] = "[take(concat('kv', tolower(uniqueString(concat(variables('masterFqdnPrefix'),variables('location'),parameters('nameSuffix'))))), 22)]"
} else {
masterVars["clusterKeyVaultName"] = ""
}
return masterVars, nil
}
func getK8sAgentVars(cs *api.ContainerService, profile *api.AgentPoolProfile) map[string]interface{} {
agentVars := map[string]interface{}{}
agentName := profile.Name
storageAccountOffset := fmt.Sprintf("%sStorageAccountOffset", agentName)
storageAccountsCount := fmt.Sprintf("%sStorageAccountsCount", agentName)
agentsCount := fmt.Sprintf("%sCount", agentName)
agentsVMNamePrefix := fmt.Sprintf("%sVMNamePrefix", agentName)
agentOffset := fmt.Sprintf("%sOffset", agentName)
agentAvailabilitySet := fmt.Sprintf("%sAvailabilitySet", agentName)
agentScaleSetPriority := fmt.Sprintf("%sScaleSetPriority", agentName)
agentScaleSetEvictionPolicy := fmt.Sprintf("%sScaleSetEvictionPolicy", agentName)
agentVMSize := fmt.Sprintf("%sVMSize", agentName)
agentVnetSubnetID := fmt.Sprintf("%sVnetSubnetID", agentName)
agentSubnetName := fmt.Sprintf("%sSubnetName", agentName)
agentVnetParts := fmt.Sprintf("%sVnetParts", agentName)
agentOsImageOffer := fmt.Sprintf("%sosImageOffer", agentName)
agentOsImageSku := fmt.Sprintf("%sosImageSKU", agentName)
agentOsImagePublisher := fmt.Sprintf("%sosImagePublisher", agentName)
agentOsImageVersion := fmt.Sprintf("%sosImageVersion", agentName)
agentOsImageName := fmt.Sprintf("%sosImageName", agentName)
agentOsImageResourceGroup := fmt.Sprintf("%sosImageResourceGroup", agentName)
if profile.IsStorageAccount() {
agentVars[storageAccountOffset] = fmt.Sprintf("[mul(variables('maxStorageAccountsPerAgent'),variables('%sIndex'))]", agentName)
agentVars[storageAccountsCount] = fmt.Sprintf("[add(div(variables('%[1]sCount'), variables('maxVMsPerStorageAccount')), mod(add(mod(variables('%[1]sCount'), variables('maxVMsPerStorageAccount')),2), add(mod(variables('%[1]sCount'), variables('maxVMsPerStorageAccount')),1)))]", agentName)
}
agentVars[agentsCount] = fmt.Sprintf("[parameters('%s')]", agentsCount)
agentVars[agentsVMNamePrefix] = cs.Properties.GetAgentVMPrefix(profile, cs.Properties.GetAgentPoolIndexByName(agentName))
if profile.IsWindows() {
agentVars["winResourceNamePrefix"] = "[substring(parameters('nameSuffix'), 0, 5)]"
}
if profile.IsAvailabilitySets() {
agentVars[agentOffset] = fmt.Sprintf("[parameters('%s')]", agentOffset)
agentVars[agentAvailabilitySet] = fmt.Sprintf("[concat('%s-availabilitySet-', parameters('nameSuffix'))]", agentName)
} else if profile.IsLowPriorityScaleSet() {
agentVars[agentScaleSetPriority] = fmt.Sprintf("[parameters('%s')]", agentScaleSetPriority)
agentVars[agentScaleSetEvictionPolicy] = fmt.Sprintf("[parameters('%s')]", agentScaleSetEvictionPolicy)
}
agentVars[agentVMSize] = fmt.Sprintf("[parameters('%s')]", agentVMSize)
if profile.IsCustomVNET() {
agentVars[agentVnetSubnetID] = fmt.Sprintf("[parameters('%s')]", agentVnetSubnetID)
agentVars[agentSubnetName] = fmt.Sprintf("[parameters('%s')]", agentVnetSubnetID)
agentVars[agentVnetParts] = fmt.Sprintf("[split(parameters('%sVnetSubnetID'),'/subnets/')]", agentName)
} else {
agentVars[agentVnetSubnetID] = fmt.Sprintf("[variables('vnetSubnetID')]")
agentVars[agentSubnetName] = fmt.Sprintf("[variables('subnetName')]")
}
agentVars[agentOsImageOffer] = fmt.Sprintf("[parameters('%sosImageOffer')]", agentName)
agentVars[agentOsImageSku] = fmt.Sprintf("[parameters('%sosImageSKU')]", agentName)
agentVars[agentOsImagePublisher] = fmt.Sprintf("[parameters('%sosImagePublisher')]", agentName)
agentVars[agentOsImageVersion] = fmt.Sprintf("[parameters('%sosImageVersion')]", agentName)
agentVars[agentOsImageName] = fmt.Sprintf("[parameters('%sosImageName')]", agentName)
agentVars[agentOsImageResourceGroup] = fmt.Sprintf("[parameters('%sosImageResourceGroup')]", agentName)
return agentVars
}
func getSizeMap() map[string]interface{} {
var sizeMap map[string]interface{}
sizeMapStr := fmt.Sprintf("{%s}", helpers.GetSizeMap())
json.Unmarshal([]byte(sizeMapStr), &sizeMap)
return sizeMap
}