/
master_node.json
204 lines (204 loc) · 8.86 KB
/
master_node.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"apiProfile": "2017-03-09-profile",
"parameters": {
"clusterName": {
"type": "string"
},
"adminUsername": {
"type": "string"
},
"adminPassword": {
"type": "securestring"
},
"vmSize": {
"type": "string"
},
"storageAccountName": {
"type": "string"
},
"networkSecurityGroupName": {
"type": "string"
},
"virtualNetworkName": {
"type": "string"
},
"subnetName": {
"type": "string"
},
"loadBalancerName": {
"type": "string"
},
"numberWorkerNodes": {
"type": "int"
},
"replication" : {
"type" : "int"
},
"_artifactsLocation": {
"type": "string"
}
},
"variables": {
"defaultName": "[parameters('clusterName')]",
"imagePublisher": "Canonical",
"imageOffer": "UbuntuServer",
"imageSku": "16.04-LTS",
"location": "[resourceGroup().location]",
"nicName": "[concat(variables('defaultName'),'nic')]",
"vmName": "[variables('defaultName')]",
"vnetID": "[resourceId('Microsoft.Network/virtualNetworks',parameters('virtualNetworkName'))]",
"nsgID": "[resourceId('Microsoft.Network/networkSecurityGroups',parameters('networkSecurityGroupName'))]",
"subnetRef": "[concat(variables('vnetID'),'/subnets/',parameters('subnetName'))]",
"loadBalancerId": "[resourceId('Microsoft.Network/loadBalancers',parameters('loadBalancerName'))]",
"masters": [
"NameNode",
"ResourceManager",
"JobHistory"
],
"numberMasters": "[length(variables('masters'))]"
},
"resources": [
{
"comments": "Network interfaces for each master node. This has a public and private IP address.",
"type": "Microsoft.Network/networkInterfaces",
"name": "[concat(variables('nicName'),variables('masters')[copyIndex()])]",
"location": "[variables('location')]",
"copy": {
"name": "masterNics",
"count": "[variables('numberMasters')]"
},
"properties": {
"networkSecurityGroup": {
"id": "[variables('nsgID')]"
},
"ipConfigurations": [
{
"name": "[concat('ipConfig',variables('masters')[copyIndex()])]",
"properties": {
"privateIPAllocationMethod": "Dynamic",
"subnet": {
"id": "[variables('subnetRef')]"
},
"loadBalancerBackendAddressPools": [
{
"id": "[concat(variables('loadBalancerId'), '/backendAddressPools/LoadBalancerBackend')]"
}
],
"loadBalancerInboundNatRules": [
{
"id": "[concat(variables('loadBalancerId'),'/inboundNatRules/',variables('masters')[copyIndex()])]"
}
]
}
}
]
}
},
{
"comments": "These are the master nodes.",
"type": "Microsoft.Compute/virtualMachines",
"name": "[concat(variables('vmName'),variables('masters')[copyIndex()])]",
"location": "[variables('location')]",
"copy": {
"name": "masterNodes",
"count": "[variables('numberMasters')]"
},
"dependsOn": [
"masterNics"
],
"properties": {
"availabilitySet": {
"id": "[resourceId('Microsoft.Compute/availabilitySets','hadoop')]"
},
"hardwareProfile": {
"vmSize": "[parameters('vmSize')]"
},
"osProfile": {
"computerName": "[concat(parameters('clusterName'),variables('masters')[copyIndex()])]",
"adminUsername": "[parameters('adminUsername')]",
"adminPassword": "[parameters('adminPassword')]",
"linuxConfiguration": {
"disablePasswordAuthentication": false
}
},
"storageProfile": {
"imageReference": {
"publisher": "[variables('imagePublisher')]",
"offer": "[variables('imageOffer')]",
"sku": "[variables('imageSku')]",
"version": "latest"
},
"osDisk": {
"name": "[concat(parameters('clusterName'),variables('masters')[copyIndex()])]",
"vhd": {
"uri": "[concat(concat(reference(resourceId(resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2015-06-15').primaryEndpoints['blob'], concat('master',copyIndex(),'/')),'os.vhd')]"
},
"caching": "ReadWrite",
"createOption": "FromImage"
},
"dataDisks": [
{
"name": "[concat('workerdatadisk',copyIndex())]",
"diskSizeGB": "128",
"vhd": {
"uri": "[concat(concat(reference(resourceId(resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2015-06-15').primaryEndpoints['blob'], concat('master',copyIndex(),'/')),'dd.vhd')]"
},
"lun": 0,
"caching": "ReadWrite",
"createOption": "Empty"
}
]
},
"networkProfile": {
"networkInterfaces": [
{
"id": "[resourceId('Microsoft.Network/networkInterfaces',concat(variables('nicName'),variables('masters')[copyIndex()]))]"
}
]
},
"diagnosticsProfile": {
"bootDiagnostics": {
"enabled": "true",
"storageUri": "[reference(resourceId(resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2015-06-15').primaryEndpoints['blob']]"
}
}
}
},
{
"name": "[concat(parameters('clusterName'),variables('masters')[copyIndex()],'/masterCustomScript')]",
"type": "Microsoft.Compute/virtualMachines/extensions",
"location": "[resourceGroup().location]",
"dependsOn": [
"masterNodes"
],
"copy": {
"name": "masterHadoopSetup",
"count": "[variables('numberMasters')]"
},
"tags": {
"displayName": "hadoopSetup"
},
"properties": {
"publisher": "Microsoft.Azure.Extensions",
"type": "CustomScript",
"typeHandlerVersion": "2.0",
"autoUpgradeMinorVersion": "true",
"settings": {
"fileUris": [
"[concat(parameters('_artifactsLocation'),'/scripts/hadoopSetup.sh')]",
"[concat(parameters('_artifactsLocation'),'/scripts/template.service')]",
"[concat(parameters('_artifactsLocation'),'/config/sources.txt')]",
"[concat(parameters('_artifactsLocation'),'/config/core-site.xml')]",
"[concat(parameters('_artifactsLocation'),'/config/hdfs-site.xml')]",
"[concat(parameters('_artifactsLocation'),'/config/mapred-site.xml')]",
"[concat(parameters('_artifactsLocation'),'/config/yarn-site.xml')]"
],
"commandToExecute": "[concat('bash ./hadoopSetup.sh ',parameters('numberWorkerNodes'),' ',parameters('replication'))]"
}
}
}
],
"outputs": {}
}