-
Notifications
You must be signed in to change notification settings - Fork 4
/
emrConfig.json
97 lines (97 loc) · 2.38 KB
/
emrConfig.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
{
"emrRegion": "us-west-2",
"sparkJobConfig": {
"name": "Spark",
"actionOnFailure": "TERMINATE",
"jar": "command-runner.jar",
"args": ["spark-submit", "--master", "yarn", "--deploy-mode", "cluster", "--class", "your.package.name.Main", "arg1", "arg2", "arg3"]
},
"additionalInfo": "{\"instanceAwsClientConfiguration\": {\"proxyHost\": \"your.proxyHost.com\", \"proxyPort\": 8099}}",
"applications": [
{
"name": "Spark"
}
],
"autoScalingRole": "EMR_AutoScaling_DefaultRole",
"bootstrapActions": [
{
"name": "BootstrapAction1",
"scriptBootstrapAction": {
"path": "s3://yourBucket/path/to/bootstrapAction1.sh",
"args": ["arg1"]
}
},
{
"name": "BootstrapAction2",
"scriptBootstrapAction": {
"path": "s3://yourBucket/path/to/bootStrapAction2.sh"
}
}
],
"customAmiId": "12345",
"instances": {
"ec2SubnetIds": [
"subnet-12345"
],
"instanceGroups": [
{
"ebsConfiguration": {
"ebsBlockDeviceConfigs": [
{
"volumeSpecification": {
"sizeInGB": 10,
"volumeType": "gp2"
},
"volumesPerInstance": 1
}
]
},
"instanceCount": 1,
"instanceRole": "MASTER",
"instanceType": "m4.4xlarge",
"name": "MasterGroup"
},
{
"ebsConfiguration": {
"ebsBlockDeviceConfigs": [
{
"volumeSpecification": {
"sizeInGB": 10,
"volumeType": "gp2"
},
"volumesPerInstance": 1
}
]
},
"instanceCount": 5,
"instanceRole": "CORE",
"instanceType": "m4.4xlarge",
"name": "CoreGroup"
}
],
"keepJobFlowAliveWhenNoSteps": false,
"terminationProtected": false
},
"jobFlowRole": "Role-For-Spark-Job",
"logUri": "s3://yourBucket/path/to/emrLogs",
"name": "name-of-spark-job",
"releaseLabel": "emr-5.19.0",
"repoUpgradeOnBoot": "SECURITY",
"securityConfiguration": "Your-Security-Config",
"serviceRole": "EMR_DefaultRole",
"tags": [
{
"key": "Key1",
"value": "Value1"
},
{
"key": "Key2",
"value": "Value2"
},
{
"key": "OwnerContact",
"value": "yourEmail@example.com"
}
],
"visibleToAllUsers": true
}