Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
167 lines (167 sloc) 5.25 KB
{
"objects": [
{
"role": "DataPipelineDefaultRole",
"subject": "Spark Job Success",
"name": "SuccessSnsAlarm",
"id": "ActionId_lUR6V",
"message": "Spark Job Success",
"type": "SnsAlarm",
"topicArn": "#{mySuccessSNSTopicARN}"
},
{
"period": "1 days",
"startDateTime": "2019-01-01T21:45:00",
"name": "Every 1 day",
"id": "DefaultSchedule",
"type": "Schedule"
},
{
"onFail": {
"ref": "ActionId_8cvBp"
},
"name": "EmrActivityObj",
"step": "#{myEmrStep}",
"runsOn": {
"ref": "EmrClusterObj"
},
"id": "EmrActivityObj",
"type": "EmrActivity",
"onSuccess": {
"ref": "ActionId_lUR6V"
}
},
{
"failureAndRerunMode": "CASCADE",
"schedule": {
"ref": "DefaultSchedule"
},
"resourceRole": "DataPipelineDefaultResourceRole",
"role": "DataPipelineDefaultRole",
"pipelineLogUri": "s3://PATH/TO/YOUR/BUCKET/logs",
"scheduleType": "cron",
"name": "Default",
"id": "Default"
},
{
"taskInstanceType": "#{myTaskInstanceType}",
"coreInstanceCount": "#{myCoreInstanceCount}",
"masterInstanceType": "#{myMasterInstanceType}",
"releaseLabel": "#{myEMRReleaseLabel}",
"type": "EmrCluster",
"terminateAfter": "50 Minutes",
"bootstrapAction": "#{myBootstrapAction}",
"taskInstanceCount": "#{myTaskInstanceCount}",
"name": "EmrClusterObj",
"coreInstanceType": "#{myCoreInstanceType}",
"keyPair": "#{myEC2KeyPair}",
"id": "EmrClusterObj",
"applications": [
"Spark",
"Ganglia"
]
},
{
"role": "DataPipelineDefaultRole",
"subject": "Spark Job Failure",
"name": "FailureSnsAlarm",
"id": "ActionId_8cvBp",
"type": "SnsAlarm",
"message": "Spark Job Failure",
"topicArn": "#{myFailureSNSTopicARN}"
}
],
"parameters": [
{
"helpText": "An existing EC2 key pair to SSH into the master node of the EMR cluster as the user \"hadoop\".",
"description": "EC2 key pair",
"optional": "true",
"id": "myEC2KeyPair",
"type": "String"
},
{
"helpLink": "https://docs.aws.amazon.com/console/datapipeline/emrsteps",
"watermark": "s3://myBucket/myPath/myStep.jar,firstArg,secondArg",
"helpText": "A step is a unit of work you submit to the cluster. You can specify one or more steps",
"description": "EMR step(s)",
"isArray": "true",
"id": "myEmrStep",
"type": "String"
},
{
"helpText": "Task instances run Hadoop tasks.",
"description": "Task node instance type",
"optional": "true",
"id": "myTaskInstanceType",
"type": "String"
},
{
"default": "m1.medium",
"helpText": "Core instances run Hadoop tasks and store data using the Hadoop Distributed File System (HDFS).",
"description": "Core node instance type",
"id": "myCoreInstanceType",
"type": "String"
},
{
"default": "emr-4.3.0",
"helpText": "Determines the base configuration of the instances in your cluster, including the Hadoop version.",
"description": "EMR Release Label",
"id": "myEMRReleaseLabel",
"type": "String"
},
{
"default": "2",
"description": "Core node instance count",
"id": "myCoreInstanceCount",
"type": "Integer"
},
{
"description": "Task node instance count",
"optional": "true",
"id": "myTaskInstanceCount",
"type": "Integer"
},
{
"helpLink": "https://docs.aws.amazon.com/console/datapipeline/emr_bootstrap_actions",
"helpText": "Bootstrap actions are scripts that are executed during setup before Hadoop starts on every cluster node.",
"description": "Bootstrap action(s)",
"isArray": "true",
"optional": "true",
"id": "myBootstrapAction",
"type": "String"
},
{
"default": "m1.medium",
"helpText": "The Master instance assigns Hadoop tasks to core and task nodes, and monitors their status.",
"description": "Master node instance type",
"id": "myMasterInstanceType",
"type": "String"
},
{
"helpLink": "https://docs.aws.amazon.com/sns/latest/dg/welcome.html",
"helpText": "SNS Topic to notify you of a successful run",
"description": "Success SNS Topic ARN",
"optional": "true",
"id": "mySuccessSNSTopicARN",
"type": "String"
},
{
"helpLink": "https://docs.aws.amazon.com/sns/latest/dg/welcome.html",
"helpText": "SNS Topic to notify you of a failed run",
"description": "Failure SNS Topic ARN",
"optional": "true",
"id": "myFailureSNSTopicARN",
"type": "String"
}
],
"values": {
"myEMRReleaseLabel": "emr-5.13.0",
"myMasterInstanceType": "r3.xlarge",
"myEC2KeyPair": "YOUR_EC2_KEY_PAIR",
"myEmrStep": "command-runner.jar,spark-submit,--deploy-mode,client,--master,yarn,--class,SparkJob,s3://PATH/TO/YOUR/BUCKET/SparkJobFatJar.jar",
"myCoreInstanceCount": "2",
"myCoreInstanceType": "r3.xlarge",
"mySuccessSNSTopicARN": "arn:aws:sns:us-east-1:############:SparkJobSuccess",
"myFailureSNSTopicARN": "arn:aws:sns:us-east-1:############:SparkJobFailure"
}
}
You can’t perform that action at this time.