datapipeline.json

{
  "objects": [
    {
      "role": "DataPipelineDefaultRole",
      "subject": "Spark Job Success",
      "name": "SuccessSnsAlarm",
      "id": "ActionId_lUR6V",
      "message": "Spark Job Success",
      "type": "SnsAlarm",
      "topicArn": "#{mySuccessSNSTopicARN}"
    },
    {
      "period": "1 days",
      "startDateTime": "2019-01-01T21:45:00",
      "name": "Every 1 day",
      "id": "DefaultSchedule",
      "type": "Schedule"
    },
    {
      "onFail": {
        "ref": "ActionId_8cvBp"
      },
      "name": "EmrActivityObj",
      "step": "#{myEmrStep}",
      "runsOn": {
        "ref": "EmrClusterObj"
      },
      "id": "EmrActivityObj",
      "type": "EmrActivity",
      "onSuccess": {
        "ref": "ActionId_lUR6V"
      }
    },
    {
      "failureAndRerunMode": "CASCADE",
      "schedule": {
        "ref": "DefaultSchedule"
      },
      "resourceRole": "DataPipelineDefaultResourceRole",
      "role": "DataPipelineDefaultRole",
      "pipelineLogUri": "s3://PATH/TO/YOUR/BUCKET/logs",
      "scheduleType": "cron",
      "name": "Default",
      "id": "Default"
    },
    {
      "taskInstanceType": "#{myTaskInstanceType}",
      "coreInstanceCount": "#{myCoreInstanceCount}",
      "masterInstanceType": "#{myMasterInstanceType}",
      "releaseLabel": "#{myEMRReleaseLabel}",
      "type": "EmrCluster",
      "terminateAfter": "50 Minutes",
      "bootstrapAction": "#{myBootstrapAction}",
      "taskInstanceCount": "#{myTaskInstanceCount}",
      "name": "EmrClusterObj",
      "coreInstanceType": "#{myCoreInstanceType}",
      "keyPair": "#{myEC2KeyPair}",
      "id": "EmrClusterObj",
      "applications": [
        "Spark",
        "Ganglia"
      ]
    },
    {
      "role": "DataPipelineDefaultRole",
      "subject": "Spark Job Failure",
      "name": "FailureSnsAlarm",
      "id": "ActionId_8cvBp",
      "type": "SnsAlarm",
      "message": "Spark Job Failure",
      "topicArn": "#{myFailureSNSTopicARN}"
    }
  ],
  "parameters": [
    {
      "helpText": "An existing EC2 key pair to SSH into the master node of the EMR cluster as the user \"hadoop\".",
      "description": "EC2 key pair",
      "optional": "true",
      "id": "myEC2KeyPair",
      "type": "String"
    },
    {
      "helpLink": "https://docs.aws.amazon.com/console/datapipeline/emrsteps",
      "watermark": "s3://myBucket/myPath/myStep.jar,firstArg,secondArg",
      "helpText": "A step is a unit of work you submit to the cluster. You can specify one or more steps",
      "description": "EMR step(s)",
      "isArray": "true",
      "id": "myEmrStep",
      "type": "String"
    },
    {
      "helpText": "Task instances run Hadoop tasks.",
      "description": "Task node instance type",
      "optional": "true",
      "id": "myTaskInstanceType",
      "type": "String"
    },
    {
      "default": "m1.medium",
      "helpText": "Core instances run Hadoop tasks and store data using the Hadoop Distributed File System (HDFS).",
      "description": "Core node instance type",
      "id": "myCoreInstanceType",
      "type": "String"
    },
    {
      "default": "emr-4.3.0",
      "helpText": "Determines the base configuration of the instances in your cluster, including the Hadoop version.",
      "description": "EMR Release Label",
      "id": "myEMRReleaseLabel",
      "type": "String"
    },
    {
      "default": "2",
      "description": "Core node instance count",
      "id": "myCoreInstanceCount",
      "type": "Integer"
    },
    {
      "description": "Task node instance count",
      "optional": "true",
      "id": "myTaskInstanceCount",
      "type": "Integer"
    },
    {
      "helpLink": "https://docs.aws.amazon.com/console/datapipeline/emr_bootstrap_actions",
      "helpText": "Bootstrap actions are scripts that are executed during setup before Hadoop starts on every cluster node.",
      "description": "Bootstrap action(s)",
      "isArray": "true",
      "optional": "true",
      "id": "myBootstrapAction",
      "type": "String"
    },
    {
      "default": "m1.medium",
      "helpText": "The Master instance assigns Hadoop tasks to core and task nodes, and monitors their status.",
      "description": "Master node instance type",
      "id": "myMasterInstanceType",
      "type": "String"
    },
    {
      "helpLink": "https://docs.aws.amazon.com/sns/latest/dg/welcome.html",
      "helpText": "SNS Topic to notify you of a successful run",
      "description": "Success SNS Topic ARN",
      "optional": "true",
      "id": "mySuccessSNSTopicARN",
      "type": "String"
    },
    {
      "helpLink": "https://docs.aws.amazon.com/sns/latest/dg/welcome.html",
      "helpText": "SNS Topic to notify you of a failed run",
      "description": "Failure SNS Topic ARN",
      "optional": "true",
      "id": "myFailureSNSTopicARN",
      "type": "String"
    }
  ],
  "values": {
    "myEMRReleaseLabel": "emr-5.13.0",
    "myMasterInstanceType": "r3.xlarge",
    "myEC2KeyPair": "YOUR_EC2_KEY_PAIR",
    "myEmrStep": "command-runner.jar,spark-submit,--deploy-mode,client,--master,yarn,--class,SparkJob,s3://PATH/TO/YOUR/BUCKET/SparkJobFatJar.jar",
    "myCoreInstanceCount": "2",
    "myCoreInstanceType": "r3.xlarge",
    "mySuccessSNSTopicARN": "arn:aws:sns:us-east-1:############:SparkJobSuccess",
    "myFailureSNSTopicARN": "arn:aws:sns:us-east-1:############:SparkJobFailure"
  }
}