emr-cluster-job-step-functions.json

{
   "StartAt":"Launch_EMR_Cluster",
   "States":{
      "Launch_EMR_Cluster":{
         "Type":"Task",
         "Resource":"arn:aws:states:::elasticmapreduce:createCluster.sync",
         "Parameters":{
            "Name":"StepFn-EMR-Hudi",
            "ServiceRole":"EMR_DefaultRole",
            "JobFlowRole":"EMR_EC2_DefaultRole",
            "EbsRootVolumeSize":10,
            "ReleaseLabel":"emr-6.4.0",
            "Applications":[
               {
                  "Name":"Hadoop"
               },
               {
                  "Name":"Spark"
               },
               {
                  "Name":"Hive"
               },
               {
                  "Name":"Livy"
               }
            ],
            "LogUri":"s3://<bucket-name>/emr/logs",
            "ManagedScalingPolicy":{
               "ComputeLimits":{
                  "MaximumCapacityUnits":2,
                  "MaximumCoreCapacityUnits":2,
                  "MaximumOnDemandCapacityUnits":2,
                  "MinimumCapacityUnits":1,
                  "UnitType":"InstanceFleetUnits"
               }
            },
            "VisibleToAllUsers":true,
            "Instances":{
               "KeepJobFlowAliveWhenNoSteps":true,
               "Ec2KeyName":"<key-pair-name>",
               "Ec2SubnetId":"<subnet-id>",
               "InstanceFleets":[
                  {
                     "InstanceFleetType":"MASTER",
                     "Name":"Master",
                     "TargetOnDemandCapacity":1,
                     "InstanceTypeConfigs":[
                        {
                           "InstanceType":"m5.xlarge"
                        }
                     ]
                  },
                  {
                     "InstanceFleetType":"CORE",
                     "TargetOnDemandCapacity":1,
                     "InstanceTypeConfigs":[
                        {
                           "InstanceType":"m5.xlarge"
                        }
                     ]
                  }
               ]
            }
         },
         "Next":"Copy_Hudi_JARs"
      },
      "Copy_Hudi_JARs":{
         "Type":"Task",
         "ResultPath":"$.Result",
         "Catch":[
            {
               "ErrorEquals":[
                  "States.ALL"
               ],
               "ResultPath":"$.error-info",
               "Next":"Terminate_EMR_Cluster"
            }
         ],
         "Resource":"arn:aws:states:::elasticmapreduce:addStep.sync",
         "Parameters":{
            "ClusterId.$":"$.ClusterId",
            "Step":{
               "Name":"Copy JAR files",
               "ActionOnFailure":"CONTINUE",
               "HadoopJarStep":{
                  "Jar":"s3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar",
                  "Args":[
                     "s3://<bucket-name>/<script-path>/bootstrap.sh"
                  ]
               }
            }
         },
         "Next":"Trigger_Spark_Job"
      },
      "Trigger_Spark_Job":{
         "Type":"Task",
         "ResultPath":"$.Result",
         "Catch":[
            {
               "ErrorEquals":[
                  "States.ALL"
               ],
               "ResultPath":"$.error-info",
               "Next":"Terminate_EMR_Cluster"
            }
         ],
         "Resource":"arn:aws:states:::elasticmapreduce:addStep.sync",
         "Parameters":{
            "ClusterId.$":"$.ClusterId",
            "Step":{
               "Name":"Spark Transform Step",
               "ActionOnFailure":"CONTINUE",
               "HadoopJarStep":{
                  "Jar":"command-runner.jar",
                  "Args":[
                     "spark-submit",
                     "--deploy-mode",
                     "cluster",
                     "--jars",
                     "/usr/lib/hudi/hudi-spark-bundle.jar",
                     "--conf",
                     "spark.serializer=org.apache.spark.serializer.KryoSerializer",
                     "s3://<bucket-name>/<script-path-name>.py"
                  ]
               }
            }
         },
         "Next":"Terminate_EMR_Cluster"
      },
      "Terminate_EMR_Cluster":{
         "Type":"Task",
         "Resource":"arn:aws:states:::elasticmapreduce:terminateCluster.sync",
         "Parameters":{
            "ClusterId.$":"$.ClusterId"
         },
         "End":true
      }
   }
}