-
Notifications
You must be signed in to change notification settings - Fork 11
/
sfn.yaml
53 lines (53 loc) · 1.99 KB
/
sfn.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
apiVersion: sfn.services.k8s.aws/v1alpha1
kind: StateMachine
metadata:
name: run-spark-job-ack
spec:
name: run-spark-job-ack
roleARN: "arn:aws:iam::xxxxxxxx:role/event-driven-pipeline-demo-sfn-execution-role" # replace with your stepfunctions_role_arn
tags:
- key: owner
value: sfn-ack
definition: |
{
"Comment": "A description of my state machine",
"StartAt": "input-output-s3",
"States": {
"input-output-s3": {
"Type": "Task",
"Resource": "arn:aws:states:::emr-containers:startJobRun.sync",
"Parameters": {
"VirtualClusterId": "f0u3vt3y4q2r1ot11m7v809y6",
"ExecutionRoleArn": "arn:aws:iam::xxxxxx:role/event-driven-pipeline-demo-emr-eks-data-team-a",
"ReleaseLabel": "emr-6.7.0-latest",
"JobDriver": {
"SparkSubmitJobDriver": {
"EntryPoint": "s3://sparkjob-demo-bucket/scripts/pyspark-taxi-trip.py",
"EntryPointArguments": [
"s3://sparkjob-demo-bucket/input/",
"s3://sparkjob-demo-bucket/output/"
],
"SparkSubmitParameters": "--conf spark.executor.instances=10"
}
},
"ConfigurationOverrides": {
"ApplicationConfiguration": [
{
"Classification": "spark-defaults",
"Properties": {
"spark.driver.cores":"1",
"spark.executor.cores":"1",
"spark.driver.memory": "10g",
"spark.executor.memory": "10g",
"spark.kubernetes.driver.podTemplateFile":"s3://sparkjob-demo-bucket/scripts/driver-pod-template.yaml",
"spark.kubernetes.executor.podTemplateFile":"s3://sparkjob-demo-bucket/scripts/executor-pod-template.yaml",
"spark.local.dir" : "/data1,/data2"
}
}
]
}
},
"End": true
}
}
}