forked from hotosm/oam-server-tiler
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run-emr.sh
executable file
·30 lines (26 loc) · 1.44 KB
/
run-emr.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
MASTER_INSTANCE=m3.xlarge
MASTER_PRICE=0.15
WORKER_INSTANCE=m3.xlarge
WORKER_PRICE=0.15
WORKER_COUNT=10
DRIVER_MEMORY=2g
NUM_EXECUTORS=40
EXECUTOR_MEMORY=2304m
EXECUTOR_CORES=1
REQUEST_URI=s3://oam-server-tiler/testfiles/test-req-partial.json
WORKSPACE_URI=s3://oam-server-tiler/workspace/emr-test-job-partial
aws emr create-cluster \
--name "OAM Tiler" \
--log-uri s3://workspace-oam-hotosm-org/logs/ \
--release-label emr-4.0.0 \
--use-default-roles \
--ec2-attributes KeyName=oam-emanuele \
--applications Name=Spark \
--instance-groups \
Name=Master,InstanceCount=1,InstanceGroupType=MASTER,InstanceType=$MASTER_INSTANCE \
Name=Workers,InstanceCount=$WORKER_COUNT,BidPrice=$WORKER_PRICE,InstanceGroupType=CORE,InstanceType=$WORKER_INSTANCE \
--bootstrap-action Path=s3://oam-tiler-emr/bootstrap.sh \
--configurations file://./emr.json
--steps \
Name=CHUNK,ActionOnFailure=CONTINUE,Type=Spark,Args=[--deploy-mode,cluster,--driver-memory,$DRIVER_MEMORY,--num-executors,$NUM_EXECUTORS,--executor-memory,$EXECUTOR_MEMORY,--executor-cores,$EXECUTOR_CORES,s3://oam-tiler-emr/chunk.py,$REQUEST_URI] \
Name=MOSAIC,ActionOnFailure=CONTINUE,Type=Spark,Jar=s3://oam-tiler-emr/mosaic.jar,Args=[--deploy-mode,cluster,--driver-memory,$DRIVER_MEMORY,--num-executors,$NUM_EXECUTORS,--executor-memory,$EXECUTOR_MEMORY,--executor-cores,$EXECUTOR_CORES,--class,org.hotosm.oam.Main,s3://oam-tiler-emr/mosaic.jar,$WORKSPACE_URI/step1_result.json]