Find file
Fetching contributors…
Cannot retrieve contributors at this time
58 lines (51 sloc) 2.23 KB
runners:
emr:
## you must edit these to point to your AWS EMR key
ec2_key_pair: kba-aws-us-east-1
ec2_key_pair_file: ~/.ssh/kba-aws-us-east-1.pem # ~/ and $ENV_VARS allowed here
## turn on logging and tracker
ssh_tunnel_to_job_tracker: true
s3_log_uri: s3://trec-kba-emr/example/logs
## do not bootstrap mrjob, because we install it using our private
## python27 install below.
bootstrap_mrjob: false
## must include this tarball created by running `make`
python_archives:
- kba_corpus.tar.gz
## these bootstrap actions happen during cluster setup
bootstrap_actions:
- s3://elasticmapreduce/bootstrap-actions/configure-hadoop -m mapred.tasktracker.map.tasks.maximum=5 -m mapred.tasktracker.reduce.tasks.maximum=5
## install our own python27 built on the m1.large instance class;
## this script also installs xz-utils and bootstraps mrjob
- s3://trec-kba-emr/emr-setup/install_python27.m1.large.sh
## could enable skipping -- see also jobconf for more skipping related params
#- s3://elasticmapreduce/bootstrap-actions/configure-hadoop -m mapred.skip.map.auto.incr.proc.count=false -m mapred.skip.mode.enabled=true
#- s3://elasticmapreduce/bootstrap-actions/configure-hadoop -m io.map.index.skip=1
## could use spot instances by uncommenting the bid prices
#ec2_instance_type: m1.large
#num_ec2_instances: 1
ec2_master_instance_type: m1.large
#ec2_master_instance_bid_price: '0.154'
#ec2_core_instance_type: m2.xlarge
#ec2_core_instance_bid_price: '0.154'
#num_ec2_core_instances: 1
#ec2_task_instance_type: m2.xlarge
#ec2_task_instance_bid_price: '0.154'
#num_ec2_task_instances: 1
enable_emr_debugging: true
jobconf:
# enable skip mode
#mapred.skip.map.auto.incr.proc.count: false
#mapred.skip.mode.enabled: true
#mapred.skip.map.max.skip.records: 1
#mapred.skip.attempts.to.start.skipping: 0
#mapred.map.max.attempts: 10
# limit total tasks
mapred.map.tasks: 40
mapred.reduce.tasks: 3
mapred.task.timeout: 300000 ## five minutes
local:
## to run locally, all we need is the tarball, which includes the
## decryption key
python_archives:
- kba_corpus.tar.gz