Permalink
Browse files

Test for add-action step

  • Loading branch information...
1 parent 3a9f4fa commit b8246c8f9856c19ba99f87fce46b7f4fcd551867 Brubacher, Federico committed Jan 7, 2013
Showing with 49 additions and 19 deletions.
  1. +10 −2 src/main/clj/com/climate/services/aws/emr.clj
  2. +39 −17 src/test/clj/com/climate/services/aws/emr_test.clj
@@ -9,6 +9,7 @@
(:import
java.io.File
com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClient
+ com.amazonaws.services.elasticmapreduce.model.TerminateJobFlowsRequest
com.amazonaws.services.elasticmapreduce.util.StepFactory
com.amazonaws.auth.BasicAWSCredentials
[com.amazonaws.services.elasticmapreduce.model
@@ -224,6 +225,13 @@
(.setActionOnFailure (str ActionOnFailure/TERMINATE_JOB_FLOW))
(.setHadoopJarStep (.newEnableDebuggingStep (StepFactory.)))))
+(defn terminate-flow-id
+ ([jobflow-id]
+ (terminate-flow-id jobflow-id *emr*))
+ ([jobflow-id emr]
+ (.terminateJobFlows emr
+ (TerminateJobFlowsRequest. (java.util.ArrayList. [jobflow-id])))))
+
(defn step-config [name alive? jar-path main-class cli-args & {:keys [action-on-failure properties]}]
"Create a step to be submitted to EMR.
jar-path is the hadoop job jar, usually an s3:// path.
@@ -238,8 +246,8 @@
(.setArgs (vec cli-args)) ;collection of strings
(.setProperties (kv-props properties))))]
(.setActionOnFailure sc (str (or action-on-failure
- (and alive? ActionOnFailure/CANCEL_AND_WAIT)
- ActionOnFailure/TERMINATE_JOB_FLOW)))))
+ (and alive? ActionOnFailure/CANCEL_AND_WAIT)
+ ActionOnFailure/TERMINATE_JOB_FLOW)))))
(defn add-steps
"Add a step to a running jobflow. Steps is a seq of StepConfig objects.
@@ -24,9 +24,13 @@
[ec2 :as ec2]
[common :as awscommon]])
(:import
+ com.amazonaws.services.elasticmapreduce.util.StepFactory
[com.amazonaws.services.elasticmapreduce.model
JobFlowDetail
- JobFlowExecutionStatusDetail]
+ JobFlowExecutionStatusDetail
+ HadoopJarStepConfig
+ ActionOnFailure
+ StepConfig]
java.util.Date))
;; Some tests are labelled as :manual, rather than :integration, because
@@ -37,6 +41,23 @@
(def aws-creds (awscommon/aws-credential-discovery))
+(def ^:dynamic *flow-args*
+ {:bootstrap-actions
+ ; Only publicly available script, so we don't have to upload the others.
+ [(bootstrap "Hadoop Config"
+ "s3://elasticmapreduce/bootstrap-actions/configure-hadoop"
+ ["-m" "mapred.map.tasks.speculative.execution=false"])]
+ :log-uri (str "s3://" bucket)
+ :keypair (:keypair aws-creds) ; the elastic-mapreduce credentials.json file as a keypair entry
+ :ami-version "latest"
+ :num-instances 2
+ :master-type "m1.xlarge"
+ :slave-type "m1.xlarge"
+ :spot-task-type "m1.xlarge"
+ :spot-task-bid "1.00"
+ :spot-task-num 1
+ :keep-alive false})
+
(use-fixtures :once
(fn [f]
(binding [s3/*s3* (s3/s3 aws-creds)
@@ -70,22 +91,7 @@
["-input" (format "s3://%s/data/simple.txt" bucket)
"-output" "/out"
"-mapper" (format "s3://%s/scripts/wc.sh" bucket)])]
- {:bootstrap-actions
- ; Only publicly available script, so we don't have to upload the others.
- [(bootstrap "Hadoop Config"
- "s3://elasticmapreduce/bootstrap-actions/configure-hadoop"
- ["-m" "mapred.map.tasks.speculative.execution=false"])]
- :log-uri (str "s3://" bucket)
- :keypair (:keypair aws-creds) ; the elastic-mapreduce credentials.json file as a keypair entry
- :ami-version "latest"
- :num-instances 2
- :master-type "m1.xlarge"
- :slave-type "m1.xlarge"
- :spot-task-type "m1.xlarge"
- :spot-task-bid "1.00"
- :spot-task-num 1
- ;too dangerous to use keep-alive in tests, so tests are limited-- no (emr/add-step) for example
- :keep-alive false}))))
+ *flow-args*))))
; Specified as a fn rather than a test. This is a hack to force it to run before
; test-wait-on-step. It will fail if the cluster has already COMPLETED.
@@ -159,3 +165,19 @@
(is= "m1.xlarge" spot-task-type)
(is= 20 spot-task-num)
(is= (format "%.3f" expected-bid) spot-task-bid)))
+
+(defn make-dummy-step []
+ (doto (StepConfig.)
+ (.setName "Dummy")
+ (.setActionOnFailure (str ActionOnFailure/TERMINATE_JOB_FLOW))
+ (.setHadoopJarStep (.newEnableDebuggingStep (StepFactory.)))))
+
+(deftest ^{:manual true} test-add-steps-to-existing-flow
+ (testing "emr/add-step"
+ (binding [*flow-args* (conj *flow-args* {:keep-alive true})]
+ (let [jf-id (setup)
+ dummy-steps (make-dummy-step)]
+ (is= 0 (.size (steps-for-jobflow jf-id)))
+ (add-steps jf-id (java.util.ArrayList. [dummy-steps]))
+ (is= 1 (.size (steps-for-jobflow jf-id)))
+ (terminate-flow-id jf-id)))))

0 comments on commit b8246c8

Please sign in to comment.