From 6e9920ba886a13227b603abe61f5a8acf278ff07 Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Tue, 13 Nov 2012 00:00:04 +0000 Subject: [PATCH 1/2] make jobs visible to all IAM users - set VisibleToAllUsers to true on all runjobflowrequests - update to latest version of the amazon sdk for java --- CHANGES.txt | 5 +++++ project.clj | 2 +- src/main/clj/com/climate/services/aws/emr.clj | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index a3340f4..c5f260c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,10 @@ CHANGES +1.2.1 + +- set the VisibleToAllUsers flag on all jobflows +- update to version 1.3.24 of the Amazon SDK for Java + 1.2.0 - s3/cp now supports S3 directory to S3 directory copies diff --git a/project.clj b/project.clj index 815c22c..21a14ea 100644 --- a/project.clj +++ b/project.clj @@ -17,7 +17,7 @@ ; aws-java-sdk-1.3.3 does not specify the correct httpclient, so we do it explicitly [org.apache.httpcomponents/httpclient "4.1.1"] - [com.amazonaws/aws-java-sdk "1.3.3" + [com.amazonaws/aws-java-sdk "1.3.24" :exclusions [javax.mail/mail org.apache.httpcomponents/httpclient]] ; TODO these two are only to support hipchat-- isolate that functionality, so these libs can be optional diff --git a/src/main/clj/com/climate/services/aws/emr.clj b/src/main/clj/com/climate/services/aws/emr.clj index b57a95d..cd3df9b 100644 --- a/src/main/clj/com/climate/services/aws/emr.clj +++ b/src/main/clj/com/climate/services/aws/emr.clj @@ -259,6 +259,7 @@ (.setAmiVersion ami-version) (.setSupportedProducts supported-products) (.setBootstrapActions bootstrap-actions) + (.setVisibleToAllUsers true) (.setSteps steps))] (.getJobFlowId (.runJobFlow *emr* request)))) From c5253f8932f19448ccc2005e49da7d1e8eab005b Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Tue, 13 Nov 2012 18:58:09 +0000 Subject: [PATCH 2/2] make visibletoallusers optional ...and off by default --- src/main/clj/com/climate/services/aws/emr.clj | 6 +++--- src/main/clj/lemur/core.clj | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/clj/com/climate/services/aws/emr.clj b/src/main/clj/com/climate/services/aws/emr.clj index cd3df9b..1e8854c 100644 --- a/src/main/clj/com/climate/services/aws/emr.clj +++ b/src/main/clj/com/climate/services/aws/emr.clj @@ -247,8 +247,8 @@ [jobflow-id steps] (AddJobFlowStepsRequest. jobflow-id steps)) -(defn start-job-flow [name steps {:keys [log-uri bootstrap-actions ami-version supported-products] - :or {bootstrap-actions [] supported-products []} +(defn start-job-flow [name steps {:keys [log-uri bootstrap-actions ami-version supported-products visible-to-all-users] + :or {bootstrap-actions [] supported-products [] visible-to-all-users false} :as all}] (log/info (str "Starting JobFlow " all)) (let [instances (instances-config all) @@ -259,7 +259,7 @@ (.setAmiVersion ami-version) (.setSupportedProducts supported-products) (.setBootstrapActions bootstrap-actions) - (.setVisibleToAllUsers true) + (.setVisibleToAllUsers visible-to-all-users) (.setSteps steps))] (.getJobFlowId (.runJobFlow *emr* request)))) diff --git a/src/main/clj/lemur/core.clj b/src/main/clj/lemur/core.clj index 0d3f6b6..7182e65 100644 --- a/src/main/clj/lemur/core.clj +++ b/src/main/clj/lemur/core.clj @@ -561,7 +561,8 @@ calls launch - take action (upload files, start cluster, etc) :keypair (:keypair evaluating-opts) :ami-version (:ami-version evaluating-opts) :hadoop-version (:hadoop-version evaluating-opts) - :supported-products (:supported-products evaluating-opts)} + :supported-products (:supported-products evaluating-opts) + :visible-to-all-users (true? (:visible-to-all-users evaluating-opts))} steps (if (:enable-debugging? evaluating-opts) (cons (emr/debug-step-config) steps)