Merge pull request #12 from barkbarkuk/master

Updates pallet to 0.72 (jclouds 1.4.2)

README.markdown

@@ -2,6 +2,30 @@
 
 This project makes it dead-simple to deploy Storm clusters on AWS. See [the wiki](https://github.com/nathanmarz/storm-deploy/wiki) for instructions on using this deploy.
 
+##
+AWS regional deploy.
+
+Make sure your ~/.pallet/config.clj contains (example for eu-west-1):
+
+             :jclouds.regions "eu-west-1"
+	     :jclouds.ec2.cc-regions "eu-west-1"
+
+Make sure you find appropriate images available in your region and place them in conf/clusters.yaml (example for eu-west-1)
+
+nimbus.image: "eu-west-1/ami-1a0f3d6e"         #64-bit ubuntu
+nimbus.hardware: "m1.large"
+
+supervisor.count: 1
+supervisor.image: "eu-west-1/ami-1a0f3d6e"         #64-bit ubuntu on eu-west-1
+supervisor.hardware: "m1.large"
+#supervisor.spot.price: 1.60
+
+
+zookeeper.count: 1
+zookeeper.image: "eu-west-1/ami-1a0f3d6e"         #64-bit ubuntu
+zookeeper.hardware: "m1.large"
+
+
 ## Acknowledgements
 
 YourKit is kindly supporting open source projects with its full-featured Java Profiler. YourKit, LLC is the creator of innovative and intelligent tools for profiling Java and .NET applications. Take a look at YourKit's leading software products: [YourKit Java Profiler](http://www.yourkit.com/java/profiler/index.jsp) and [YourKit .NET Profiler](http://www.yourkit.com/.net/profiler/index.jsp).

conf/clusters.yaml

@@ -2,17 +2,17 @@
 # CLUSTERS CONFIG FILE
 ################################################################################
 
-nimbus.image: "us-east-1/ami-08f40561"         #64-bit ubuntu
+nimbus.image: "eu-west-1/ami-1a0f3d6e"         #64-bit ubuntu
 nimbus.hardware: "m1.large"
 
-supervisor.count: 2
-supervisor.image: "us-east-1/ami-08f40561"         #64-bit ubuntu on us-east
-supervisor.hardware: "c1.xlarge"
+supervisor.count: 1
+supervisor.image: "eu-west-1/ami-1a0f3d6e"         #64-bit ubuntu on eu-west-1
+supervisor.hardware: "m1.large"
 #supervisor.spot.price: 1.60
 
 
 zookeeper.count: 1
-zookeeper.image: "us-east-1/ami-08f40561"         #64-bit ubuntu
+zookeeper.image: "eu-west-1/ami-1a0f3d6e"         #64-bit ubuntu
 zookeeper.hardware: "m1.large"
 
 

project.clj

@@ -12,7 +12,8 @@
   :dependencies [
                  [storm "0.5.4"]
                  [commons-codec "1.4"]
-                 [org.cloudhoist/pallet "0.6.1"]
+                 [org.cloudhoist/pallet "0.7.2"]
+                 [org.cloudhoist/pallet-jclouds "1.4.2"]
                  [org.cloudhoist/java "0.5.0"]
                  [org.cloudhoist/git "0.5.0"]
                  [org.cloudhoist/ssh-key "0.5.0"]
@@ -23,14 +24,14 @@
                  [org.cloudhoist/nagios-config "0.5.0"]
                  [org.cloudhoist/crontab "0.5.0"]
 
-                 [org.jclouds.provider/aws-ec2 "1.0.0"]
-                 [org.jclouds.provider/aws-s3 "1.0.0"]
-                 [com.jcraft/jsch "0.1.44-1"]  ; is this necessary?
+                 [org.jclouds.driver/jclouds-sshj "1.4.2"]
+                 [org.jclouds.provider/aws-ec2 "1.4.2"]
+                 [org.jclouds.provider/aws-s3 "1.4.2"]
 
                  [log4j/log4j "1.2.14"]
                  [jvyaml "1.0.0"]]
 
   :dev-dependencies [[swank-clojure "1.2.1"]
-                     [org.cloudhoist/pallet-lein "0.2.0"]])
+                     [org.cloudhoist/pallet-lein "0.5.2"]])
 
 

resources/log4j.xml

@@ -20,6 +20,16 @@
     </layout>
   </appender>
 
+  <appender name="LOGOFEVERYTHING" class="org.apache.log4j.DailyRollingFileAppender">
+      <param name="File" value="logs/everything.log" />
+      <param name="Append" value="true" />
+      <param name="DatePattern" value="'.'yyyy-MM-dd" />
+      <param name="Threshold" value="TRACE" />
+      <layout class="org.apache.log4j.PatternLayout">
+          <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+      </layout>
+  </appender>
+
   <appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
     <param name="File" value="logs/jclouds-wire.log" />
     <param name="Append" value="true" />
@@ -86,13 +96,20 @@
     <appender-ref ref="ASYNCCOMPUTE" />
   </category>
 
-<!--
-  <root>
-    <priority value ="INFO" />
-    <appender-ref ref="console" />
-  </root>
-  -->
+  <category name="backtype.storm">
+      <priority value="DEBUG" />
+      <appender-ref ref="console" />
+  </category>
 
+  <category name="java.logging">
+      <priority value="INFO" />
+      <appender-ref ref="console" />
+  </category>
 
+  <root>
+    <priority value ="DEBUG" />
+    <!-- <appender-ref ref="console" /> -->
+      <appender-ref ref="LOGOFEVERYTHING" />
+  </root>
 
 </log4j:configuration>

src/clj/backtype/storm/crate/ganglia.clj

@@ -17,7 +17,7 @@
   [session]
   (-> session
       (package/packages
-       :aptitude ["rrdtool" "librrds-perl" "librrd2-dev" "php5-gd"
+       :aptitude ["rrdtool" "librrds-perl" "librrd-dev" "php5-gd"
                   "ganglia-monitor" "ganglia-webfrontend" "gmetad"])
       (file/symbolic-link
        "/usr/share/ganglia-webfrontend" "/var/www/ganglia")))

src/clj/backtype/storm/crate/leiningen.clj

@@ -1,7 +1,7 @@
 (ns backtype.storm.crate.leiningen
   (:require
    [pallet.resource.remote-file :as remote-file]
-   [pallet.resource.exec-script :as exec-script]))
+   [pallet.action.exec-script :as exec-script]))
 
 ;; this is 1.5.2. freezing version to ensure deploy is stable
 (def download-url "https://raw.github.com/technomancy/leiningen/a1fa43400295d57a9acfed10735c1235904a9407/bin/lein")

src/clj/backtype/storm/crate/storm.clj

@@ -1,7 +1,9 @@
 (ns backtype.storm.crate.storm
   (:use [clojure.contrib.def :only [defnk]]
         [pallet.compute :only [running? primary-ip private-ip]]
-        [org.jclouds.compute :only [nodes-with-tag]]
+        [pallet.compute.jclouds]
+        [org.jclouds.compute2 :only [nodes-in-group]]
+
         [pallet.configure :only [compute-service-properties pallet-config]])
   (:require
    [backtype.storm.crate.zeromq :as zeromq]
@@ -12,31 +14,31 @@
    [pallet.resource.package :as package]
    [pallet.resource.directory :as directory]
    [pallet.resource.remote-file :as remote-file]
-   [pallet.resource.exec-script :as exec-script]))
+   [pallet.action.exec-script :as exec-script]))
 
 (defn storm-config
   ([] (storm-config "default"))
   ([conf-name] (compute-service-properties (pallet-config) [conf-name])))
 
 (defn nimbus-ip [compute name]
-  (let [running-nodes (filter running? (nodes-with-tag (str "nimbus-" name) compute))]
+  (let [running-nodes (filter running? (map (partial jclouds-node->node compute) (nodes-in-group compute (str "nimbus-" name))))]
     (assert (= (count running-nodes) 1))
     (primary-ip (first running-nodes))))
 
 (defn nimbus-private-ip [compute name]
-  (let [running-nodes (filter running? (nodes-with-tag (str "nimbus-" name) compute))]
+  (let [running-nodes (filter running? (map (partial jclouds-node->node compute) (nodes-in-group compute (str "nimbus-" name))))]
     (assert (= (count running-nodes) 1))
     (private-ip (first running-nodes))))
 
 
 (defn zookeeper-ips [compute name]
   (let [running-nodes (filter running?
-                              (nodes-with-tag (str "zookeeper-" name) compute))]
+    (map (partial jclouds-node->node compute) (nodes-in-group compute (str "zookeeper-" name))))]
     (map primary-ip running-nodes)))
 
 (defn supervisor-ips [compute name]
   (let [running-nodes (filter running?
-                              (nodes-with-tag (str "supervisor-" name) compute))]
+    (map (partial jclouds-node->node compute) (nodes-in-group compute (str "supervisor-" name))))]
     (map primary-ip running-nodes)))
 
 (defn- install-dependencies [request]

src/clj/backtype/storm/crate/zeromq.clj

@@ -1,6 +1,6 @@
 (ns backtype.storm.crate.zeromq
   (:require
-   [pallet.resource.exec-script :as exec-script]
+   [pallet.action.exec-script :as exec-script]
    [pallet.resource.package :as package]
    [pallet.resource.remote-directory :as remote-directory]
    [pallet.resource.directory :as directory]

src/clj/backtype/storm/crate/zookeeper.clj

@@ -13,7 +13,7 @@
    [pallet.stevedore :as stevedore]
    [clojure.string :as string]
    [pallet.resource.package :as package]
-   [pallet.resource.exec-script :as exec-script]
+   [pallet.action.exec-script :as exec-script]
    [pallet.crate.crontab :as crontab]
   )
   (:use

src/clj/backtype/storm/node.clj

@@ -23,9 +23,9 @@
    [pallet.resource.exec-script :as exec-script])
   (:use
    [backtype.storm config]
-   [pallet compute core resource]
+   [pallet compute core resource phase]
    [pallet [utils :only [make-user]]]
-   [org.jclouds.compute :only [nodes-with-tag]]
+   [org.jclouds.compute2 :only [nodes-in-group]]
    [clojure.walk]))
 
 (defn parse-release [release]
@@ -66,13 +66,13 @@
                                   req
                                   (:username *USER*)
                                   (:public-key-path *USER*)))
-            :configure (phase
+            :configure (phase-fn
                         (java/java :openjdk))}))
 
 (defn zookeeper-server-spec []
      (server-spec
       :extends (base-server-spec)
-      :phases {:configure (phase
+      :phases {:configure (phase-fn
                            (zookeeper/install :version "3.3.5")
                            (zookeeper/configure
                             :clientPort (storm-conf "storm.zookeeper.port")
@@ -83,25 +83,25 @@
 (defn storm-base-server-spec [name]
      (server-spec
       :extends (base-server-spec)
-      :phases {:post-configure (phase
+      :phases {:post-configure (phase-fn
                                 (storm/write-storm-yaml
                                  name
                                  storm-yaml-path))
-               :configure (phase
+               :configure (phase-fn
                            (configure-ssh-client :host-key-checking false))
-               :exec (phase
+               :exec (phase-fn
                       (storm/exec-daemon)
                       (ganglia/ganglia-finish))}))
 
 (defn supervisor-server-spec [name release]
      (server-spec
       :extends (storm-base-server-spec name)
-      :phases {:configure (phase
+      :phases {:configure (phase-fn
                            (ganglia/ganglia-node (nimbus-name name))
                            (storm/install-supervisor
                             release
                             "/mnt/storm"))
-               :post-configure (phase
+               :post-configure (phase-fn
                                 (ganglia/ganglia-finish)
                                 (storm/write-storm-exec
                                  "supervisor"))}))
@@ -121,19 +121,19 @@
 (defn nimbus-server-spec [name release]
      (server-spec
       :extends (storm-base-server-spec name)
-      :phases {:configure (phase
+      :phases {:configure (phase-fn
                            (ganglia/ganglia-master (nimbus-name name))
                            (storm/install-nimbus
                             release
                             "/mnt/storm")
                            (storm/install-ui)
                            (maybe-install-drpc release))
-               :post-configure (phase
+               :post-configure (phase-fn
                                 (ganglia/ganglia-finish)
                                 (storm/write-storm-exec
                                  "nimbus")
                                  )
-               :exec (phase
+               :exec (phase-fn
                         (storm/exec-ui)
                         (maybe-exec-drpc release))}))
 

src/clj/backtype/storm/provision.clj

@@ -1,19 +1,24 @@
 (ns backtype.storm.provision
   (:import [java.io File])
   (:use [clojure.contrib.command-line]
-        [pallet.compute]
-        [pallet.configure]
+        [pallet.compute :exclude [admin-user]]
         [backtype.storm security]
         [pallet.core]
-        [org.jclouds.compute :only [nodes-with-tag]]
-        [backtype.storm.util :only [with-var-roots]])
+        [org.jclouds.compute2 :only [nodes-in-group]]
+        [backtype.storm.util :only [with-var-roots]]
+        [clojure.tools.logging]
+        )
+  (:require [pallet.configure])
+  (:require [pallet.compute.jclouds])
   (:require [backtype.storm.node :as node])
   (:require [backtype.storm.crate.storm :as storm])
   (:require [backtype.storm.deploy-util :as util]))
 
+(log-capture! "java.logging")
+
 ;; memoize this
 (defn my-region []
-  (-> (pallet-config) :services :default :jclouds.regions)
+  (-> (pallet.configure/pallet-config) :services :default :jclouds.regions)
   )
 
 (defn jclouds-group [& group-pieces]
@@ -24,10 +29,10 @@
        ))
 
 (defn- print-ips-for-tag! [aws tag-str]
-  (let [running-node (filter running? (nodes-with-tag tag-str aws))]
-    (println "TAG:     " tag-str)
-    (println "PUBLIC:  " (map primary-ip running-node))
-    (println "PRIVATE: " (map private-ip running-node))))
+  (let [running-node (filter running? (map (partial pallet.compute.jclouds/jclouds-node->node aws) (nodes-in-group aws tag-str)))]
+    (info (str "TAG:     " tag-str))
+    (info (str "PUBLIC:  " (map primary-ip running-node)))
+    (info (str "PRIVATE: " (map private-ip running-node)))))
 
 (defn print-all-ips! [aws name]
   (let [all-tags [(str "zookeeper-" name) (str "nimbus-" name) (str "supervisor-" name)]]
@@ -50,34 +55,39 @@
     (spit (str conf-dir "/supervisor.yaml") supervisor-yaml)))
 
 (defn attach! [aws name]
-  (println "Attaching to Available Cluster...")
+  (info "Attaching to Available Cluster...")
   (sync-storm-conf-dir aws name)
-  (authorizeme aws (jclouds-group "nimbus-" name) 80)
-  (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "nimbus.thrift.port"))
-  (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "ui.port"))
-  (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "drpc.port"))
-  (println "Attaching Complete."))
+  (authorizeme aws (jclouds-group "nimbus-" name) 80 (my-region))
+  (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "nimbus.thrift.port") (my-region))
+  (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "ui.port") (my-region))
+  (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "drpc.port") (my-region))
+  (info "Attaching Complete."))
 
 (defn start-with-nodes! [aws name nimbus supervisor zookeeper]
   (let [nimbus (node/nimbus* name nimbus)
         supervisor (node/supervisor* name supervisor)
         zookeeper (node/zookeeper name zookeeper)
         sn (int (node/clusters-conf "supervisor.count" 1))
         zn (int (node/clusters-conf "zookeeper.count" 1))]
-    (println (format "Provisioning nodes [nn=1, sn=%d, zn=%d]" sn zn))
+    (info (format "Provisioning nodes [nn=1, sn=%d, zn=%d]" sn zn))
     (converge {nimbus 1
               supervisor sn
               zookeeper zn
               }
             :compute aws
             )
+    (debug "Finished converge")
+
     (authorize-group aws (my-region) (jclouds-group "nimbus-" name) (jclouds-group "supervisor-" name))
     (authorize-group aws (my-region) (jclouds-group "supervisor-" name) (jclouds-group "nimbus-" name))
+    (debug "Finished authorizing groups")
 
     (lift nimbus :compute aws :phase [:post-configure :exec])
     (lift supervisor :compute aws :phase [:post-configure :exec])
+    (debug "Finished post-configure and exec phases")
+
     (attach! aws name)
-    (println "Provisioning Complete.")
+    (info "Provisioning Complete.")
     (print-all-ips! aws name)))
 
 (defn start! [aws name release]