Skip to content
Browse files

Merge pull request #12 from barkbarkuk/master

Updates pallet to 0.72 (jclouds 1.4.2)
  • Loading branch information...
2 parents 85a1851 + 4ec0b9c commit 8c6ad07c324afe15bd8b7e8aace1e46148470cb7 @KasperMadsen KasperMadsen committed
View
24 README.markdown
@@ -2,6 +2,30 @@
This project makes it dead-simple to deploy Storm clusters on AWS. See [the wiki](https://github.com/nathanmarz/storm-deploy/wiki) for instructions on using this deploy.
+##
+AWS regional deploy.
+
+Make sure your ~/.pallet/config.clj contains (example for eu-west-1):
+
+ :jclouds.regions "eu-west-1"
+ :jclouds.ec2.cc-regions "eu-west-1"
+
+Make sure you find appropriate images available in your region and place them in conf/clusters.yaml (example for eu-west-1)
+
+nimbus.image: "eu-west-1/ami-1a0f3d6e" #64-bit ubuntu
+nimbus.hardware: "m1.large"
+
+supervisor.count: 1
+supervisor.image: "eu-west-1/ami-1a0f3d6e" #64-bit ubuntu on eu-west-1
+supervisor.hardware: "m1.large"
+#supervisor.spot.price: 1.60
+
+
+zookeeper.count: 1
+zookeeper.image: "eu-west-1/ami-1a0f3d6e" #64-bit ubuntu
+zookeeper.hardware: "m1.large"
+
+
## Acknowledgements
YourKit is kindly supporting open source projects with its full-featured Java Profiler. YourKit, LLC is the creator of innovative and intelligent tools for profiling Java and .NET applications. Take a look at YourKit's leading software products: [YourKit Java Profiler](http://www.yourkit.com/java/profiler/index.jsp) and [YourKit .NET Profiler](http://www.yourkit.com/.net/profiler/index.jsp).
View
10 conf/clusters.yaml
@@ -2,17 +2,17 @@
# CLUSTERS CONFIG FILE
################################################################################
-nimbus.image: "us-east-1/ami-08f40561" #64-bit ubuntu
+nimbus.image: "eu-west-1/ami-1a0f3d6e" #64-bit ubuntu
nimbus.hardware: "m1.large"
-supervisor.count: 2
-supervisor.image: "us-east-1/ami-08f40561" #64-bit ubuntu on us-east
-supervisor.hardware: "c1.xlarge"
+supervisor.count: 1
+supervisor.image: "eu-west-1/ami-1a0f3d6e" #64-bit ubuntu on eu-west-1
+supervisor.hardware: "m1.large"
#supervisor.spot.price: 1.60
zookeeper.count: 1
-zookeeper.image: "us-east-1/ami-08f40561" #64-bit ubuntu
+zookeeper.image: "eu-west-1/ami-1a0f3d6e" #64-bit ubuntu
zookeeper.hardware: "m1.large"
View
11 project.clj
@@ -12,7 +12,8 @@
:dependencies [
[storm "0.5.4"]
[commons-codec "1.4"]
- [org.cloudhoist/pallet "0.6.1"]
+ [org.cloudhoist/pallet "0.7.2"]
+ [org.cloudhoist/pallet-jclouds "1.4.2"]
[org.cloudhoist/java "0.5.0"]
[org.cloudhoist/git "0.5.0"]
[org.cloudhoist/ssh-key "0.5.0"]
@@ -23,14 +24,14 @@
[org.cloudhoist/nagios-config "0.5.0"]
[org.cloudhoist/crontab "0.5.0"]
- [org.jclouds.provider/aws-ec2 "1.0.0"]
- [org.jclouds.provider/aws-s3 "1.0.0"]
- [com.jcraft/jsch "0.1.44-1"] ; is this necessary?
+ [org.jclouds.driver/jclouds-sshj "1.4.2"]
+ [org.jclouds.provider/aws-ec2 "1.4.2"]
+ [org.jclouds.provider/aws-s3 "1.4.2"]
[log4j/log4j "1.2.14"]
[jvyaml "1.0.0"]]
:dev-dependencies [[swank-clojure "1.2.1"]
- [org.cloudhoist/pallet-lein "0.2.0"]])
+ [org.cloudhoist/pallet-lein "0.5.2"]])
View
29 resources/log4j.xml
@@ -20,6 +20,16 @@
</layout>
</appender>
+ <appender name="LOGOFEVERYTHING" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="logs/everything.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
<appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="logs/jclouds-wire.log" />
<param name="Append" value="true" />
@@ -86,13 +96,20 @@
<appender-ref ref="ASYNCCOMPUTE" />
</category>
-<!--
- <root>
- <priority value ="INFO" />
- <appender-ref ref="console" />
- </root>
- -->
+ <category name="backtype.storm">
+ <priority value="DEBUG" />
+ <appender-ref ref="console" />
+ </category>
+ <category name="java.logging">
+ <priority value="INFO" />
+ <appender-ref ref="console" />
+ </category>
+ <root>
+ <priority value ="DEBUG" />
+ <!-- <appender-ref ref="console" /> -->
+ <appender-ref ref="LOGOFEVERYTHING" />
+ </root>
</log4j:configuration>
View
2 src/clj/backtype/storm/crate/ganglia.clj
@@ -17,7 +17,7 @@
[session]
(-> session
(package/packages
- :aptitude ["rrdtool" "librrds-perl" "librrd2-dev" "php5-gd"
+ :aptitude ["rrdtool" "librrds-perl" "librrd-dev" "php5-gd"
"ganglia-monitor" "ganglia-webfrontend" "gmetad"])
(file/symbolic-link
"/usr/share/ganglia-webfrontend" "/var/www/ganglia")))
View
2 src/clj/backtype/storm/crate/leiningen.clj
@@ -1,7 +1,7 @@
(ns backtype.storm.crate.leiningen
(:require
[pallet.resource.remote-file :as remote-file]
- [pallet.resource.exec-script :as exec-script]))
+ [pallet.action.exec-script :as exec-script]))
;; this is 1.5.2. freezing version to ensure deploy is stable
(def download-url "https://raw.github.com/technomancy/leiningen/a1fa43400295d57a9acfed10735c1235904a9407/bin/lein")
View
14 src/clj/backtype/storm/crate/storm.clj
@@ -1,7 +1,9 @@
(ns backtype.storm.crate.storm
(:use [clojure.contrib.def :only [defnk]]
[pallet.compute :only [running? primary-ip private-ip]]
- [org.jclouds.compute :only [nodes-with-tag]]
+ [pallet.compute.jclouds]
+ [org.jclouds.compute2 :only [nodes-in-group]]
+
[pallet.configure :only [compute-service-properties pallet-config]])
(:require
[backtype.storm.crate.zeromq :as zeromq]
@@ -12,31 +14,31 @@
[pallet.resource.package :as package]
[pallet.resource.directory :as directory]
[pallet.resource.remote-file :as remote-file]
- [pallet.resource.exec-script :as exec-script]))
+ [pallet.action.exec-script :as exec-script]))
(defn storm-config
([] (storm-config "default"))
([conf-name] (compute-service-properties (pallet-config) [conf-name])))
(defn nimbus-ip [compute name]
- (let [running-nodes (filter running? (nodes-with-tag (str "nimbus-" name) compute))]
+ (let [running-nodes (filter running? (map (partial jclouds-node->node compute) (nodes-in-group compute (str "nimbus-" name))))]
(assert (= (count running-nodes) 1))
(primary-ip (first running-nodes))))
(defn nimbus-private-ip [compute name]
- (let [running-nodes (filter running? (nodes-with-tag (str "nimbus-" name) compute))]
+ (let [running-nodes (filter running? (map (partial jclouds-node->node compute) (nodes-in-group compute (str "nimbus-" name))))]
(assert (= (count running-nodes) 1))
(private-ip (first running-nodes))))
(defn zookeeper-ips [compute name]
(let [running-nodes (filter running?
- (nodes-with-tag (str "zookeeper-" name) compute))]
+ (map (partial jclouds-node->node compute) (nodes-in-group compute (str "zookeeper-" name))))]
(map primary-ip running-nodes)))
(defn supervisor-ips [compute name]
(let [running-nodes (filter running?
- (nodes-with-tag (str "supervisor-" name) compute))]
+ (map (partial jclouds-node->node compute) (nodes-in-group compute (str "supervisor-" name))))]
(map primary-ip running-nodes)))
(defn- install-dependencies [request]
View
2 src/clj/backtype/storm/crate/zeromq.clj
@@ -1,6 +1,6 @@
(ns backtype.storm.crate.zeromq
(:require
- [pallet.resource.exec-script :as exec-script]
+ [pallet.action.exec-script :as exec-script]
[pallet.resource.package :as package]
[pallet.resource.remote-directory :as remote-directory]
[pallet.resource.directory :as directory]
View
2 src/clj/backtype/storm/crate/zookeeper.clj
@@ -13,7 +13,7 @@
[pallet.stevedore :as stevedore]
[clojure.string :as string]
[pallet.resource.package :as package]
- [pallet.resource.exec-script :as exec-script]
+ [pallet.action.exec-script :as exec-script]
[pallet.crate.crontab :as crontab]
)
(:use
View
24 src/clj/backtype/storm/node.clj
@@ -23,9 +23,9 @@
[pallet.resource.exec-script :as exec-script])
(:use
[backtype.storm config]
- [pallet compute core resource]
+ [pallet compute core resource phase]
[pallet [utils :only [make-user]]]
- [org.jclouds.compute :only [nodes-with-tag]]
+ [org.jclouds.compute2 :only [nodes-in-group]]
[clojure.walk]))
(defn parse-release [release]
@@ -66,13 +66,13 @@
req
(:username *USER*)
(:public-key-path *USER*)))
- :configure (phase
+ :configure (phase-fn
(java/java :openjdk))}))
(defn zookeeper-server-spec []
(server-spec
:extends (base-server-spec)
- :phases {:configure (phase
+ :phases {:configure (phase-fn
(zookeeper/install :version "3.3.5")
(zookeeper/configure
:clientPort (storm-conf "storm.zookeeper.port")
@@ -83,25 +83,25 @@
(defn storm-base-server-spec [name]
(server-spec
:extends (base-server-spec)
- :phases {:post-configure (phase
+ :phases {:post-configure (phase-fn
(storm/write-storm-yaml
name
storm-yaml-path))
- :configure (phase
+ :configure (phase-fn
(configure-ssh-client :host-key-checking false))
- :exec (phase
+ :exec (phase-fn
(storm/exec-daemon)
(ganglia/ganglia-finish))}))
(defn supervisor-server-spec [name release]
(server-spec
:extends (storm-base-server-spec name)
- :phases {:configure (phase
+ :phases {:configure (phase-fn
(ganglia/ganglia-node (nimbus-name name))
(storm/install-supervisor
release
"/mnt/storm"))
- :post-configure (phase
+ :post-configure (phase-fn
(ganglia/ganglia-finish)
(storm/write-storm-exec
"supervisor"))}))
@@ -121,19 +121,19 @@
(defn nimbus-server-spec [name release]
(server-spec
:extends (storm-base-server-spec name)
- :phases {:configure (phase
+ :phases {:configure (phase-fn
(ganglia/ganglia-master (nimbus-name name))
(storm/install-nimbus
release
"/mnt/storm")
(storm/install-ui)
(maybe-install-drpc release))
- :post-configure (phase
+ :post-configure (phase-fn
(ganglia/ganglia-finish)
(storm/write-storm-exec
"nimbus")
)
- :exec (phase
+ :exec (phase-fn
(storm/exec-ui)
(maybe-exec-drpc release))}))
View
44 src/clj/backtype/storm/provision.clj
@@ -1,19 +1,24 @@
(ns backtype.storm.provision
(:import [java.io File])
(:use [clojure.contrib.command-line]
- [pallet.compute]
- [pallet.configure]
+ [pallet.compute :exclude [admin-user]]
[backtype.storm security]
[pallet.core]
- [org.jclouds.compute :only [nodes-with-tag]]
- [backtype.storm.util :only [with-var-roots]])
+ [org.jclouds.compute2 :only [nodes-in-group]]
+ [backtype.storm.util :only [with-var-roots]]
+ [clojure.tools.logging]
+ )
+ (:require [pallet.configure])
+ (:require [pallet.compute.jclouds])
(:require [backtype.storm.node :as node])
(:require [backtype.storm.crate.storm :as storm])
(:require [backtype.storm.deploy-util :as util]))
+(log-capture! "java.logging")
+
;; memoize this
(defn my-region []
- (-> (pallet-config) :services :default :jclouds.regions)
+ (-> (pallet.configure/pallet-config) :services :default :jclouds.regions)
)
(defn jclouds-group [& group-pieces]
@@ -24,10 +29,10 @@
))
(defn- print-ips-for-tag! [aws tag-str]
- (let [running-node (filter running? (nodes-with-tag tag-str aws))]
- (println "TAG: " tag-str)
- (println "PUBLIC: " (map primary-ip running-node))
- (println "PRIVATE: " (map private-ip running-node))))
+ (let [running-node (filter running? (map (partial pallet.compute.jclouds/jclouds-node->node aws) (nodes-in-group aws tag-str)))]
+ (info (str "TAG: " tag-str))
+ (info (str "PUBLIC: " (map primary-ip running-node)))
+ (info (str "PRIVATE: " (map private-ip running-node)))))
(defn print-all-ips! [aws name]
(let [all-tags [(str "zookeeper-" name) (str "nimbus-" name) (str "supervisor-" name)]]
@@ -50,13 +55,13 @@
(spit (str conf-dir "/supervisor.yaml") supervisor-yaml)))
(defn attach! [aws name]
- (println "Attaching to Available Cluster...")
+ (info "Attaching to Available Cluster...")
(sync-storm-conf-dir aws name)
- (authorizeme aws (jclouds-group "nimbus-" name) 80)
- (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "nimbus.thrift.port"))
- (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "ui.port"))
- (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "drpc.port"))
- (println "Attaching Complete."))
+ (authorizeme aws (jclouds-group "nimbus-" name) 80 (my-region))
+ (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "nimbus.thrift.port") (my-region))
+ (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "ui.port") (my-region))
+ (authorizeme aws (jclouds-group "nimbus-" name) (node/storm-conf "drpc.port") (my-region))
+ (info "Attaching Complete."))
(defn start-with-nodes! [aws name nimbus supervisor zookeeper]
(let [nimbus (node/nimbus* name nimbus)
@@ -64,20 +69,25 @@
zookeeper (node/zookeeper name zookeeper)
sn (int (node/clusters-conf "supervisor.count" 1))
zn (int (node/clusters-conf "zookeeper.count" 1))]
- (println (format "Provisioning nodes [nn=1, sn=%d, zn=%d]" sn zn))
+ (info (format "Provisioning nodes [nn=1, sn=%d, zn=%d]" sn zn))
(converge {nimbus 1
supervisor sn
zookeeper zn
}
:compute aws
)
+ (debug "Finished converge")
+
(authorize-group aws (my-region) (jclouds-group "nimbus-" name) (jclouds-group "supervisor-" name))
(authorize-group aws (my-region) (jclouds-group "supervisor-" name) (jclouds-group "nimbus-" name))
+ (debug "Finished authorizing groups")
(lift nimbus :compute aws :phase [:post-configure :exec])
(lift supervisor :compute aws :phase [:post-configure :exec])
+ (debug "Finished post-configure and exec phases")
+
(attach! aws name)
- (println "Provisioning Complete.")
+ (info "Provisioning Complete.")
(print-all-ips! aws name)))
(defn start! [aws name release]
View
8 src/clj/backtype/storm/security.clj
@@ -22,7 +22,7 @@
:doc "A clojure binding for the jclouds AWS security group interface."}
backtype.storm.security
(:require (org.jclouds [compute2 :as compute])
- [org.jclouds.ec2.ebs :as ebs])
+ [org.jclouds.ec2.ebs2 :as ebs])
(:import org.jclouds.ec2.domain.IpProtocol
org.jclouds.ec2.domain.SecurityGroup
org.jclouds.ec2.services.SecurityGroupClient
@@ -88,7 +88,7 @@
(.authorizeSecurityGroupIngressInRegion
(sg-service compute) (ebs/get-region region) (.getName group) (get-protocol protocol) from-port to-port (or ip-range "0.0.0.0/0"))
(throw (IllegalArgumentException.
- (str "Can't find security group for name " group-name))))))
+ (str "Can't find security group for name " group-name region ip-range from-port to-port))))))
(def my-ip
(memoize
@@ -99,9 +99,9 @@
ret
))))
-(defn authorizeme [compute group-name port]
+(defn authorizeme [compute group-name port region]
(try
- (authorize compute group-name port :ip-range (str (my-ip) "/32")
+ (authorize compute group-name port :ip-range (str (my-ip) "/32") :region region
)
(catch IllegalStateException _)
))

0 comments on commit 8c6ad07

Please sign in to comment.
Something went wrong with that request. Please try again.