Permalink
Browse files

Merge branch 'master' of github.com:nathanmarz/storm-deploy

Conflicts:
	project.clj
  • Loading branch information...
2 parents 5724708 + 09a96e5 commit c6077188e11d2edc4cf4318f0060c6525ec791a0 @travis travis committed Mar 25, 2013
View
@@ -2,24 +2,15 @@
# CLUSTERS CONFIG FILE
################################################################################
-nimbus.image: "us-east-1/ami-08f40561" #64-bit ubuntu
+nimbus.image: "us-east-1/ami-d726abbe" #64-bit ubuntu
nimbus.hardware: "m1.large"
supervisor.count: 2
-supervisor.image: "us-east-1/ami-08f40561" #64-bit ubuntu on us-east
-supervisor.hardware: "c1.xlarge"
+supervisor.image: "us-east-1/ami-d726abbe" #64-bit ubuntu on eu-east-1
+supervisor.hardware: "m1.large"
#supervisor.spot.price: 1.60
zookeeper.count: 1
-zookeeper.image: "us-east-1/ami-08f40561" #64-bit ubuntu
+zookeeper.image: "us-east-1/ami-d726abbe" #64-bit ubuntu
zookeeper.hardware: "m1.large"
-
-
-
-################################################################################
-# Other Options:
-#
-# Need to use us-west-1?
-# Change all the image id's to "us-west-1/ami-a17e2ee4"
-# See alestic.com for other AMI options.
View
@@ -12,7 +12,8 @@
:dependencies [
[storm "0.5.4"]
[commons-codec "1.4"]
- [org.cloudhoist/pallet "0.6.1"]
+ [org.cloudhoist/pallet "0.7.2"]
+ [org.cloudhoist/pallet-jclouds "1.4.2"]
[org.cloudhoist/java "0.5.0"]
[org.cloudhoist/git "0.5.0"]
[org.cloudhoist/ssh-key "0.5.0"]
@@ -23,15 +24,20 @@
[org.cloudhoist/nagios-config "0.5.0"]
[org.cloudhoist/crontab "0.5.0"]
- [org.jclouds.provider/aws-ec2 "1.0.0"]
- [org.jclouds.provider/aws-s3 "1.0.0"]
- [com.jcraft/jsch "0.1.44-1"] ; is this necessary?
+ [com.jcraft/jsch.agentproxy.usocket-jna "0.0.5"]
+ [com.jcraft/jsch.agentproxy.usocket-nc "0.0.5"]
+ [com.jcraft/jsch.agentproxy.sshagent "0.0.5"]
+ [com.jcraft/jsch.agentproxy.pageant "0.0.5"]
+ [com.jcraft/jsch.agentproxy.core "0.0.5"]
+ [com.jcraft/jsch.agentproxy.jsch "0.0.5"]
+ [com.jcraft/jsch "0.1.49"]
+ [org.jclouds.driver/jclouds-sshj "1.4.2"]
+ [org.jclouds.provider/aws-ec2 "1.4.2"]
+ [org.jclouds.provider/aws-s3 "1.4.2"]
[log4j/log4j "1.2.14"]
[jvyaml "1.0.0"]]
:dev-dependencies [[swank-clojure "1.2.1"]
- [org.cloudhoist/pallet-lein "0.2.0"]]
+ [org.cloudhoist/pallet-lein "0.5.2"]]
:min-lein-version "2.0.0")
-
-
View
@@ -20,6 +20,16 @@
</layout>
</appender>
+ <appender name="LOGOFEVERYTHING" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="logs/everything.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
<appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="logs/jclouds-wire.log" />
<param name="Append" value="true" />
@@ -86,13 +96,20 @@
<appender-ref ref="ASYNCCOMPUTE" />
</category>
-<!--
- <root>
- <priority value ="INFO" />
- <appender-ref ref="console" />
- </root>
- -->
+ <category name="backtype.storm">
+ <priority value="DEBUG" />
+ <appender-ref ref="console" />
+ </category>
+ <category name="java.logging">
+ <priority value="INFO" />
+ <appender-ref ref="console" />
+ </category>
+ <root>
+ <priority value ="DEBUG" />
+ <!-- <appender-ref ref="console" /> -->
+ <appender-ref ref="LOGOFEVERYTHING" />
+ </root>
</log4j:configuration>
@@ -17,7 +17,7 @@
[session]
(-> session
(package/packages
- :aptitude ["rrdtool" "librrds-perl" "librrd2-dev" "php5-gd"
+ :aptitude ["rrdtool" "librrds-perl" "librrd-dev" "php5-gd" "libapache2-mod-php5"
"ganglia-monitor" "ganglia-webfrontend" "gmetad"])
(file/symbolic-link
"/usr/share/ganglia-webfrontend" "/var/www/ganglia")))
@@ -1,7 +1,7 @@
(ns backtype.storm.crate.leiningen
(:require
[pallet.resource.remote-file :as remote-file]
- [pallet.resource.exec-script :as exec-script]))
+ [pallet.action.exec-script :as exec-script]))
;; this is 1.5.2. freezing version to ensure deploy is stable
(def download-url "https://raw.github.com/technomancy/leiningen/a1fa43400295d57a9acfed10735c1235904a9407/bin/lein")
@@ -1,7 +1,9 @@
(ns backtype.storm.crate.storm
(:use [clojure.contrib.def :only [defnk]]
[pallet.compute :only [running? primary-ip private-ip]]
- [org.jclouds.compute :only [nodes-with-tag]]
+ [pallet.compute.jclouds]
+ [org.jclouds.compute2 :only [nodes-in-group]]
+
[pallet.configure :only [compute-service-properties pallet-config]])
(:require
[backtype.storm.crate.zeromq :as zeromq]
@@ -12,31 +14,31 @@
[pallet.resource.package :as package]
[pallet.resource.directory :as directory]
[pallet.resource.remote-file :as remote-file]
- [pallet.resource.exec-script :as exec-script]))
+ [pallet.action.exec-script :as exec-script]))
(defn storm-config
([] (storm-config "default"))
([conf-name] (compute-service-properties (pallet-config) [conf-name])))
(defn nimbus-ip [compute name]
- (let [running-nodes (filter running? (nodes-with-tag (str "nimbus-" name) compute))]
+ (let [running-nodes (filter running? (map (partial jclouds-node->node compute) (nodes-in-group compute (str "nimbus-" name))))]
(assert (= (count running-nodes) 1))
(primary-ip (first running-nodes))))
(defn nimbus-private-ip [compute name]
- (let [running-nodes (filter running? (nodes-with-tag (str "nimbus-" name) compute))]
+ (let [running-nodes (filter running? (map (partial jclouds-node->node compute) (nodes-in-group compute (str "nimbus-" name))))]
(assert (= (count running-nodes) 1))
(private-ip (first running-nodes))))
(defn zookeeper-ips [compute name]
(let [running-nodes (filter running?
- (nodes-with-tag (str "zookeeper-" name) compute))]
+ (map (partial jclouds-node->node compute) (nodes-in-group compute (str "zookeeper-" name))))]
(map primary-ip running-nodes)))
(defn supervisor-ips [compute name]
(let [running-nodes (filter running?
- (nodes-with-tag (str "supervisor-" name) compute))]
+ (map (partial jclouds-node->node compute) (nodes-in-group compute (str "supervisor-" name))))]
(map primary-ip running-nodes)))
(defn- install-dependencies [request]
@@ -52,40 +54,26 @@
(package/package "zip")
))
-(defn storm-release-url [release]
- (format "https://github.com/downloads/nathanmarz/storm/storm-%s.zip" release))
-
-(defn download-release [request release]
- (-> request
- (remote-file/remote-file
- (format "$HOME/storm-%s.zip" release)
- :url (storm-release-url release)
- :no-versioning true)
- ))
-
-(defn build-release-from-head [request]
- (-> request
- (exec-script/exec-checked-script
- "Build storm"
-
- (cd "$HOME")
- (mkdir -p "build")
- (cd "$HOME/build")
- (if-not (directory? "storm")
- (git clone "git://github.com/nathanmarz/storm.git"))
-
- (cd storm)
- (git pull)
- (sh "bin/build_release.sh")
- (cp "*.zip $HOME/")
- )
- ))
-
(defn get-release [request release]
- (if release
- (download-release request release)
- (build-release-from-head request)
- ))
+ (let [url "git://github.com/nathanmarz/storm.git"
+ rl (if (empty? release) "" release)] ; empty string for pallet
+
+ (-> request
+ (exec-script/exec-checked-script
+ "Build storm"
+ (cd "$HOME")
+ (mkdir -p "build")
+ (cd "$HOME/build")
+
+ (when-not (directory? "storm")
+ (if (not (empty? ~rl))
+ (git clone -b ~rl ~url)
+ (git clone ~url)))
+
+ (cd storm)
+ (git pull)
+ (bash "bin/build_release.sh")
+ (cp "*.zip $HOME/")))))
(defn make [request release]
(->
@@ -1,6 +1,6 @@
(ns backtype.storm.crate.zeromq
(:require
- [pallet.resource.exec-script :as exec-script]
+ [pallet.action.exec-script :as exec-script]
[pallet.resource.package :as package]
[pallet.resource.remote-directory :as remote-directory]
[pallet.resource.directory :as directory]
@@ -58,13 +58,18 @@
(var tmpdir (~directory/make-temp-dir "rf"))
(cd (quoted @tmpdir))
;; use frozen version of jzmq
- (git clone "https://github.com/nathanmarz/jzmq.git")
+ (git clone "git://github.com/nathanmarz/jzmq.git")
(cd "jzmq")
(export (str "JAVA_HOME="
@(dirname @(dirname @(dirname @(update-alternatives "--list" java))))))
+ ("touch src/classdist_noinst.stamp")
+ (cd "src")
+ ("CLASSPATH=.:./.:$CLASSPATH javac -d . org/zeromq/ZMQ.java org/zeromq/App.java org/zeromq/ZMQForwarder.java org/zeromq/EmbeddedLibraryTools.java org/zeromq/ZMQQueue.java org/zeromq/ZMQStreamer.java org/zeromq/ZMQException.java")
+ (cd "..")
+
("./autogen.sh")
("./configure")
(make)
@@ -13,7 +13,7 @@
[pallet.stevedore :as stevedore]
[clojure.string :as string]
[pallet.resource.package :as package]
- [pallet.resource.exec-script :as exec-script]
+ [pallet.action.exec-script :as exec-script]
[pallet.crate.crontab :as crontab]
)
(:use
@@ -23,9 +23,9 @@
[pallet.resource.exec-script :as exec-script])
(:use
[backtype.storm config]
- [pallet compute core resource]
+ [pallet compute core resource phase]
[pallet [utils :only [make-user]]]
- [org.jclouds.compute :only [nodes-with-tag]]
+ [org.jclouds.compute2 :only [nodes-in-group]]
[clojure.walk]))
(defn parse-release [release]
@@ -66,13 +66,13 @@
req
(:username *USER*)
(:public-key-path *USER*)))
- :configure (phase
+ :configure (phase-fn
(java/java :openjdk))}))
(defn zookeeper-server-spec []
(server-spec
:extends (base-server-spec)
- :phases {:configure (phase
+ :phases {:configure (phase-fn
(zookeeper/install :version "3.3.5")
(zookeeper/configure
:clientPort (storm-conf "storm.zookeeper.port")
@@ -83,25 +83,25 @@
(defn storm-base-server-spec [name]
(server-spec
:extends (base-server-spec)
- :phases {:post-configure (phase
+ :phases {:post-configure (phase-fn
(storm/write-storm-yaml
name
storm-yaml-path))
- :configure (phase
+ :configure (phase-fn
(configure-ssh-client :host-key-checking false))
- :exec (phase
+ :exec (phase-fn
(storm/exec-daemon)
(ganglia/ganglia-finish))}))
(defn supervisor-server-spec [name release]
(server-spec
:extends (storm-base-server-spec name)
- :phases {:configure (phase
+ :phases {:configure (phase-fn
(ganglia/ganglia-node (nimbus-name name))
(storm/install-supervisor
release
"/mnt/storm"))
- :post-configure (phase
+ :post-configure (phase-fn
(ganglia/ganglia-finish)
(storm/write-storm-exec
"supervisor"))}))
@@ -121,19 +121,19 @@
(defn nimbus-server-spec [name release]
(server-spec
:extends (storm-base-server-spec name)
- :phases {:configure (phase
+ :phases {:configure (phase-fn
(ganglia/ganglia-master (nimbus-name name))
(storm/install-nimbus
release
"/mnt/storm")
(storm/install-ui)
(maybe-install-drpc release))
- :post-configure (phase
+ :post-configure (phase-fn
(ganglia/ganglia-finish)
(storm/write-storm-exec
"nimbus")
)
- :exec (phase
+ :exec (phase-fn
(storm/exec-ui)
(maybe-exec-drpc release))}))
Oops, something went wrong.

0 comments on commit c607718

Please sign in to comment.