Permalink
Browse files

Allows storm-deploy to select both a storm branch and a SHA1 commit

The storm `--release` model is very unclear and not all that useful.
Currently there is a choice of passing a no release, in which case you
get a version of storm based on the latest commit from master or you
get to pass a release which gets you the latest commit from that
branch, e.g.,

    lein deploy-storm --start --name mycluster --release 0.9.0

gets you storm 0.9.0wip16 (roughly, actually it gets you commit
(d12c33543c9b2e8c8e35908f8672fa06edd18c2e, which is between 0.9.0-wip16
and 0.9.0-wip17)

and

    lein deploy-storm --start --name mycluster

gets you the latest commit from master. It's not possible to get
anything inbetween.

This commit drops the `--release` argument in favour of a `--branch`
argument (equivalent to `--release`) and a `--commit` argument, which
takes a SHA1. If a commit is passed then we attempt to checkout that
SHA1 into a new branch and use that. This gives us much more
flexibility, e.g., to checkout storm 0.9.0-rc2 you would run

        lein deploy-storm --start --name mycluster --branch master
--commit 32098d5b2694434ea43d430a4703fbe51bab268f

If you want the latest version from a specific branch (say 0.8.3) you
can execute

        lein deploy-storm --start --name mycluster --branch 0.8.3

and if you want the bleeding edge you can still execute this get the
latest commit from master

        lein deploy-storm --start --name mycluster

The diff on this commit is bigger than it might have been because i
deliberately changed `release` to `branch` wherever possible. I think
now that storm has both releases (since 0.9.9-rc1) and branches we
should make the code closer to what's actually happening (i.e.,
checking out branches rather than releases).
  • Loading branch information...
1 parent 34b0d2a commit 476553f4b129f3c402f630d3cb20b2fbdc1955b6 @lorcan lorcan committed Oct 3, 2013
@@ -4,7 +4,9 @@
[pallet.action.exec-script :as exec-script]))
;; this is 1.5.2. freezing version to ensure deploy is stable
-(def download-url "https://raw.github.com/technomancy/leiningen/a1fa43400295d57a9acfed10735c1235904a9407/bin/lein")
+;; (def download-url "https://raw.github.com/technomancy/leiningen/a1fa43400295d57a9acfed10735c1235904a9407/bin/lein")
+;; this is 2.3.2. freezing version to ensure deploy is stable
+(def download-url "https://raw.github.com/technomancy/leiningen/7d7426b14326fc5257d82d97c314e2ea8455597e/bin/lein")
(defn install [request]
(-> request
@@ -54,9 +54,9 @@
(package/package "zip")
))
-(defn get-release [request release]
+(defn get-release [request branch commit]
(let [url "git://github.com/nathanmarz/storm.git"
- rl (if (empty? release) "" release)] ; empty string for pallet
+ sha1 (if (empty? commit) "" commit)] ; empty string for pallet
(-> request
(exec-script/exec-checked-script
@@ -66,23 +66,24 @@
(cd "$HOME/build")
(when-not (directory? "storm")
- (if (not (empty? ~rl))
- (git clone -b ~rl ~url)
- (git clone ~url)))
+ (git clone -b branch ~url ))
(cd storm)
(git pull)
+ (if (not (empty? ~sha1))
+ (git checkout -b newbranch ~sha1)
+ )
(bash "bin/build_release.sh")
(cp "*.zip $HOME/")))))
-(defn make [request release]
+(defn make [request branch commit]
(->
request
(exec-script/exec-checked-script
"clean up home"
(cd "$HOME")
(rm "-f *.zip"))
- (get-release release)
+ (get-release branch commit)
(exec-script/exec-checked-script
"prepare daemon"
(cd "$HOME")
@@ -91,6 +92,7 @@
(ln "-s $HOME/`ls | grep zip | sed s/.zip//` storm")
(mkdir -p "daemon")
+ (mkdir -p "$HOME/storm/log4j")
(chmod "755" "$HOME/storm/log4j")
(touch "$HOME/storm/log4j/storm.log.properties")
(touch "$HOME/storm/log4j/log4j.properties")
@@ -102,13 +104,13 @@
(directory/directory "$HOME/storm/bin" :mode "755")
))
-(defn install-supervisor [request release local-dir-path]
+(defn install-supervisor [request branch commit local-dir-path]
(->
request
(install-dependencies)
(directory/directory local-dir-path :owner "storm" :mode "700")
- (make release)))
+ (make branch commit)))
(defn write-ui-exec [request path]
(-> request
@@ -154,12 +156,12 @@
(write-drpc-exec "$HOME/drpc/run")
))
-(defn install-nimbus [request release local-dir-path]
+(defn install-nimbus [request branch commit local-dir-path]
(->
request
(directory/directory local-dir-path :owner "storm" :mode "700")
(install-dependencies)
- (make release)))
+ (make branch commit)))
(defn exec-daemon [request]
(->
@@ -28,11 +28,11 @@
[org.jclouds.compute2 :only [nodes-in-group]]
[clojure.walk]))
-(defn parse-release [release]
- (map #(Integer/parseInt %) (.split release "\\.")))
+(defn parse-branch [branch]
+ (map #(Integer/parseInt %) (.split branch "\\.")))
-(defn release> [release1 release2]
- (->> (map - (parse-release release1) (parse-release release2))
+(defn branch> [branch1 branch2]
+ (->> (map - (parse-branch branch1) (parse-branch branch2))
(take-while #(>= % 0))
(some pos?)))
@@ -93,49 +93,49 @@
(storm/exec-daemon)
(ganglia/ganglia-finish))}))
-(defn supervisor-server-spec [name release]
+(defn supervisor-server-spec [name branch commit]
(server-spec
:extends (storm-base-server-spec name)
:phases {:configure (phase-fn
(ganglia/ganglia-node (nimbus-name name))
(storm/install-supervisor
- release
+ branch commit
"/mnt/storm"))
:post-configure (phase-fn
(ganglia/ganglia-finish)
(storm/write-storm-exec
"supervisor"))}))
-(defn maybe-install-drpc [req release]
- (if (or (not release) (release> release "0.5.3"))
+(defn maybe-install-drpc [req branch]
+ (if (or (not branch) (= branch "master") (branch> branch "0.5.3"))
(storm/install-drpc req)
req
))
-(defn maybe-exec-drpc [req release]
- (if (or (not release) (release> release "0.5.3"))
+(defn maybe-exec-drpc [req branch]
+ (if (or (not branch) (= branch "master") (branch> branch "0.5.3"))
(storm/exec-drpc req)
req
))
-(defn nimbus-server-spec [name release]
+(defn nimbus-server-spec [name branch commit]
(server-spec
:extends (storm-base-server-spec name)
:phases {:configure (phase-fn
(ganglia/ganglia-master (nimbus-name name))
(storm/install-nimbus
- release
+ branch commit
"/mnt/storm")
(storm/install-ui)
- (maybe-install-drpc release))
+ (maybe-install-drpc branch))
:post-configure (phase-fn
(ganglia/ganglia-finish)
(storm/write-storm-exec
"nimbus")
)
:exec (phase-fn
(storm/exec-ui)
- (maybe-exec-drpc release))}))
+ (maybe-exec-drpc branch))}))
(defn node-spec-from-config [group-name inbound-ports]
(letfn [(assoc-with-conf-key [image image-key conf-key & {:keys [f] :or {f identity}}]
@@ -169,8 +169,8 @@
[(storm-conf "nimbus.thrift.port")])
:extends server-spec))
-(defn nimbus [name release]
- (nimbus* name (nimbus-server-spec name release)))
+(defn nimbus [name branch commit]
+ (nimbus* name (nimbus-server-spec name branch commit)))
(defn supervisor* [name server-spec]
(group-spec
@@ -179,7 +179,7 @@
(storm-conf "supervisor.slots.ports"))
:extends server-spec))
-(defn supervisor [name release]
- (supervisor* name (supervisor-server-spec name release)))
+(defn supervisor [name branch commit]
+ (supervisor* name (supervisor-server-spec name branch commit)))
@@ -35,9 +35,9 @@
(doseq [tag all-tags]
(print-ips-for-tag! aws tag))))
-(defn converge! [name release aws sn zn nn]
- (converge {(node/nimbus name release) nn
- (node/supervisor name release) sn
+(defn converge! [name branch commit aws sn zn nn]
+ (converge {(node/nimbus name branch commit) nn
+ (node/supervisor name branch commit) sn
(node/zookeeper name) zn
}
:compute aws))
@@ -86,9 +86,9 @@
(info "Provisioning Complete.")
(print-all-ips! aws name)))
-(defn start! [aws name release]
- (println "Starting cluster with release" release)
- (start-with-nodes! aws name (node/nimbus-server-spec name release) (node/supervisor-server-spec name release) (node/zookeeper-server-spec))
+(defn start! [aws name branch commit]
+ (println "Starting cluster with storm branch " branch (if (empty? commit) "" (str " and commit " commit)))
+ (start-with-nodes! aws name (node/nimbus-server-spec name branch commit) (node/supervisor-server-spec name branch commit) (node/zookeeper-server-spec))
)
(defn upgrade-with-nodes! [aws name nimbus supervisor zookeeper]
@@ -104,14 +104,14 @@
(println "Upgrade Complete.")))
-(defn upgrade! [aws name release]
- (println "Upgrading cluster with release" release)
- (upgrade-with-nodes! aws name (node/nimbus-server-spec name release) (node/supervisor-server-spec name release) (node/zookeeper-server-spec))
+(defn upgrade! [aws name branch commit]
+ (println "Upgrading cluster with storm branch " branch (if (empty? commit) "" (str " and commit " commit)))
+ (upgrade-with-nodes! aws name (node/nimbus-server-spec name branch commit) (node/supervisor-server-spec name branch commit) (node/zookeeper-server-spec))
)
(defn stop! [aws name]
(println "Shutting Down nodes...")
- (converge! name nil aws 0 0 0)
+ (converge! name nil nil aws 0 0 0)
(println "Shutdown Finished."))
(defn mk-aws []
@@ -136,12 +136,13 @@
[upgrade? "Upgrade existing cluster"]
[ips? "Print Cluster IP Addresses?"]
[name "Cluster name" "dev"]
- [release "Release version" nil]]
+ [branch "Branch" "master"]; default branch is master.
+ [commit "Commit SHA1" nil]]; default is not to pass a commit
(cond
stop? (stop! aws name)
- start? (start! aws name release)
- upgrade? (upgrade! aws name release)
+ start? (start! aws name branch commit)
+ upgrade? (upgrade! aws name branch commit)
attach? (attach! aws name)
ips? (print-all-ips! aws name)
:else (println "Must pass --start, --stop , upgrade, --attach or --ips")))))

0 comments on commit 476553f

Please sign in to comment.