update from apache/incubator-storm

CHANGELOG.md

@@ -1,8 +1,18 @@
 ## 0.9.2-incubating (unreleased)
+ * STORM-12: reduce thread usage of netty transport
+ * STORM-281: fix and issue with config parsing that could lead to leaking file descriptors
+ * STORM-196: When JVM_OPTS are set, storm jar fails to detect storm.jar from environment
+ * STORM-260: Fix a potential race condition with simulated time in Storm's unit tests
+ * STORM-258: Update commons-io version to 2.4
+ * STORM-270: don't package .clj files in release jars.
+ * STORM-273: Error while running storm topologies on Windows using "storm jar"
+ * STROM-247: Replace links to github resources in storm script
+ * STORM-263: Update Kryo version to 2.21+
  * STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative"
  * STORM-186: fix float secs to millis long convertion
  * STORM-70: Upgrade to ZK-3.4.5 and curator-1.3.3
  * STORM-146: Unit test regression when storm is compiled with 3.4.5 zookeeper
+
 ## 0.9.1-incubating
 * Fix to prevent Nimbus from hanging if random data is sent to nimbus thrift port
 * Improved support for running on Windows platforms

bin/storm

@@ -22,6 +22,7 @@ import random
 import subprocess as sub
 import getopt
 import re
+import shlex
 
 def identity(x):
     return x
@@ -45,7 +46,7 @@ if (not os.path.isfile(USER_CONF_DIR + "/storm.yaml")):
     USER_CONF_DIR = CLUSTER_CONF_DIR
 CONFIG_OPTS = []
 CONFFILE = ""
-JAR_JVM_OPTS = os.getenv('STORM_JAR_JVM_OPTS', '')
+JAR_JVM_OPTS = shlex.split(os.getenv('STORM_JAR_JVM_OPTS', ''))
 
 
 def get_config_opts():
@@ -147,15 +148,15 @@ def jar(jarfile, klass, *args):
     Runs the main method of class with the specified arguments. 
     The storm jars and configs in ~/.storm are put on the classpath. 
     The process is configured so that StormSubmitter 
-    (http://nathanmarz.github.com/storm/doc/backtype/storm/StormSubmitter.html)
+    (http://storm.incubator.apache.org/apidocs/backtype/storm/StormSubmitter.html)
     will upload the jar at topology-jar-path when the topology is submitted.
     """
     exec_storm_class(
         klass,
         jvmtype="-client",
         extrajars=[jarfile, USER_CONF_DIR, STORM_DIR + "/bin"],
         args=args,
-        jvmopts=[' '.join(filter(None, [JAR_JVM_OPTS, "-Dstorm.jar=" + jarfile]))])
+        jvmopts=JAR_JVM_OPTS + ["-Dstorm.jar=" + jarfile])
 
 def kill(*args):
     """Syntax: [storm kill topology-name [-w wait-time-secs]]
@@ -262,7 +263,7 @@ def nimbus(klass="backtype.storm.daemon.nimbus"):
     supervision with a tool like daemontools or monit. 
 
     See Setting up a Storm cluster for more information.
-    (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
+    (http://storm.incubator.apache.org/documentation/Setting-up-a-Storm-cluster)
     """
     cppaths = [CLUSTER_CONF_DIR]
     jvmopts = parse_args(confvalue("nimbus.childopts", cppaths)) + [
@@ -282,7 +283,7 @@ def supervisor(klass="backtype.storm.daemon.supervisor"):
     under supervision with a tool like daemontools or monit. 
 
     See Setting up a Storm cluster for more information.
-    (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
+    (http://storm.incubator.apache.org/documentation/Setting-up-a-Storm-cluster)
     """
     cppaths = [CLUSTER_CONF_DIR]
     jvmopts = parse_args(confvalue("supervisor.childopts", cppaths)) + [
@@ -303,7 +304,7 @@ def ui():
     should be run under supervision with a tool like daemontools or monit. 
 
     See Setting up a Storm cluster for more information.
-    (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
+    (http://storm.incubator.apache.org/documentation/Setting-up-a-Storm-cluster)
     """
     cppaths = [CLUSTER_CONF_DIR]
     jvmopts = parse_args(confvalue("ui.childopts", cppaths)) + [
@@ -324,7 +325,7 @@ def logviewer():
     tool like daemontools or monit. 
 
     See Setting up a Storm cluster for more information.
-    (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
+    (http://storm.incubator.apache.org/documentation/Setting-up-a-Storm-cluster)
     """
     cppaths = [CLUSTER_CONF_DIR]
     jvmopts = parse_args(confvalue("logviewer.childopts", cppaths)) + [
@@ -344,7 +345,7 @@ def drpc():
     with a tool like daemontools or monit. 
 
     See Distributed RPC for more information.
-    (https://github.com/nathanmarz/storm/wiki/Distributed-RPC)
+    (http://storm.incubator.apache.org/documentation/Distributed-RPC)
     """
     cppaths = [CLUSTER_CONF_DIR]
     jvmopts = parse_args(confvalue("drpc.childopts", cppaths)) + [
@@ -392,7 +393,7 @@ def print_commands():
     """Print all client commands and link to documentation"""
     print "Commands:\n\t",  "\n\t".join(sorted(COMMANDS.keys()))
     print "\nHelp:", "\n\thelp", "\n\thelp <command>"
-    print "\nDocumentation for the storm client can be found at https://github.com/nathanmarz/storm/wiki/Command-line-client\n"
+    print "\nDocumentation for the storm client can be found at http://storm.incubator.apache.org/documentation/Command-line-client.html\n"
     print "Configs can be overridden using one or more -c flags, e.g. \"storm list -c nimbus.host=nimbus.mycompany.com\"\n"
 
 def print_usage(command=None):

examples/storm-starter/README.markdown

@@ -26,7 +26,7 @@ Next, make sure you have the storm-starter code available on your machine.  Git/
 following command to download the latest storm-starter code and change to the new directory that contains the downloaded
 code.
 
-    $ git clone git://github.com/apache/incubator-storm.git && cd storm/examples/storm-starter
+    $ git clone git://github.com/apache/incubator-storm.git && cd incubator-storm/examples/storm-starter
 
 
 ## storm-starter overview
@@ -47,16 +47,6 @@ If you want to learn more about how Storm works, please head over to the
 [Storm project page](http://storm.incubator.apache.org).
 
 
-<a name="leiningen"></a>
-
-# Using storm-starter with Leiningen
-
-## Install Leiningen
-
-The storm-starter build uses [Leiningen](http://leiningen.org/) 2.0.  Install Leiningen by following the
-[leiningen installation instructions](https://github.com/technomancy/leiningen).
-
-
 <a name="maven"></a>
 
 # Using storm-starter with Maven
@@ -106,7 +96,7 @@ The following instructions will import storm-starter as a new project in Intelli
 
 
 * Open _File > Import Project..._ and navigate to the storm-starter directory of your storm clone (e.g.
-  `~/git/storm/examples/storm-starter`).
+  `~/git/incubator-storm/examples/storm-starter`).
 * Select _Import project from external model_, select "Maven", and click _Next_.
 * In the following screen, enable the checkbox _Import Maven projects automatically_.  Leave all other values at their
   defaults.  Click _Next_.

pom.xml

@@ -114,7 +114,7 @@
         <developer>
             <id>jjackson</id>
             <name>Jason Jackson</name>
-            <email>jason@cvk.ca</email>
+            <email>jasonjckn@gmail.com</email>
             <roles>
                 <role>Committer</role>
             </roles>
@@ -153,7 +153,7 @@
         <clojure.version>1.4.0</clojure.version>
         <compojure.version>1.1.3</compojure.version>
         <hiccup.version>0.3.6</hiccup.version>
-        <commons-io.verson>1.4</commons-io.verson>
+        <commons-io.verson>2.4</commons-io.verson>
         <commons-lang.version>2.5</commons-lang.version>
         <commons-exec.version>1.1</commons-exec.version>
         <clj-time.version>0.4.1</clj-time.version>
@@ -162,7 +162,7 @@
         <ring.version>0.3.11</ring.version>
         <clojure.tools.logging.version>0.2.3</clojure.tools.logging.version>
         <clojure.math.numeric-tower.version>0.0.1</clojure.math.numeric-tower.version>
-        <carbonite.version>1.3.2</carbonite.version>
+        <carbonite.version>1.4.0</carbonite.version>
         <snakeyaml.version>1.11</snakeyaml.version>
         <httpclient.version>4.1.1</httpclient.version>
         <clojure.tools.cli.version>0.2.2</clojure.tools.cli.version>

storm-core/pom.xml

@@ -190,6 +190,10 @@
                         <testSourceDirectory>test/clj</testSourceDirectory>
                     </testSourceDirectories>
                     <warnOnReflection>false</warnOnReflection>
+                    <copyDeclaredNamespaceOnly>true</copyDeclaredNamespaceOnly>
+                    <copiedNamespaces>
+                        <copiedNamespace>none</copiedNamespace>
+                    </copiedNamespaces>
                 </configuration>
                 <executions>
                     <execution>

storm-core/src/clj/backtype/storm/ui/core.clj

@@ -485,7 +485,7 @@
            :let [disp ((display-map k) k)]]
        [(link-to (if (= k window) {:class "red"} {})
                  (url-format "/topology/%s?window=%s" id k)
-                 disp)
+                 (escape-html disp))
         (get-in stats [:emitted k])
         (get-in stats [:transferred k])
         (float-str (get-in stats [:complete-latencies k]))
@@ -717,7 +717,7 @@
            :let [disp ((display-map k) k)]]
        [(link-to (if (= k window) {:class "red"} {})
                  (url-format "/topology/%s/component/%s?window=%s" topology-id id k)
-                 disp)
+                 (escape-html disp))
         (get-in stats [:emitted k])
         (get-in stats [:transferred k])
         (float-str (get-in stats [:complete-latencies k]))
@@ -935,7 +935,7 @@
            :let [disp ((display-map k) k)]]
        [(link-to (if (= k window) {:class "red"} {})
                  (url-format "/topology/%s/component/%s?window=%s" topology-id id k)
-                 disp)
+                 (escape-html disp))
         (get-in stats [:emitted k])
         (get-in stats [:transferred k])
         (float-str (get-in stats [:execute-latencies k]))
@@ -1016,34 +1016,40 @@
        (-> (main-page)
            ui-template))
   (GET "/topology/:id" [:as {cookies :cookies} id & m]
-       (let [include-sys? (get-include-sys? cookies)]
+       (let [include-sys? (get-include-sys? cookies)
+            id (java.net.URLDecoder/decode id)]
          (try
            (-> (topology-page id (:window m) include-sys?)
              (concat [(mk-system-toggle-button include-sys?)])
              ui-template)
            (catch Exception e (resp/redirect "/")))))
   (GET "/topology/:id/component/:component" [:as {cookies :cookies} id component & m]
-       (let [include-sys? (get-include-sys? cookies)]
+       (let [include-sys? (get-include-sys? cookies)
+            id (java.net.URLDecoder/decode id)
+            component (java.net.URLDecoder/decode component)]
          (-> (component-page id component (:window m) include-sys?)
              (concat [(mk-system-toggle-button include-sys?)])
              ui-template)))
   (POST "/topology/:id/activate" [id]
     (with-nimbus nimbus
-      (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+      (let [id (java.net.URLDecoder/decode id)
+            tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
             name (.get_name tplg)]
         (.activate nimbus name)
         (log-message "Activating topology '" name "'")))
     (resp/redirect (str "/topology/" id)))
   (POST "/topology/:id/deactivate" [id]
     (with-nimbus nimbus
-      (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+      (let [id (java.net.URLDecoder/decode id)
+            tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
             name (.get_name tplg)]
         (.deactivate nimbus name)
         (log-message "Deactivating topology '" name "'")))
     (resp/redirect (str "/topology/" id)))
   (POST "/topology/:id/rebalance/:wait-time" [id wait-time]
     (with-nimbus nimbus
-      (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+      (let [id (java.net.URLDecoder/decode id)
+            tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
             name (.get_name tplg)
             options (RebalanceOptions.)]
         (.set_wait_secs options (Integer/parseInt wait-time))
@@ -1052,7 +1058,8 @@
     (resp/redirect (str "/topology/" id)))
   (POST "/topology/:id/kill/:wait-time" [id wait-time]
     (with-nimbus nimbus
-      (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+      (let [id (java.net.URLDecoder/decode id)
+            tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
             name (.get_name tplg)
             options (KillOptions.)]
         (.set_wait_secs options (Integer/parseInt wait-time))

storm-core/src/clj/backtype/storm/util.clj

@@ -759,7 +759,7 @@
 
 (defn zip-contains-dir? [zipfile target]
   (let [entries (->> zipfile (ZipFile.) .entries enumeration-seq (map (memfn getName)))]
-    (some? #(.startsWith % (str target file-path-separator)) entries)
+    (some? #(.startsWith % (str target "/")) entries)
     ))
 
 (defn url-encode [s]