Release 2.0.4-SNAPSHOT

bin/check_jstorm_Supervisor.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+JAVA_HOME=/opt/taobao/java
+export PATH=$PATH:$JAVA_HOME/bin
+
+LOG=/home/admin/logs/check.log
+SP=`ps -ef |grep com.alibaba.jstorm.daemon.supervisor.Supervisor |grep -v grep |wc -l`
+if [ $SP -lt 1 ];then
+	mkdir -p /home/admin/logs
+        echo -e "`date` [ERROR] no process and restart Jstorm Suppervisor" >>$LOG
+        cd /home/admin/bin; nohup /home/admin/jstorm/bin/jstorm supervisor >/dev/null 2>&1  &
+else
+        echo -e "`date`  [INFO:] return $SP Jstorm Supervisor ok " >>$LOG
+fi

jstorm-server/bin/jstorm.py → bin/jstorm.py

@@ -1,3 +1,21 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 #!/usr/bin/python
 
 import os
@@ -24,8 +42,10 @@ def cygpath(x):
 CLIENT_CONF_FILE = ""
 JSTORM_DIR = "/".join(os.path.realpath( __file__ ).split("/")[:-2])
 JSTORM_CONF_DIR = os.getenv("JSTORM_CONF_DIR", JSTORM_DIR + "/conf" )
-LOG4J_CONF = JSTORM_CONF_DIR + "/jstorm.log4j.properties"
+LOGBACK_CONF = JSTORM_CONF_DIR + "/jstorm.logback.xml"
 CONFIG_OPTS = []
+EXCLUDE_JARS = []
+INCLUDE_JARS = []
 STATUS = 0
 
 
@@ -41,13 +61,15 @@ def get_config_opts():
     return "-Dstorm.options=" + (','.join(CONFIG_OPTS)).replace(' ', "%%%%")
 
 def get_client_childopts():
-    ret = (" -Dstorm.root.logger=INFO,stdout -Dlog4j.configuration=File:%s/conf/aloha_log4j.properties "  %JSTORM_DIR)
+    ret = (" -Dstorm.root.logger=INFO,stdout -Dlogback.configurationFile=" + JSTORM_DIR +
+           "/conf/client_logback.xml -Dlog4j.configuration=File:" + JSTORM_DIR + 
+           "/conf/client_log4j.properties")
     if CLIENT_CONF_FILE != "":
         ret += (" -Dstorm.conf.file=" + CLIENT_CONF_FILE)
     return ret
 
 def get_server_childopts(log_name):
-    ret = (" -Dlogfile.name=%s -Dlog4j.configuration=File:%s"  %(log_name, LOG4J_CONF))
+    ret = (" -Dlogfile.name=%s -Dlogback.configurationFile=%s"  %(log_name, LOGBACK_CONF))
     return ret
 
 if not os.path.exists(JSTORM_DIR + "/RELEASE"):
@@ -61,15 +83,27 @@ def get_jars_full(adir):
     files = os.listdir(adir)
     ret = []
     for f in files:
-        if f.endswith(".jar"):
+        if f.endswith(".jar") == False:
+            continue
+        filter = False
+        for exclude_jar in EXCLUDE_JARS:
+            if f.find(exclude_jar) >= 0:
+                filter = True
+                break
+
+        if filter == True:
+            print "Don't add " + f + " to classpath"
+        else:
             ret.append(adir + "/" + f)
     return ret
 
 def get_classpath(extrajars):
     ret = []
+    ret.extend(extrajars)
     ret.extend(get_jars_full(JSTORM_DIR))
     ret.extend(get_jars_full(JSTORM_DIR + "/lib"))
-    ret.extend(extrajars)
+    ret.extend(INCLUDE_JARS)
+
     return normclasspath(":".join(ret))
 
 def confvalue(name, extrapaths):
@@ -119,7 +153,7 @@ def jar(jarfile, klass, *args):
     """Syntax: [jstorm jar topology-jar-path class ...]
 
     Runs the main method of class with the specified arguments. 
-    The jstorm jars and configs in ~/.jstorm are put on the classpath. 
+    The jstorm jars and configs in $JSTORM_CONF_DIR/storm.yaml are put on the classpath. 
     The process is configured so that StormSubmitter 
     (https://github.com/alibaba/jstorm/wiki/JStorm-Chinese-Documentation)
     will upload the jar at topology-jar-path when the topology is submitted.
@@ -229,6 +263,17 @@ def restart(*args):
         extrajars=[JSTORM_CONF_DIR, JSTORM_DIR + "/bin", CLIENT_CONF_FILE],
         childopts=childopts)
 
+def update_config(*args):
+    """Syntax: [jstorm restart topology-name [conf]]
+    """
+    childopts = get_client_childopts()
+    exec_storm_class(
+        "backtype.storm.command.update_config",
+        args=args,
+        jvmtype="-client -Xms256m -Xmx256m",
+        extrajars=[JSTORM_CONF_DIR, JSTORM_DIR + "/bin", CLIENT_CONF_FILE],
+        childopts=childopts)
+
 def nimbus():
     """Syntax: [jstorm nimbus]
 
@@ -291,9 +336,11 @@ def print_classpath():
 
 def print_commands():
     """Print all client commands and link to documentation"""
-    print "jstorm command [--config client_storm.yaml] [command parameter]"
+    print "jstorm command [--config client_storm.yaml] [--exclude-jars exclude1.jar,exclude2.jar] [-c key1=value1,key2=value2][command parameter]"
     print "Commands:\n\t",  "\n\t".join(sorted(COMMANDS.keys()))
-    print "\n\t[--config client_storm.yaml]\t\t optional, setting client's storm.yaml"
+    print "\n\t[--config client_storm.yaml]\t\t\t optional, setting client's storm.yaml"
+    print "\n\t[--exclude-jars exclude1.jar,exclude2.jar]\t optional, exclude jars, avoid jar conflict"
+    print "\n\t[-c key1=value1,key2=value2]\t\t\t optional, add key=value pair to configuration"
     print "\nHelp:", "\n\thelp", "\n\thelp <command>"
     print "\nDocumentation for the jstorm client can be found at https://github.com/alibaba/jstorm/wiki/JStorm-Chinese-Documentation\n"
 
@@ -341,14 +388,26 @@ def list(*args):
             "drpc": drpc, "supervisor": supervisor, "localconfvalue": print_localconfvalue,
             "remoteconfvalue": print_remoteconfvalue, "classpath": print_classpath,
             "activate": activate, "deactivate": deactivate, "rebalance": rebalance, "help": print_usage,
-	    "metricsMonitor": metrics_Monitor, "list": list, "restart": restart}
+            "metricsMonitor": metrics_Monitor, "list": list, "restart": restart, "update_config": update_config}
 
 def parse_config(config_list):
     global CONFIG_OPTS
     if len(config_list) > 0:
         for config in config_list:
             CONFIG_OPTS.append(config)
 
+def parse_exclude_jars(jars):
+    global EXCLUDE_JARS
+    EXCLUDE_JARS = jars.split(",")
+    print " Excludes jars:"
+    print EXCLUDE_JARS
+
+def parse_include_jars(jars):
+    global INCLUDE_JARS
+    INCLUDE_JARS = jars.split(",")
+    print " Include jars:"
+    print INCLUDE_JARS
+
 def parse_config_opts(args):
   curr = args[:]
   curr.reverse()
@@ -362,6 +421,10 @@ def parse_config_opts(args):
     elif token == "--config":
       global CLIENT_CONF_FILE
       CLIENT_CONF_FILE = curr.pop()
+    elif token == "--exclude-jars":
+      parse_exclude_jars(curr.pop())
+    elif token == "--include-jars":
+      parse_include_jars(curr.pop())
     else:
       args_list.append(token)
 

bin/start.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+
+if [ -e  ~/.bashrc ]
+then
+    source ~/.bashrc
+fi
+
+if [ -e  ~/.bash_profile ]
+then
+    source ~/.bash_profile
+fi
+
+if [ "x$JAVA_HOME" != "x" ]
+then
+    echo "JAVA_HOME has been set " 
+else
+    export JAVA_HOME=/opt/taobao/java
+fi
+echo "JAVA_HOME =" $JAVA_HOME
+
+if [ "x$JSTORM_HOME" != "x" ]
+then
+    echo "JSTORM_HOME has been set "
+else
+    export JSTORM_HOME=/home/admin/jstorm
+fi
+echo "JSTORM_HOME =" $JSTORM_HOME
+
+if [ "x$JSTORM_CONF_DIR_PATH" != "x" ]
+then
+    echo "JSTORM_CONF_DIR_PATH has been set " 
+else
+    export JSTORM_CONF_DIR_PATH=$JSTORM_HOME/conf
+fi
+echo "JSTORM_CONF_DIR_PATH =" $JSTORM_CONF_DIR_PATH
+
+
+
+export PATH=$JAVA_HOME/bin:$JSTORM_HOME/bin:$PATH
+
+
+which java
+
+if [ $? -eq 0 ]
+then
+    echo "Find java"
+else
+    echo "No java, please install java firstly !!!"
+    exit 1
+fi
+
+function startJStorm()
+{
+	PROCESS=$1
+  echo "start $PROCESS"
+  cd $JSTORM_HOME/bin; nohup $JSTORM_HOME/bin/jstorm $PROCESS >/dev/null 2>&1 &
+	sleep 4
+	rm -rf nohup
+	ps -ef|grep $2
+}
+
+
+
+HOSTNAME=`hostname -i`
+NIMBUS_HOST=`grep "nimbus.host:" $JSTORM_CONF_DIR_PATH/storm.yaml  | grep -w $HOSTNAME`
+SUPERVISOR_HOST_START=`grep "supervisor.host.start:" $JSTORM_CONF_DIR_PATH/storm.yaml  | grep -w "false"`
+
+if [ "X${NIMBUS_HOST}" != "X" ]
+then
+	startJStorm "nimbus" "NimbusServer"
+fi
+
+if [ "X${SUPERVISOR_HOST_START}" == "X" ]
+then
+	startJStorm "supervisor" "Supervisor"
+fi
+
+echo "Successfully  start jstorm daemon...."

conf/client_logback.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration scan="true" scanPeriod="30 seconds">
+	<appender name="A1" class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern>%-4r [%t] %-5p %c - %m%n</pattern>
+		</encoder>
+	</appender>
+	<logger name="org.apache.zookeeper" level="ERROR" />
+	<logger name="com.netflix.curator" level="ERROR" />
+	<logger name="com.alibaba.jstorm.common.metric" level="ERROR" />	
+	<logger name="com.alibaba.jstorm.daemon.nimbus.TopologyMetricsRunnable" level="ERROR" />
+	<logger name="com.alibaba.jstorm.metric" level="ERROR" />
+
+	<root level="INFO">
+		<appender-ref ref="A1" />
+	</root>
+</configuration>

jstorm-server/conf/jstorm.logback.xml → conf/jstorm.logback.xml

@@ -25,7 +25,7 @@
 		class="ch.qos.logback.core.rolling.RollingFileAppender">
 		<file>${jstorm.home}/logs/${logfile.name}.metrics</file>
 		<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
-			<fileNamePattern>${logfile.name}.metrics.%i</fileNamePattern>
+			<fileNamePattern>${jstorm.home}/logs/${logfile.name}.metrics.%i</fileNamePattern>
 			<minIndex>1</minIndex>
 			<maxIndex>5</maxIndex>
 		</rollingPolicy>
@@ -45,12 +45,12 @@
 	</root>
 
 	<logger name="com.alibaba.jstorm"
-		additivity="true">
+		additivity="false">
 		<level value="INFO" />
 		<appender-ref ref="A1" />
 	</logger>
 
-	<logger name="com.alibaba.jstorm.daemon.worker.metrics"
+	<logger name="com.alibaba.jstorm.common.metric"
 		additivity="false">
 		<level value="INFO" />
 		<appender-ref ref="METRICS" />
@@ -68,6 +68,12 @@
 		<appender-ref ref="METRICS" />
 	</logger>
 
+	<logger name="com.alibaba.jstorm.daemon.nimbus.TopologyMetricsRunnable"
+		additivity="false">
+		<level value="INFO" />
+		<appender-ref ref="METRICS" />
+	</logger>
+
 	<logger name="com.alibaba.jstorm.metric"
 		additivity="false">
 		<level value="INFO" />

dev-tools/add_apache_license.sh

@@ -0,0 +1 @@
+find . -name \*.java -exec sh -c "if ! grep -q 'LICENSE-2.0' '{}';then mv '{}' tmp && cp LICENSEHEADER.txt '{}' && cat tmp >> '{}' && rm tmp;fi" \;

dev-tools/java_license_header.txt

@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

dev-tools/py_license_header.txt

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+