apache · shani-einav · Apr 20, 2016 · Apr 20, 2016 · Apr 20, 2016 · Apr 15, 2016
diff --git a/.gitignore b/.gitignore
@@ -2,9 +2,7 @@
 /classes
 /lib
 deploy/lib
-deploy/logs
 .emacs-project
-*.jar
 bin/jzmq
 .DS_Store
 deploy/classes
@@ -21,20 +19,38 @@ _release
 *.zip
 *.tar.gz
 .lein-deps-sum
-*.iml
 target
 /.project/
 /.lein-plugins/
-*.ipr
-*.iws
-.idea
 #.*
 !/.travis.yml
 !/.gitignore
 _site
 dependency-reduced-pom.xml
-derby.log
 metastore_db
+build
+/docs/javadocs
+*.class
+
+# logs
+logs
+*.log
+
+# Eclipse
 .settings/
 .project
 .classpath
+
+# Intellij
+*.iml
+*.ipr
+*.iws
+.idea
+
+# Package Files
+*.jar
+*.war
+*.ear
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,156 @@
-## 2.0.0
+## 1.0.2
+ * STORM-1876: Option to build storm-kafka and storm-kafka-client with different kafka client version
+ * STORM-1989: X-Frame-Options support for Storm UI
+ * STORM-1962: support python 3 and 2 in multilang
+ * STORM-1964: Unexpected behavior when using count window together with timestamp extraction
+ * STORM-1977: Restore logic: give up leadership when elected as leader but doesn't have one or more topology codes on local 
+ * STORM-1890 ensure we refetch static resources after package build
+ * STORM-1730: LocalCluster#shutdown() does not terminate all storm threds/thread pools.
+ * STORM-1976: STORM-1976 Remove cleanup-corrupt-topologies!
+ * STORM-1939: Frequent InterruptedException raised by ShellBoltMessageQueue.poll
+ * STORM-1968: Storm logviewer does not work for nimbus.log in secure cluster
+ * STORM-1928: ShellSpout should check heartbeat while ShellSpout is waiting for subprocess to sync
+ * STORM-1922: Supervisor summary default order by host
+ * STORM-1956: Disabling Backpressure by default
+ * STORM-1934: Fix race condition between sync-supervisor and sync-processes
+ * STORM-1895: blobstore replication-factor argument
+ * STORM-118: Docs: typo in transactional-commit-flow.png
+ * STORM-1633: Document blobstore to command-line-client.md
+ * STORM-1899: Release HBase connection when topology shutdown
+ * STORM-1844: Some tests are flaky due to low timeout
+ * STORM-1946: initialize lastHeartbeatTimestamp before starting heartbeat task
+ * STORM-1941 Nimbus discovery can fail when zookeeper reconnect happens
+ * STORM-1937 Fix WindowTridentProcessor cause NullPointerException
+ * STORM-1924: Add a config file parameter to HDFS test topologies
+ * STORM-1705: Cap number of retries for a failed message
+ * STORM-1884: Prioritize pendingPrepare over pendingCommit
+ * STORM-1575: fix TwitterSampleSpout NPE on close
+ * STORM-1874: Update logger private permissions
+ * STORM-1865: update command line client document
+ * STORM-1771: HiveState should flushAndClose before closing old or idle Hive connections
+ * STORM-1882: Expose TextFileReader public
+ * STORM-1864 : StormSubmitter should throw respective exceptions and log respective errors for registered submitter hook invocation
+ * STORM-1887: Fixed help message for set_log_level command
+ * STORM-1878: Flux can now handle IStatefulBolts
+ * STORM-1861: Storm submit command returns exit code of 0 even when it fails.
+ * STORM-1755: Revert the kafka client version upgrade in storm-kafka module
+ * STORM-1853: Replace ClassLoaderObjectInputStream with ObjectInputStream
+ * STORM-1868: Modify TridentKafkaWordCount to run in distributed mode
+ * STORM-1859: Ack late tuples in windowed mode
+ * STORM-1851: Fix default nimbus impersonation authorizer config
+ * STORM-1848: Make KafkaMessageId and Partition serializable to support event logging
+ * STORM-1862: Flux ShellSpout and ShellBolt can't emit to named streams
+ * Storm-1728: TransactionalTridentKafkaSpout error
+ * STORM-1850: State Checkpointing Documentation update
+ * STORM-1674: Idle KafkaSpout consumes more bandwidth than needed
+ * STORM-1842: Forward references in storm.thrift cause tooling issues
+ * STORM-1745: Add partition to log output in PartitionManager
+ * STORM-1735: Nimbus should log that replication succeeded when min replicas was reached exactly
+ * STORM-1835: add lock info in thread dump
+ * STORM-1749: Fix storm-starter github links
+ * STORM-1764: Pacemaker is throwing some stack traces
+ * STORM-1761: Storm-Solr Example Throws ArrayIndexOutOfBoundsException in Remote Cluster Mode
+ * STORM-1756: Explicitly null KafkaServer reference in KafkaTestBroker to prevent out of memory on large test classes.
+ * STORM-1750: Ensure worker dies when report-error-and-die is called.
+ * STORM-1715: using Jedis Protocol.DEFAULT_HOST to replace DEFAULT_HOST
+ * STORM-1713: Replace NotImplementedException with UnsupportedOperationException
+ * STORM-1678: abstract batch processing to common api `BatchHelper`
+ * STORM-1773: Utils.javaDeserialize() doesn't work with primitive types
+ * STORM-1661: Introduce a config to turn off blobstore acl validation in insecure mode
+
+## 1.0.1
+ * STORM-1741: remove unconditional setting of JAVA_HOME from storm-env.sh
+ * STORM-1739: update the minor JAVA version dependency in 0.10.0 and above
+ * STORM-1727: document 1.0 package renaming and how to use the migration tool
+ * STORM-1733: Flush stdout and stderr before calling "os.execvp" to prevent log loss.
+ * STORM-1729: Get rid of reflections while recording stats
+ * STORM-1731: Avoid looking up debug / backpressure enable flags within critical path
+ * STORM-1535: Make sure hdfs key tab login happens only once for multiple bolts/executors.
+ * STORM-1725: Kafka Spout New Consumer API - KafkaSpoutRetryExponential Backoff method should use HashMap instead of TreeMap not to throw Exception
+ * STORM-1544: Document Debug/Sampling of Topologies
+ * STORM-1679: add storm Scheduler documents
+ * STORM-1704: When logviewer_search.html opens daemon file, next search always show no result
+ * STORM-1714: StatefulBolts ends up as normal bolts while using TopologyBuilder.setBolt without parallelism
+ * STORM-1683: only check non-system streams by default
+ * STORM-1680: Provide configuration to set min fetch size in KafkaSpout
+ * STORM-1649: Optimize Kryo instaces creation in HBaseWindowsStore
+ * STORM-1696: status not sync if zk fails in backpressure
+ * STORM-1693: Move stats cleanup to executor shutdown
+ * STORM-1585: Add DDL support for UDFs in storm-sql
+ * STORM-1681: Bug in scheduling cyclic topologies when scheduling with RAS
+ * STORM-1706: Add RELEASE and storm-env.sh to storm-diet assembly
+ * STORM-1613: Upgraded HBase version to 1.1.0
+ * STORM-1687: divide by zero in stats
 
 ## 1.0.0
+ * STORM-1670: LocalState#get(String) can throw FileNotFoundException which may result supervisor.clj#sync-processes stop assigning new workers/assignments
+ * STORM-1677: Test resource files are excluded from source distribution, which makes logviewer-test failing
+ * STORM-676: Storm Trident support for sliding/tumbling windows
+ * STORM-1671: Enable logviewer to delete a dir without yaml
+ * STORM-822: Kafka Spout New Consumer API
+ * STORM-1673: log4j2/worker.xml refers old package of LoggerMetricsConsumer
+ * STORM-1632 Disable event logging by default
+ * STORM-1667: Log the IO exception when deleting worker pid dir
+ * STORM-1669: Fix SolrUpdateBolt flush bug
+ * STORM-1668: Flux silently fails while setting a non-existent property.
+ * STORM-1573: Add batch support for MongoInsertBolt
+ * STORM-1660: remove flux gitignore file and move rules to top level gitignore
+ * STORM-1622: Rename classes with older third party shaded packages
+ * STORM-1537: Upgrade to kryo 3
+ * STORM-1556: nimbus.clj/wait-for-desired-code-replication wrong reset for current-replication-count-jar in local mode
+ * STORM-1636: Supervisor shutdown with worker id pass in being nil
+ * STORM-1641: make subtree node creation consistent
+ * STORM-1604: Delayed transition should handle NotALeaderException
+ * STORM-1602: Blobstore UTs are failed on Windows
+ * STORM-1629: Files/move doesn't work properly with non-empty directory in Windows
+ * STORM-1616: Add RAS API for Trident
+ * STORM-1483: add storm-mongodb connector
+ * STORM-1614: backpressure init and cleanup changes
+ * STORM-1549: Add support for resetting tuple timeout from bolts via the OutputCollector
+ * STORM-971: Metric for messages lost due to kafka retention
+ * STORM-1608: Fix stateful topology acking behavior
+ * STORM-1609: Netty Client is not best effort delivery on failed Connection
+ * STORM-1620: Update curator to fix CURATOR-209
+ * STORM-1469: Decommission SimpleTransportPlugin and configuration
+ * STORM-1469: Adding Plain Sasl Transport Plugin
+ * STORM-1588: Do not add event logger details if number of event loggers is zero
+ * STORM-1606: print the information of testcase which is on failure
+ * STORM-1436: Set Travis Heap size to fit in memory limits in travis builds.
+ * STORM-1529: Change default worker temp directory location for workers
+ * STORM-1543: DRPCSpout should always try to reconnect disconnected DRPCInvocationsClient
+ * STORM-1561: Supervisor should relaunch worker if assignments have changed
+ * STORM-1601: Check if /backpressure/storm-id node exists before requesting children
+ * STORM-1574: Better handle backpressure exception etc.
+ * STORM-1587: Avoid NPE while prining Metrics
+ * STORM-1570: Storm SQL support for nested fields and array
+ * STORM-1576: fix ConcurrentModificationException in addCheckpointInputs
+ * STORM-1521: When using Kerberos login from keytab with multiple bolts/executors ticket is not renewed
+ * STORM-1488: UI Topology Page component last error timestamp is from 1970
+ * STORM-1542: Remove profile action retry in case of non-zero exit code
+ * STORM-1540: Fix Debug/Sampling for Trident
+ * STORM-1569: Allowing users to specify the nimbus thrift server queue size.
+ * STORM-1552: Fix topology event sampling log dir 
+ * STORM-1511: min/max operations support on a trident stream
+ * STORM-1522: REST API throws invalid worker log links
+ * STORM-1532: Fix readCommandLineOpts to parse JSON correctly
+ * STORM-1541: Change scope of 'hadoop-minicluster' to test
+ * STORM-1539: Improve Storm ACK-ing performance
+ * STORM-1519: Storm syslog logging not confirming to RFC5426 3.1
+ * STORM-1533: IntegerValidator for metric consumer parallelism hint
+ * STORM-1534: Pick correct version of jackson-annotations jar
+ * STORM-1476: Filter -c options from args and add them as part of storm.options
+ * STORM-1520: Nimbus Clojure/Zookeeper issue ("stateChanged" method not found)
+ * STORM-1531: Junit and mockito dependencies need to have correct scope defined in storm-elasticsearch pom.xml
+ * STORM-1526: Improve Storm core performance
+ * STORM-1524: Add Pluggable daemon metrics Reporters
+ * STORM-1517: Add peek api in trident stream
+ * STORM-1455: kafka spout should not reset to the beginning of partition when offsetoutofrange exception occurs
+ * STORM-1518: Backport of STORM-1504
+ * STORM-1505: Add map, flatMap and filter functions in trident stream
+ * STORM-1510: Fix broken nimbus log link
+ * STORM-1503: Worker should not crash on failure to send heartbeats to Pacemaker/ZK
+ * STORM-1176: Checkpoint window evaluated/expired state
+ * STORM-1494: Add link to supervisor log in Storm UI
  * STORM-1496: Nimbus periodically throws blobstore-related exception
  * STORM-1484: ignore subproject .classpath & .project file
  * STORM-1478: make bolts getComponentConfiguration method cleaner/simpler
@@ -249,8 +399,16 @@
  * STORM-1144: Display requested and assigned cpu/mem resources for schedulers in UI
  * STORM-1217: making small fixes in RAS
 
-## 0.10.1
+## 0.10.2
+ * STORM-1834: Documentation How to Generate Certificates For Local Testing SSL Setup
+ * STORM-1754: Correct java version in 0.10.x storm-starter
+ * STORM-1750: Ensure worker dies when report-error-and-die is called.
+ * STORM-1739: update the minor JAVA version dependency in 0.10.0 and above
+ * STORM-1733: Flush stdout and stderr before calling "os.execvp" to prevent log loss
 
+## 0.10.1
+ * STORM-584: Fix logging for LoggingMetricsConsumer metrics.log file
+ * STORM-1596: Do not use single Kerberos TGT instance between multiple threads
  * STORM-1481: avoid Math.abs(Integer) get a negative value
  * STORM-1121: Deprecate test only configuraton nimbus.reassign
  * STORM-1180: FLUX logo wasn't appearing quite right

diff --git a/README.markdown b/README.markdown
@@ -91,6 +91,7 @@ under the License.
 * Boyang Jerry Peng ([@jerrypeng](https://github.com/jerrypeng))
 * Zhuo Liu ([@zhuoliu](https://github.com/zhuoliu))
 * Haohui Mai ([@haohui](https://github.com/haohui))
+* Longda Feng ([@longda](https://github.com/longdafeng))
 
 ## Contributors
 

diff --git a/SECURITY.md b/SECURITY.md
@@ -133,7 +133,31 @@ To set up 2-way authentication:
 1. `drpc.https.want.client.auth` (If this set to true, server requests for client certificate authentication, but keeps the connection even if no authentication is provided)
 2. `drpc.https.need.client.auth` (If this set to true, server requires the client to provide authentication)
 
+#### GENERATE CERTIFICATES FOR LOCAL TESTING SSL SETUP
 
+Run the following script and fill in the values and passwords when prompted. The `keyalg` must be set to `RSA`
+
+```bash
+#!/bin/bash
+
+DIR=/Users/user/certs/dir/
+
+keytool -keystore $DIR/server.keystore.jks -alias localhost -validity 365 -keyalg RSA -genkey
+
+openssl req -new -x509 -keyout $DIR/ca-key -out $DIR/ca-cert -days 365
+
+keytool -keystore $DIR/server.truststore.jks -alias CARoot -import -file $DIR/ca-cert
+
+keytool -keystore $DIR/client.truststore.jks -alias CARoot -import -file $DIR/ca-cert
+
+keytool -keystore $DIR/server.keystore.jks -alias localhost -certreq -file $DIR/cert-file
+
+openssl x509 -req -CA $DIR/ca-cert -CAkey $DIR/ca-key -in $DIR/cert-file -out $DIR/cert-signed -days 365 -CAcreateserial -passin pass:test12
+
+keytool -keystore $DIR/server.keystore.jks -alias CARoot -import -file $DIR/ca-cert
+
+keytool -keystore $DIR/server.keystore.jks -alias localhost -import -file $DIR/cert-signed
+```
 
 ## Authentication (Kerberos)
 
@@ -469,7 +493,6 @@ nimbus.groups:
    - "storm"
 ```
 
-
 ### DRPC
 Hopefully more on this soon
 

diff --git a/bin/flight.bash b/bin/flight.bash
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-JDKPATH="/home/y/share/yjava_jdk/java"
+JDKPATH=$JAVA_HOME
 BINPATH="/usr/bin"
 USER=`whoami`
 
@@ -59,7 +59,7 @@ function dump_record {
 
 function jstack_record {
     FILENAME=jstack-$1-${NOW}.txt
-    $BINPATH/jstack $1 > "$2/${FILENAME}"
+    $BINPATH/jstack $1 > "$2/${FILENAME}" 2>&1
 }
 
 function jmap_record {

diff --git a/bin/storm-config.cmd b/bin/storm-config.cmd
@@ -86,6 +86,10 @@ if not defined STORM_LOG_DIR (
 @rem retrieve storm.log4j2.conf.dir from conf file
 @rem
 
+if not defined CMD_TEMP_FILE (
+  set CMD_TEMP_FILE=tmpfile
+)
+
 "%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" org.apache.storm.command.config_value storm.log4j2.conf.dir > %CMD_TEMP_FILE%
 
 FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
@@ -125,7 +129,11 @@ FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
 
 
 :storm_opts
- set STORM_OPTS=-Dstorm.options= -Dstorm.home=%STORM_HOME% -Djava.library.path=%JAVA_LIBRARY_PATH%;%JAVA_HOME%\bin;%JAVA_HOME%\lib;%JAVA_HOME%\jre\bin;%JAVA_HOME%\jre\lib
+ if "%set_storm_options%"=="true" (
+  set STORM_OPTS=-Dstorm.options=
+ )
+
+ set STORM_OPTS=%STORM_OPTS% -Dstorm.home=%STORM_HOME% -Djava.library.path=%JAVA_LIBRARY_PATH%;%JAVA_HOME%\bin;%JAVA_HOME%\lib;%JAVA_HOME%\jre\bin;%JAVA_HOME%\jre\lib
  set STORM_OPTS=%STORM_OPTS% -Dlog4j.configurationFile=%STORM_LOG4J2_CONFIGURATION_FILE%
  set STORM_OPTS=%STORM_OPTS% -Dstorm.log.dir=%STORM_LOG_DIR%
  del /F %CMD_TEMP_FILE%

diff --git a/bin/storm.cmd b/bin/storm.cmd
@@ -37,9 +37,14 @@
 :main
   setlocal enabledelayedexpansion
 
+  set storm-command=%1
+
+  if not "%storm-command%" == "jar" (
+    set set_storm_options=true
+  )
+
   call %~dp0storm-config.cmd
 
-  set storm-command=%1
   if not defined storm-command (
       goto print_usage
   )
@@ -66,19 +71,45 @@
   )
 
   if %storm-command% == jar (
-    set STORM_OPTS=%STORM_CLIENT_OPTS% %STORM_OPTS% -Dstorm.jar=%2
-    set CLASSPATH=%CLASSPATH%;%2
-    set CLASS=%3
-    set args=%4
+    set config-options=
+
     goto start
     :start
     shift
-    if [%4] == [] goto done
-    set args=%args% %4
+    if [%1] == [] goto done
+
+    if '%1'=='-c' (
+      set c-opt=first
+      goto start
+    )
+
+    if "%c-opt%"=="first" (
+      set config-options=%config-options%,%1
+      set c-opt=second
+      goto start
+    )
+
+    if "%c-opt%"=="second" (
+      set config-options=%config-options%=%~1
+      set c-opt=
+      goto start
+    )
+
+    set args=%args% %1
     goto start
 
     :done
-    set storm-command-arguments=%args%
+    for /F "tokens=1,2,*" %%a in ("%args%") do (
+      set first-arg=%%a
+      set second-arg=%%b
+      set remaining-args=%%c
+    )
+    set STORM_OPTS=%STORM_CLIENT_OPTS% %STORM_OPTS% -Dstorm.jar=%first-arg%
+    set STORM_OPTS=%STORM_OPTS% -Dstorm.options=%config-options%
+    set CLASSPATH=%CLASSPATH%;%first-arg%
+    set CLASS=%second-arg%
+    set storm-command-arguments=%remaining-args%
+
   )
 
   if not defined STORM_LOG_FILE (