Permalink
Browse files

move master to 0.9.0

  • Loading branch information...
2 parents 446037a + 3e29f13 commit 8935c381188691eb3ed8d9bebf9d3bd8eb1b05cf @nathanmarz nathanmarz committed Jan 24, 2013
Showing with 1,284 additions and 272 deletions.
  1. +3 −0 .gitignore
  2. +8 −0 CHANGELOG.md
  3. +20 −14 bin/build_release.sh
  4. +14 −11 bin/storm
  5. +24 −19 bin/to_maven.sh
  6. +1 −1 conf/defaults.yaml
  7. +0 −10 conf/log4j.properties
  8. +13 −0 conf/logback.xml
  9. +7 −0 conf/storm.yaml.example
  10. +0 −9 log4j/storm.log.properties
  11. +24 −0 logback/cluster.xml
  12. +29 −22 project.clj
  13. +1 −1 src/clj/backtype/storm/bootstrap.clj
  14. +64 −0 src/clj/backtype/storm/daemon/builtin_metrics.clj
  15. +63 −10 src/clj/backtype/storm/daemon/common.clj
  16. +92 −35 src/clj/backtype/storm/daemon/executor.clj
  17. +1 −1 src/clj/backtype/storm/daemon/supervisor.clj
  18. +13 −7 src/clj/backtype/storm/daemon/task.clj
  19. +1 −1 src/clj/backtype/storm/daemon/worker.clj
  20. +53 −0 src/clj/backtype/storm/metric/testing.clj
  21. +4 −2 src/clj/backtype/storm/testing.clj
  22. +1 −1 src/clj/backtype/storm/util.clj
  23. +32 −0 src/jvm/backtype/storm/Config.java
  24. +4 −0 src/jvm/backtype/storm/Constants.java
  25. +3 −2 src/jvm/backtype/storm/StormSubmitter.java
  26. +3 −2 src/jvm/backtype/storm/coordination/BatchBoltExecutor.java
  27. +3 −2 src/jvm/backtype/storm/coordination/CoordinatedBolt.java
  28. +3 −2 src/jvm/backtype/storm/drpc/DRPCSpout.java
  29. +3 −2 src/jvm/backtype/storm/drpc/JoinResult.java
  30. +3 −2 src/jvm/backtype/storm/drpc/ReturnResults.java
  31. +47 −0 src/jvm/backtype/storm/metric/MetricsConsumerBolt.java
  32. +17 −0 src/jvm/backtype/storm/metric/api/AssignableMetric.java
  33. +21 −0 src/jvm/backtype/storm/metric/api/CombinedMetric.java
  34. +24 −0 src/jvm/backtype/storm/metric/api/CountMetric.java
  35. +6 −0 src/jvm/backtype/storm/metric/api/ICombiner.java
  36. +5 −0 src/jvm/backtype/storm/metric/api/IMetric.java
  37. +43 −0 src/jvm/backtype/storm/metric/api/IMetricsConsumer.java
  38. +7 −0 src/jvm/backtype/storm/metric/api/IReducer.java
  39. +38 −0 src/jvm/backtype/storm/metric/api/MeanReducer.java
  40. +28 −0 src/jvm/backtype/storm/metric/api/MultiCountMetric.java
  41. +33 −0 src/jvm/backtype/storm/metric/api/MultiReducedMetric.java
  42. +21 −0 src/jvm/backtype/storm/metric/api/ReducedMetric.java
  43. +5 −2 src/jvm/backtype/storm/serialization/SerializationFactory.java
  44. +3 −2 src/jvm/backtype/storm/spout/ShellSpout.java
  45. +5 −0 src/jvm/backtype/storm/task/IErrorReporter.java
  46. +9 −1 src/jvm/backtype/storm/task/IMetricsContext.java
  47. +1 −2 src/jvm/backtype/storm/task/IOutputCollector.java
  48. +3 −2 src/jvm/backtype/storm/task/ShellBolt.java
  49. +57 −1 src/jvm/backtype/storm/task/TopologyContext.java
  50. +3 −2 src/jvm/backtype/storm/testing/TestAggregatesCounter.java
  51. +3 −2 src/jvm/backtype/storm/testing/TestGlobalCount.java
  52. +3 −2 src/jvm/backtype/storm/testing/TestWordCounter.java
  53. +3 −2 src/jvm/backtype/storm/testing/TestWordSpout.java
  54. +6 −3 src/jvm/backtype/storm/topology/BasicBoltExecutor.java
  55. +3 −2 src/jvm/backtype/storm/transactional/TransactionalSpoutBatchExecutor.java
  56. +3 −2 src/jvm/backtype/storm/transactional/TransactionalSpoutCoordinator.java
  57. +3 −2 src/jvm/backtype/storm/utils/Time.java
  58. +15 −0 src/jvm/storm/trident/operation/TridentOperationContext.java
  59. +1 −1 src/jvm/storm/trident/spout/BatchSpoutExecutor.java
  60. +15 −6 src/jvm/storm/trident/spout/IOpaquePartitionedTridentSpout.java
  61. +20 −10 src/jvm/storm/trident/spout/IPartitionedTridentSpout.java
  62. +8 −0 src/jvm/storm/trident/spout/ISpoutPartition.java
  63. +3 −1 src/jvm/storm/trident/spout/ITridentSpout.java
  64. +76 −35 src/jvm/storm/trident/spout/OpaquePartitionedTridentSpoutExecutor.java
  65. +43 −15 src/jvm/storm/trident/spout/PartitionedTridentSpoutExecutor.java
  66. +1 −1 src/jvm/storm/trident/spout/RichSpoutBatchExecutor.java
  67. +8 −14 src/jvm/storm/trident/spout/TridentSpoutCoordinator.java
  68. +3 −2 src/jvm/storm/trident/spout/TridentSpoutExecutor.java
  69. +2 −1 src/jvm/storm/trident/testing/FeederBatchSpout.java
  70. +4 −2 src/jvm/storm/trident/topology/MasterBatchCoordinator.java
  71. +10 −1 src/jvm/storm/trident/topology/state/RotatingTransactionalState.java
  72. +185 −0 test/clj/backtype/storm/metrics_test.clj
View
@@ -21,6 +21,9 @@ _release
*.zip
.lein-deps-sum
*.iml
+/target
+/.project/
+/.lein-plugins/
*.ipr
*.iws
.idea
View
@@ -1,3 +1,11 @@
+## Unreleased (0.9.0)
+
+ * All logging now done with slf4j
+ * Replaced log4j logging system with logback
+ * Logs are now limited to 1GB per worker (configurable via logging configuration file)
+ * Build upgraded to leiningen 2.0
+ * Revamped Trident spout interfaces to support more dynamic spouts, such as a spout who reads from a changing set of brokers
+
## 0.8.2
* Added backtype.storm.scheduler.IsolationScheduler. This lets you run topologies that are completely isolated at the machine level. Configure Nimbus to isolate certain topologies, and how many machines to give to each of those topologies, with the isolation.scheduler.machines config in Nimbus's storm.yaml. Topologies run on the cluster that are not listed there will share whatever remaining machines there are on the cluster after machines are allocated to the listed topologies.
View
@@ -1,29 +1,35 @@
#!/bin/bash
+function quit {
+ exit 1
+}
+trap quit 1 2 3 15 #Ctrl+C exits.
-RELEASE=`cat project.clj | sed '6q;d' | awk '{print $3}' | sed -e 's/\"//' | sed -e 's/\"//'`
+RELEASE=`head -1 project.clj | awk '{print $3}' | sed -e 's/\"//' | sed -e 's/\"//'`
+LEIN=`which lein2 || which lein`
+export LEIN_ROOT=1
echo Making release $RELEASE
DIR=_release/storm-$RELEASE
rm -rf _release
-export LEIN_ROOT=1
-rm *.zip
-rm *jar
-rm -rf lib
-rm -rf classes
-lein deps
-lein jar
-mkdir -p $DIR
-mkdir $DIR/lib
-cp storm*jar $DIR/
-cp lib/*.jar $DIR/lib
+rm -f *.zip
+$LEIN with-profile release clean
+$LEIN with-profile release deps
+$LEIN with-profile release jar
+$LEIN with-profile release pom
+mvn dependency:copy-dependencies
+
+mkdir -p $DIR/lib
+cp target/storm-*.jar $DIR/storm-${RELEASE}.jar
+cp target/dependency/*.jar $DIR/lib
cp CHANGELOG.md $DIR/
echo $RELEASE > $DIR/RELEASE
-cp -R log4j $DIR/
-mkdir $DIR/logs
+mkdir -p $DIR/logback
+mkdir -p $DIR/logs
+cp -R logback/cluster.xml $DIR/logback/cluster.xml
mkdir $DIR/conf
cp conf/storm.yaml.example $DIR/conf/storm.yaml
View
@@ -243,10 +243,10 @@ def nimbus(klass="backtype.storm.daemon.nimbus"):
See Setting up a Storm cluster for more information.
(https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
"""
- cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"]
+ cppaths = [STORM_DIR + "/conf"]
jvmopts = parse_args(confvalue("nimbus.childopts", cppaths)) + [
"-Dlogfile.name=nimbus.log",
- "-Dlog4j.configuration=storm.log.properties",
+ "-Dlogback.configurationFile=" + STORM_DIR + "/logback/cluster.xml",
]
exec_storm_class(
klass,
@@ -263,10 +263,10 @@ def supervisor(klass="backtype.storm.daemon.supervisor"):
See Setting up a Storm cluster for more information.
(https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
"""
- cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"]
+ cppaths = [STORM_DIR + "/conf"]
jvmopts = parse_args(confvalue("supervisor.childopts", cppaths)) + [
"-Dlogfile.name=supervisor.log",
- "-Dlog4j.configuration=storm.log.properties",
+ "-Dlogback.configurationFile=" + STORM_DIR + "/logback/cluster.xml",
]
exec_storm_class(
klass,
@@ -284,16 +284,16 @@ def ui():
See Setting up a Storm cluster for more information.
(https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
"""
- cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"]
+ cppaths = [STORM_DIR + "/conf"]
jvmopts = parse_args(confvalue("ui.childopts", cppaths)) + [
"-Dlogfile.name=ui.log",
- "-Dlog4j.configuration=storm.log.properties",
+ "-Dlogback.configurationFile=" + STORM_DIR + "/logback/cluster.xml",
]
exec_storm_class(
"backtype.storm.ui.core",
jvmtype="-server",
jvmopts=jvmopts,
- extrajars=[STORM_DIR + "/log4j", STORM_DIR, STORM_DIR + "/conf"])
+ extrajars=[STORM_DIR, STORM_DIR + "/conf"])
def drpc():
"""Syntax: [storm drpc]
@@ -304,12 +304,15 @@ def drpc():
See Distributed RPC for more information.
(https://github.com/nathanmarz/storm/wiki/Distributed-RPC)
"""
- jvmopts = ["-Xmx768m", "-Dlogfile.name=drpc.log", "-Dlog4j.configuration=storm.log.properties"]
+ jvmopts = ["-Xmx768m",
+ "-Dlogfile.name=drpc.log",
+ "-Dlogback.configurationFile=" + STORM_DIR + "/logback/cluster.xml"
+ ]
exec_storm_class(
"backtype.storm.daemon.drpc",
jvmtype="-server",
jvmopts=jvmopts,
- extrajars=[STORM_DIR + "/log4j", STORM_DIR + "/conf"])
+ extrajars=[STORM_DIR + "/conf"])
def dev_zookeeper():
"""Syntax: [storm dev-zookeeper]
@@ -318,11 +321,11 @@ def dev_zookeeper():
"storm.zookeeper.port" as its port. This is only intended for development/testing, the
Zookeeper instance launched is not configured to be used in production.
"""
- cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"]
+ cppaths = [STORM_DIR + "/conf"]
exec_storm_class(
"backtype.storm.command.dev_zookeeper",
jvmtype="-server",
- extrajars=[STORM_DIR + "/log4j", STORM_DIR + "/conf"])
+ extrajars=[STORM_DIR + "/conf"])
def version():
"""Syntax: [storm version]
View
@@ -1,22 +1,27 @@
-#!/bin/bash
+#!/bin/bash
+function quit {
+ exit 1
+}
+trap quit 1 2 3 15 #Ctrl+C exits.
-RELEASE=`cat project.clj | sed '6q;d' | awk '{print $3}' | sed -e 's/\"//' | sed -e 's/\"//'`
+RELEASE=`head -1 project.clj | awk '{print $3}' | sed -e 's/\"//' | sed -e 's/\"//'`
+LEIN=`which lein2 || which lein`
-rm -rf classes
-rm *jar
-rm *xml
-lein jar
-lein pom
-scp storm*jar pom.xml clojars@clojars.org:
+echo ==== Storm Jar ====
+$LEIN with-profile release clean
+$LEIN with-profile release jar
+$LEIN with-profile release pom
+scp target/storm*jar pom.xml clojars@clojars.org:
+rm -Rf target *.xml
-rm *jar
-rm -rf classes
-rm conf/log4j.properties
-lein jar
-mv pom.xml old-pom.xml
-sed 's/artifactId\>storm/artifactId\>storm-lib/g' old-pom.xml > pom.xml
-mv storm-$RELEASE.jar storm-lib-$RELEASE.jar
-scp storm*jar pom.xml clojars@clojars.org:
-rm *xml
-rm *jar
-git checkout conf/log4j.properties
+echo ==== Storm-Lib Jar ====
+rm conf/logback.xml
+$LEIN with-profile lib clean
+$LEIN with-profile lib jar
+$LEIN with-profile lib pom
+sed -i '' -e 's/artifactId\>storm/artifactId\>storm-lib/g' pom.xml
+mv target/storm-$RELEASE.jar target/storm-lib-$RELEASE.jar
+scp target/storm*jar pom.xml clojars@clojars.org:
+rm -Rf target *.xml
+
+git checkout conf/logback.xml
View
@@ -89,6 +89,7 @@ topology.max.task.parallelism: null
topology.max.spout.pending: null
topology.state.synchronization.timeout.secs: 60
topology.stats.sample.rate: 0.05
+topology.builtin.metrics.bucket.size.secs: 60
topology.fall.back.on.java.serialization: true
topology.worker.childopts: null
topology.executor.receive.buffer.size: 1024 #batched
@@ -105,5 +106,4 @@ topology.max.error.report.per.interval: 5
topology.kryo.factory: "backtype.storm.serialization.DefaultKryoFactory"
topology.trident.batch.emit.interval.millis: 500
-
dev.zookeeper.path: "/tmp/dev-storm-zookeeper"
View
@@ -1,10 +0,0 @@
-#This file should be deleted when deployed to server (workaround to leiningen classpath putting dev resources on path)
-#This file is needed for tests
-
-log4j.rootLogger=INFO, A1
-
-log4j.appender.A1=org.apache.log4j.ConsoleAppender
-
-log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
-log4j.category.org.apache.zookeeper=warn
View
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration scan="true" scanPeriod="30 seconds">
+ <appender name="A1" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
+ </encoder>
+ </appender>
+ <logger name="org.apache.zookeeper" level="WARN"/>
+ <root level="INFO">
+ <appender-ref ref="A1"/>
+ </root>
+</configuration>
View
@@ -21,3 +21,10 @@
# drpc.servers:
# - "server1"
# - "server2"
+
+## Metrics Consumers
+# topology.metrics.consumers.register:
+# - class: "org.mycompany.MyMetricsConsumer"
+# argument:
+# - endpoint: "metrics-collector.mycompany.org"
+# parallelism.hint: 1
@@ -1,9 +0,0 @@
-log4j.rootLogger=INFO, A1
-
-
-log4j.appender.A1 = org.apache.log4j.DailyRollingFileAppender
-log4j.appender.A1.File = ${storm.home}/logs/${logfile.name}
-log4j.appender.A1.Append = true
-log4j.appender.A1.DatePattern = '.'yyy-MM-dd
-log4j.appender.A1.layout = org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n
View
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<configuration scan="true" scanPeriod="60 seconds">
+ <appender name="A1" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${storm.home}/logs/${logfile.name}</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${storm.home}/logs/${logfile.name}.%i</fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+
+ <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>100MB</maxFileSize>
+ </triggeringPolicy>
+
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="INFO">
+ <appender-ref ref="A1"/>
+ </root>
+</configuration>
View
@@ -1,43 +1,50 @@
-(def lein-version (System/getenv "LEIN_VERSION"))
-(if-not (re-find #"^1\..*$" lein-version)
- (do (println (str "ERROR: requires Leiningen 1.x but you are using " lein-version))
- (System/exit 1)))
-
-(defproject storm "0.8.2-wip15"
- :source-path "src/clj"
- :test-path "test/clj"
- :java-source-path "src/jvm"
- :javac-options {:debug "true" :fork "true"}
- :resources-path "conf"
- :dev-resources-path "src/dev"
- :repositories {"sonatype" "http://oss.sonatype.org/content/groups/public/"}
+(defproject storm/storm "0.9.0-wip15"
+ :url "http://storm-project.clj"
+ :description "Distributed and fault-tolerant realtime computation"
+ :license {:name "Eclipse Public License - Version 1.0" :url "https://github.com/nathanmarz/storm/blob/master/LICENSE.html"}
:dependencies [[org.clojure/clojure "1.4.0"]
[commons-io "1.4"]
[org.apache.commons/commons-exec "1.1"]
- [storm/libthrift7 "0.7.0"]
+ [storm/libthrift7 "0.7.0"
+ :exclusions [org.slf4j/slf4j-api]]
[clj-time "0.4.1"]
- [log4j/log4j "1.2.16"]
- [com.netflix.curator/curator-framework "1.0.1"]
+ [com.netflix.curator/curator-framework "1.0.1"
+ :exclusions [log4j/log4j]]
[backtype/jzmq "2.1.0"]
[com.googlecode.json-simple/json-simple "1.1"]
[compojure "1.1.3"]
[hiccup "0.3.6"]
[ring/ring-jetty-adapter "0.3.11"]
[org.clojure/tools.logging "0.2.3"]
[org.clojure/math.numeric-tower "0.0.1"]
- [org.slf4j/slf4j-log4j12 "1.5.8"]
[storm/carbonite "1.5.0"]
[org.yaml/snakeyaml "1.9"]
[org.apache.httpcomponents/httpclient "4.1.1"]
[storm/tools.cli "0.2.2"]
[com.googlecode.disruptor/disruptor "2.10.1"]
[storm/jgrapht "0.8.3"]
[com.google.guava/guava "13.0"]
+ [ch.qos.logback/logback-classic "1.0.6"]
+ [org.slf4j/log4j-over-slf4j "1.6.6"]
]
- :dev-dependencies [
- [swank-clojure "1.4.0-SNAPSHOT" :exclusions [org.clojure/clojure]]
- ]
+
+ :source-paths ["src/clj"]
+ :java-source-paths ["src/jvm"]
+ :test-paths ["test/clj"]
+ :resource-paths ["conf"]
+
+ :profiles {:dev {:resource-paths ["src/dev"]}
+ :release {}
+ :lib {}
+ }
+
+ :plugins [[lein-swank "1.4.4"]]
+
+ :repositories {"sonatype"
+ "http://oss.sonatype.org/content/groups/public/"}
+
+ :javac-options {:debug true}
:jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"]
- :extra-classpath-dirs ["src/ui"]
+
:aot :all
-)
+ :min-lein-version "2.0.0")
@@ -28,7 +28,7 @@
[event :as event] [process-simulator :as psim]]))
(require (quote [clojure.set :as set]))
(require (quote [backtype.storm [stats :as stats] [disruptor :as disruptor]]))
- (import (quote [org.apache.log4j PropertyConfigurator Logger]))
+ (import (quote [org.slf4j Logger]))
(import (quote [com.lmax.disruptor InsufficientCapacityException]))
(import (quote [backtype.storm.generated Nimbus Nimbus$Processor
Oops, something went wrong.

0 comments on commit 8935c38

Please sign in to comment.