Permalink
Browse files

fix merge conflicts

  • Loading branch information...
2 parents 2d7affe + 692b573 commit 8ccb15a7641dfc97de0aa90a36a285e5e9efa023 @nathanmarz committed Oct 11, 2012
Showing with 5,165 additions and 1,706 deletions.
  1. +3 −0 .gitignore
  2. +13 −1 CHANGELOG.md
  3. +3 −2 README.markdown
  4. +4 −0 conf/defaults.yaml
  5. +0 −1 project.clj
  6. +7 −0 src/clj/backtype/storm/LocalCluster.clj
  7. +2 −1 src/clj/backtype/storm/bootstrap.clj
  8. +190 −183 src/clj/backtype/storm/daemon/executor.clj
  9. +69 −49 src/clj/backtype/storm/daemon/nimbus.clj
  10. +11 −7 src/clj/backtype/storm/daemon/supervisor.clj
  11. +6 −0 src/clj/backtype/storm/testing.clj
  12. +69 −46 src/clj/backtype/storm/ui/core.clj
  13. +1 −1 src/clj/storm/trident/testing.clj
  14. +111 −23 src/jvm/backtype/storm/Config.java
  15. +2 −0 src/jvm/backtype/storm/ILocalCluster.java
  16. +23 −1 src/jvm/backtype/storm/StormSubmitter.java
  17. +2,982 −1,238 src/jvm/backtype/storm/generated/Nimbus.java
  18. +340 −0 src/jvm/backtype/storm/generated/SubmitOptions.java
  19. +44 −0 src/jvm/backtype/storm/generated/TopologyInitialStatus.java
  20. +11 −0 src/jvm/backtype/storm/nimbus/DefaultTopologyValidator.java
  21. +10 −0 src/jvm/backtype/storm/nimbus/ITopologyValidator.java
  22. +47 −0 src/jvm/backtype/storm/serialization/DefaultKryoFactory.java
  23. +23 −0 src/jvm/backtype/storm/serialization/IKryoFactory.java
  24. +8 −21 src/jvm/backtype/storm/serialization/SerializationFactory.java
  25. +9 −0 src/jvm/backtype/storm/utils/Utils.java
  26. +34 −0 src/jvm/backtype/storm/utils/WindowedTimeThrottler.java
  27. +14 −4 src/jvm/storm/trident/Stream.java
  28. +1 −1 src/jvm/storm/trident/TridentState.java
  29. +43 −11 src/jvm/storm/trident/TridentTopology.java
  30. +4 −0 src/jvm/storm/trident/fluent/GroupedStream.java
  31. +29 −0 src/jvm/storm/trident/operation/builtin/TupleCollectionGet.java
  32. +3 −1 src/jvm/storm/trident/planner/Node.java
  33. +2 −2 src/jvm/storm/trident/planner/PartitionNode.java
  34. +2 −2 src/jvm/storm/trident/planner/ProcessorNode.java
  35. +1 −1 src/jvm/storm/trident/planner/SpoutNode.java
  36. +7 −0 src/jvm/storm/trident/spout/IBatchID.java
  37. +32 −0 src/jvm/storm/trident/spout/RichSpoutBatchId.java
  38. +21 −0 src/jvm/storm/trident/spout/RichSpoutBatchIdSerializer.java
  39. +9 −3 src/jvm/storm/trident/spout/RichSpoutBatchTriggerer.java
  40. +9 −0 src/jvm/storm/trident/state/ITupleCollection.java
  41. +2 −2 src/jvm/storm/trident/state/map/SnapshottableMap.java
  42. +112 −44 src/jvm/storm/trident/testing/LRUMemoryMapState.java
  43. +111 −43 src/jvm/storm/trident/testing/MemoryMapState.java
  44. +4 −2 src/jvm/storm/trident/topology/BatchInfo.java
  45. +54 −4 src/jvm/storm/trident/topology/MasterBatchCoordinator.java
  46. +10 −4 src/jvm/storm/trident/topology/TransactionAttempt.java
  47. +25 −6 src/jvm/storm/trident/topology/TridentBoltExecutor.java
  48. +14 −0 src/py/storm/Nimbus-remote
  49. +393 −0 src/py/storm/Nimbus.py
  50. +76 −0 src/py/storm/ttypes.py
  51. +11 −1 src/storm.thrift
  52. +3 −0 src/ui/public/css/style.css
  53. +81 −0 src/ui/public/js/script.js
  54. +44 −1 test/clj/backtype/storm/integration_test.clj
  55. +26 −0 test/clj/storm/trident/integration_test.clj
View
@@ -24,3 +24,6 @@ _release
/target
/.project/
/.lein-plugins/
+*.ipr
+*.iws
+.idea
View
@@ -1,9 +1,21 @@
-## 0.8.2
+## Unreleased
* Added high water mark to ZeroMQ sockets (defaults to 10000) configurable with zmq.hwm
* Storm UI now uses nimbus.host to find Nimbus rather than always using localhost (thanks Frostman)
* Added report-error! to Clojure DSL
* Automatically throttle errors sent to Zookeeper/Storm UI when too many are reported in a time interval (all errors are still logged) Configured with TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL and TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS
+ * Kryo instance used for serialization can now be controlled via IKryoFactory interface and TOPOLOGY_KRYO_FACTORY config
+ * Add ability to plug in custom code into Nimbus to allow/disallow topologies to be submitted via NIMBUS_TOPOLOGY_VALIDATOR config
+ * Added TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS config to control how often a batch can be emitted in a Trident topology. Defaults to 500 milliseconds. This is used to prevent too much load from being placed on Zookeeper in the case that batches are being processed super quickly.
+ * Log any topology submissions errors in nimbus.log
+ * Add static helpers in Config when using regular maps
+ * Make Trident much more memory efficient during failures by immediately removing state for failed attempts when a more recent attempt is seen
+ * Add ability to name portions of a Trident computation and have those names appear in the Storm UI
+ * Show Nimbus and topology configurations through Storm UI (thanks rnfein)
+ * Added ITupleCollection interface for TridentState's and TupleCollectionGet QueryFunction for getting the full contents of a state. MemoryMapState and LRUMemoryMapState implement this
+ * Can now submit a topology in inactive state. Storm will wait to call open/prepare on the spouts/bolts until it is first activated.
+ * Can now activate, deactive, rebalance, and kill topologies from the Storm UI (thanks Frostman)
+ * Bug fix: Fix race condition in supervisor that would lead to supervisor continuously crashing due to not finding "stormconf.ser" file for an already killed topology
## 0.8.1
View
@@ -30,10 +30,10 @@ You must not remove this notice, or any other, from this software.
## Core contributors
* James Xu ([@xumingming](https://github.com/xumingming))
+* Jason Jackson ([@jason_j](http://twitter.com/jason_j))
## Contributors
-* Jason Jackson ([@jason_j](http://twitter.com/jason_j))
* Christopher Bertels ([@bakkdoor](http://twitter.com/bakkdoor))
* Michael Montano ([@michaelmontano](http://twitter.com/michaelmontano))
* Dennis Zhuang ([@killme2008](https://github.com/killme2008))
@@ -59,7 +59,8 @@ You must not remove this notice, or any other, from this software.
* Sjoerd Mulder ([@sjoerdmulder](https://github.com/sjoerdmulder))
* Yuta Okamoto ([@okapies](https://github.com/okapies))
* Barry Hart ([@barrywhart](https://github.com/barrywhart))
-* Sergey Lukjanov([@Frostman](https://github.com/Frostman))
+* Sergey Lukjanov ([@Frostman](https://github.com/Frostman))
+* Ross Feinstein ([@rnfein](https://github.com/rnfein))
## Acknowledgements
View
@@ -29,6 +29,7 @@ nimbus.inbox.jar.expiration.secs: 3600
nimbus.task.launch.secs: 120
nimbus.reassign: true
nimbus.file.copy.expiration.secs: 600
+nimbus.topology.validator: "backtype.storm.nimbus.DefaultTopologyValidator"
### ui.* configs are for the master
ui.port: 8080
@@ -99,5 +100,8 @@ topology.spout.wait.strategy: "backtype.storm.spout.SleepSpoutWaitStrategy"
topology.sleep.spout.wait.strategy.time.ms: 1
topology.error.throttle.interval.secs: 10
topology.max.error.report.per.interval: 5
+topology.kryo.factory: "backtype.storm.serialization.DefaultKryoFactory"
+topology.trident.batch.emit.interval.millis: 500
+
dev.zookeeper.path: "/tmp/dev-storm-zookeeper"
View
@@ -1,4 +1,3 @@
-
(defproject storm/storm "0.9.0-SNAPSHOT"
:url "http://storm-project.clj"
:description "Distributed and fault-tolerant realtime computation"
@@ -21,6 +21,13 @@
conf
topology))
+(defn -submitTopologyWithOpts [this name conf topology submit-opts]
+ (submit-local-topology-with-opts (:nimbus (. this state))
+ name
+ conf
+ topology
+ submit-opts))
+
(defn -shutdown [this]
(kill-local-storm-cluster (. this state)))
@@ -36,7 +36,8 @@
InvalidTopologyException ClusterSummary TopologyInfo
TopologySummary ExecutorSummary ExecutorStats ExecutorSpecificStats
SpoutStats BoltStats ErrorInfo SupervisorSummary ExecutorInfo
- KillOptions RebalanceOptions JavaObject JavaObjectArg]))
+ KillOptions SubmitOptions RebalanceOptions JavaObject JavaObjectArg
+ TopologyInitialStatus]))
(import (quote [backtype.storm.daemon.common StormBase Assignment
SupervisorInfo WorkerHeartbeat]))
(import (quote [backtype.storm.grouping CustomStreamGrouping]))
Oops, something went wrong.

0 comments on commit 8ccb15a

Please sign in to comment.