Permalink
Browse files

Merge branch 'S4-95' into dev

  • Loading branch information...
matthieumorel committed Jan 18, 2013
2 parents 57a3999 + 8c4c7a5 commit 2a95d3549573f02c2fb8462121d7b54ac0bd4dfd
Showing with 3,268 additions and 1,378 deletions.
  1. +14 −9 build.gradle
  2. BIN lib/reflectasm-1.07-shaded.jar
  3. +4 −3 settings.gradle
  4. +6 −2 subprojects/s4-base/src/main/java/org/apache/s4/base/Emitter.java
  5. +9 −5 subprojects/s4-base/src/main/java/org/apache/s4/base/Event.java
  6. +0 −72 subprojects/s4-base/src/main/java/org/apache/s4/base/EventMessage.java
  7. +4 −9 subprojects/s4-base/src/main/java/org/apache/s4/base/Listener.java
  8. +24 −0 subprojects/s4-base/src/main/java/org/apache/s4/base/Receiver.java
  9. +32 −0 subprojects/s4-base/src/main/java/org/apache/s4/base/Sender.java
  10. +5 −3 subprojects/s4-base/src/main/java/org/apache/s4/base/SerializerDeserializer.java
  11. +3 −0 subprojects/s4-base/src/main/java/org/apache/s4/base/package-info.java
  12. +100 −0 subprojects/s4-benchmarks/README.md
  13. +92 −0 subprojects/s4-benchmarks/bench-cluster.sh
  14. +7 −0 subprojects/s4-benchmarks/config/injector.config
  15. +3 −0 subprojects/s4-benchmarks/config/node.config
  16. +82 −0 subprojects/s4-benchmarks/s4-benchmarks.gradle
  17. +109 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/dag/DagApp.java
  18. +42 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/dag/FirstPE.java
  19. +30 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/dag/LastPE.java
  20. +42 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/dag/PipePE.java
  21. +79 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/prodcon/ProducerConsumerApp.java
  22. +36 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/prodcon/SimplePE1.java
  23. +30 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/prodcon/SimplePE2.java
  24. +15 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/utils/InjectionLimiterModule.java
  25. +207 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/utils/Injector.java
  26. +70 −0 subprojects/s4-benchmarks/src/main/java/org/apache/s4/benchmark/utils/Utils.java
  27. +14 −0 subprojects/s4-benchmarks/src/main/resources/logback.xml
  28. +19 −0 subprojects/s4-benchmarks/startInjector.sh
  29. +16 −0 subprojects/s4-benchmarks/startNode.sh
  30. +10 −6 subprojects/s4-comm/src/main/java/org/apache/s4/comm/DefaultCommModule.java
  31. +29 −0 subprojects/s4-comm/src/main/java/org/apache/s4/comm/DeserializerExecutorFactory.java
  32. +38 −33 subprojects/s4-comm/src/main/java/org/apache/s4/comm/serialize/KryoSerDeser.java
  33. +9 −0 subprojects/s4-comm/src/main/java/org/apache/s4/comm/serialize/SerializerDeserializerFactory.java
  34. +26 −0 ...rojects/s4-comm/src/main/java/org/apache/s4/comm/staging/BlockingDeserializerExecutorFactory.java
  35. +214 −0 subprojects/s4-comm/src/main/java/org/apache/s4/comm/staging/BlockingThreadPoolExecutorService.java
  36. +44 −0 ...ects/s4-comm/src/main/java/org/apache/s4/comm/staging/MemoryAwareDeserializerExecutorFactory.java
  37. +142 −0 ...rojects/s4-comm/src/main/java/org/apache/s4/comm/staging/ThrottlingThreadPoolExecutorService.java
  38. +11 −6 subprojects/s4-comm/src/main/java/org/apache/s4/comm/tcp/RemoteEmitters.java
  39. +59 −12 subprojects/s4-comm/src/main/java/org/apache/s4/comm/tcp/TCPEmitter.java
  40. +31 −30 subprojects/s4-comm/src/main/java/org/apache/s4/comm/tcp/TCPListener.java
  41. +6 −4 subprojects/s4-comm/src/main/java/org/apache/s4/comm/tcp/TCPRemoteEmitter.java
  42. +8 −6 subprojects/s4-comm/src/main/java/org/apache/s4/comm/udp/UDPEmitter.java
  43. +19 −12 subprojects/s4-comm/src/main/java/org/apache/s4/comm/udp/UDPListener.java
  44. +3 −2 subprojects/s4-comm/src/main/java/org/apache/s4/comm/udp/UDPRemoteEmitter.java
  45. +26 −0 subprojects/s4-comm/src/main/java/org/apache/s4/comm/util/EmitterMetrics.java
  46. +26 −1 subprojects/s4-comm/src/main/resources/default.s4.comm.properties
  47. +0 −159 subprojects/s4-comm/src/test/java/org/apache/s4/comm/DeliveryTestUtil.java
  48. +0 −34 subprojects/s4-comm/src/test/java/org/apache/s4/comm/tcp/MultiPartitionDeliveryTest.java
  49. +0 −34 subprojects/s4-comm/src/test/java/org/apache/s4/comm/tcp/SimpleDeliveryTest.java
  50. +45 −0 subprojects/s4-comm/src/test/java/org/apache/s4/comm/tcp/TCPBasicTest.java
  51. +0 −68 subprojects/s4-comm/src/test/java/org/apache/s4/comm/tcp/TCPCommTest.java
  52. +2 −1 subprojects/s4-comm/src/test/java/org/apache/s4/comm/topology/AssignmentsFromZKTest1.java
  53. +2 −1 subprojects/s4-comm/src/test/java/org/apache/s4/comm/topology/ClustersFromZKTest.java
  54. +0 −44 subprojects/s4-comm/src/test/java/org/apache/s4/comm/topology/ZKBaseTest.java
  55. +0 −27 subprojects/s4-comm/src/test/java/org/apache/s4/comm/udp/MultiPartitionDeliveryTest.java
  56. +0 −27 subprojects/s4-comm/src/test/java/org/apache/s4/comm/udp/SimpleDeliveryTest.java
  57. +42 −0 subprojects/s4-comm/src/test/java/org/apache/s4/comm/udp/UDPBasicTest.java
  58. +0 −79 subprojects/s4-comm/src/test/java/org/apache/s4/comm/udp/UDPCommTest.java
  59. +0 −196 subprojects/s4-comm/src/test/java/org/apache/s4/comm/util/PartitionInfo.java
  60. +0 −116 subprojects/s4-comm/src/test/java/org/apache/s4/comm/util/ProtocolTestUtil.java
  61. +3 −0 subprojects/s4-comm/src/test/java/org/apache/s4/fixtures/CommTestUtils.java
  62. +38 −0 subprojects/s4-comm/src/test/java/org/apache/s4/fixtures/MockReceiver.java
  63. +27 −0 subprojects/s4-comm/src/test/java/org/apache/s4/fixtures/MockReceiverModule.java
  64. +22 −0 subprojects/s4-comm/src/test/java/org/apache/s4/fixtures/NoOpReceiver.java
  65. +27 −0 subprojects/s4-comm/src/test/java/org/apache/s4/fixtures/NoOpReceiverModule.java
  66. +18 −0 subprojects/s4-comm/src/test/java/org/apache/s4/fixtures/TCPTransportModule.java
  67. +18 −0 subprojects/s4-comm/src/test/java/org/apache/s4/fixtures/UDPTransportModule.java
  68. +17 −0 subprojects/s4-comm/src/test/java/org/apache/s4/fixtures/ZkBasedTest.java
  69. +7 −0 subprojects/s4-comm/src/test/resources/udp.s4.comm.properties
  70. +2 −1 subprojects/s4-core/s4-core.gradle
  71. +27 −6 subprojects/s4-core/src/main/java/org/apache/s4/core/App.java
  72. +38 −0 subprojects/s4-core/src/main/java/org/apache/s4/core/AppModule.java
  73. +13 −5 subprojects/s4-core/src/main/java/org/apache/s4/core/DefaultCoreModule.java
  74. +18 −0 subprojects/s4-core/src/main/java/org/apache/s4/core/Main.java
  75. +11 −3 subprojects/s4-core/src/main/java/org/apache/s4/core/ProcessingElement.java
  76. +34 −39 subprojects/s4-core/src/main/java/org/apache/s4/core/{Receiver.java → ReceiverImpl.java}
  77. +14 −8 subprojects/s4-core/src/main/java/org/apache/s4/core/RemoteSender.java
  78. +60 −22 subprojects/s4-core/src/main/java/org/apache/s4/core/RemoteSenders.java
  79. +1 −6 subprojects/s4-core/src/main/java/org/apache/s4/core/RemoteStream.java
  80. +0 −124 subprojects/s4-core/src/main/java/org/apache/s4/core/Sender.java
  81. +178 −0 subprojects/s4-core/src/main/java/org/apache/s4/core/SenderImpl.java
  82. +6 −3 subprojects/s4-core/src/main/java/org/apache/s4/core/Server.java
  83. +111 −84 subprojects/s4-core/src/main/java/org/apache/s4/core/Stream.java
  84. +2 −1 subprojects/s4-core/src/main/java/org/apache/s4/core/ft/FileSystemBackendCheckpointingModule.java
  85. +5 −14 subprojects/s4-core/src/main/java/org/apache/s4/core/ft/SafeKeeper.java
  86. +1 −1 subprojects/s4-core/src/main/java/org/apache/s4/core/gen/OverloadDispatcherGenerator.java
  87. +22 −0 ...s4-core/src/main/java/org/apache/s4/core/staging/BlockingRemoteSendersExecutorServiceFactory.java
  88. +35 −0 ...ojects/s4-core/src/main/java/org/apache/s4/core/staging/BlockingSenderExecutorServiceFactory.java
  89. +28 −0 ...ojects/s4-core/src/main/java/org/apache/s4/core/staging/BlockingStreamExecutorServiceFactory.java
  90. +20 −0 ...ore/src/main/java/org/apache/s4/core/staging/LoadSheddingRemoteSendersExecutorServiceFactory.java
  91. +62 −0 ...ts/s4-core/src/main/java/org/apache/s4/core/staging/LoadSheddingSenderExecutorServiceFactory.java
  92. +63 −0 ...ts/s4-core/src/main/java/org/apache/s4/core/staging/LoadSheddingStreamExecutorServiceFactory.java
  93. +12 −0 ...rojects/s4-core/src/main/java/org/apache/s4/core/staging/RemoteSendersExecutorServiceFactory.java
  94. +13 −0 subprojects/s4-core/src/main/java/org/apache/s4/core/staging/SenderExecutorServiceFactory.java
  95. +31 −0 subprojects/s4-core/src/main/java/org/apache/s4/core/staging/StreamExecutorServiceFactory.java
  96. +21 −0 ...-core/src/main/java/org/apache/s4/core/staging/ThrottlingRemoteSendersExecutorServiceFactory.java
  97. +43 −0 ...ects/s4-core/src/main/java/org/apache/s4/core/staging/ThrottlingSenderExecutorServiceFactory.java
  98. +197 −0 subprojects/s4-core/src/main/java/org/apache/s4/core/util/S4Metrics.java
  99. +5 −3 subprojects/s4-core/src/test/java/org/apache/s4/core/TriggerTest.java
  100. +8 −5 subprojects/s4-core/src/test/java/org/apache/s4/core/ft/CheckpointingTest.java
  101. +7 −5 subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountTest.java
  102. +17 −9 subprojects/s4-core/src/test/java/org/apache/s4/core/ft/RecoveryTest.java
  103. +6 −3 subprojects/s4-core/src/test/java/org/apache/s4/core/timers/MultithreadingTest.java
  104. +0 −1 subprojects/s4-core/src/test/java/org/apache/s4/core/windowing/WindowingPE1.java
  105. +6 −4 subprojects/s4-core/src/test/java/org/apache/s4/core/windowing/WindowingPETest.java
  106. +6 −4 subprojects/s4-core/src/test/java/org/apache/s4/deploy/TestAutomaticDeployment.java
  107. +8 −4 subprojects/s4-core/src/test/java/org/apache/s4/fixtures/CoreTestUtils.java
  108. +16 −4 subprojects/s4-core/src/test/java/org/apache/s4/fixtures/MockCommModule.java
  109. +22 −7 subprojects/s4-core/src/test/java/org/apache/s4/fixtures/MockCoreModule.java
  110. +9 −6 subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountTest.java
  111. +6 −5 subprojects/s4-example/src/main/java/org/apache/s4/example/counter/MyApp.java
  112. +21 −0 test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TwitterCounterApp.java
View
@@ -53,15 +53,16 @@ allprojects {
/* All project libraries must be defined here. */
project.ext["libraries"] = [
- guava: 'com.google.guava:guava:12.0.1',
+ guava: 'com.google.guava:guava:13.0.1',
gson: 'com.google.code.gson:gson:1.6',
guice: 'com.google.inject:guice:3.0',
aop_alliance: 'aopalliance:aopalliance:1.0',
guice_assist: 'com.google.inject.extensions:guice-assistedinject:3.0',
- kryo: 'com.googlecode:kryo:1.04',
+ kryo: 'com.esotericsoftware.kryo:kryo:2.20',
minlog: 'com.googlecode:minlog:1.2',
- reflectasm: 'com.googlecode:reflectasm:1.01',
- netty: 'org.jboss.netty:netty:3.2.5.Final',
+ // NOTE shaded jar is not resolved correctly, we include it in /lib directory
+ reflectasm: 'com.esotericsoftware.reflectasm:reflectasm:1.07-shaded',
+ netty: 'io.netty:netty:3.5.11.Final',
mockito_core: 'org.mockito:mockito-core:1.9.0',
commons_config: 'commons-configuration:commons-configuration:1.6',
commons_codec: 'commons-codec:commons-codec:1.4',
@@ -75,18 +76,19 @@ project.ext["libraries"] = [
log4j: 'log4j:log4j:1.2.15',
logback_core: 'ch.qos.logback:logback-core:0.9.29',
logback_classic: 'ch.qos.logback:logback-classic:0.9.29',
- zk: 'org.apache.zookeeper:zookeeper:3.3.1',
+ zk: 'org.apache.zookeeper:zookeeper:3.3.3',
jcip: 'net.jcip:jcip-annotations:1.0',
junit: 'junit:junit:4.10',
zkclient: 'com.github.sgroschupf:zkclient:0.1',
diezel: 'net.ericaro:diezel-maven-plugin:1.0.0-beta-4',
jcommander: 'com.beust:jcommander:1.25',
- asm: 'asm:asm:3.2',
+ asm: 'org.ow2.asm:asm:4.0',
javax_inject: 'javax.inject:javax.inject:1',
gradle_base_services: 'org.gradle:gradle-base-services:1.0',
gradle_core: 'org.gradle:gradle-core:1.0',
gradle_tooling_api: 'org.gradle:gradle-tooling-api:1.0',
- gradle_wrapper: 'gradle-wrapper:gradle-wrapper:1.0'
+ gradle_wrapper: 'gradle-wrapper:gradle-wrapper:1.0',
+ metrics: 'com.yammer.metrics:metrics-core:2.1.3'
]
subprojects {
@@ -106,6 +108,7 @@ subprojects {
exclude group: 'com.sun.jdmk', module: 'jmxtools'
exclude group: 'com.sun.jmx', module: 'jmxri'
exclude group: 'javax.jms', module: 'jms'
+ exclude group: 'javax.mail', module: 'mail'
}
dependencies {
@@ -131,6 +134,7 @@ subprojects {
/* Misc. */
compile( libraries.jcip )
compile( libraries.zk )
+ compile( libraries.metrics )
/* Testing. */
testCompile( libraries.junit )
@@ -140,11 +144,12 @@ subprojects {
runtime(libraries.aop_alliance)
runtime(libraries.minlog)
runtime(libraries.gson)
+ compile(libraries.reflectasm)
runtime(libraries.objenesis)
runtime(libraries.kryo)
runtime(libraries.netty)
runtime(libraries.asm)
- runtime(libraries.javax_inject)
+ compile(libraries.javax_inject)
runtime(libraries.commons_codec)
}
@@ -168,7 +173,7 @@ subprojects {
evaluationDependsOnChildren()
-project.ext["platformProjects"] = [project(':s4-base'), project(':s4-core'), project(':s4-comm'), project(':s4-tools')]
+project.ext["platformProjects"] = [project(':s4-base'), project(':s4-core'), project(':s4-comm'), project(':s4-tools'), project(':s4-benchmarks')]
configurations {
platformLibs
Binary file not shown.
View
@@ -19,9 +19,10 @@
include 's4-base'
include 's4-core'
include 's4-comm'
-include 's4-edsl'
-include 's4-example'
-include 's4-tools'
+//include 's4-edsl'
+//include 's4-example'
+include 's4-tools'
+include 's4-benchmarks'
//include 's4-example'
//include ':test-apps:simple-adapter-1'
include ':test-apps:simple-deployable-app-1'
@@ -18,9 +18,11 @@
package org.apache.s4.base;
+import java.nio.ByteBuffer;
+
/**
* Defines an event emitter, responsible for sending an event to a given partition of the cluster.
- *
+ *
*/
public interface Emitter {
@@ -32,8 +34,10 @@
* - message payload that needs to be sent
*
* @return - true - if message is sent across successfully - false - if send fails
+ * @throws InterruptedException
+ * if interrupted during blocking send operation
*/
- boolean send(int partitionId, EventMessage message);
+ boolean send(int partitionId, ByteBuffer message) throws InterruptedException;
int getPartitionCount();
@@ -24,6 +24,7 @@
import org.slf4j.LoggerFactory;
import com.google.common.collect.Maps;
+import com.google.common.primitives.Primitives;
/**
* The base event class in S4. The base class supports generic key/value pairs which us useful for rapid prototyping and
@@ -134,12 +135,15 @@ public void setAppId(int appId) {
Data<?> data = map.get(key);
- if (type != data.type) {
- logger.error("Trying to get a value of type {} for an attribute of type {}.", type, data.type);
- return null;
+ try {
+ return (T) data.value;
+ } catch (ClassCastException e) {
+ if (!Primitives.wrap(type).isAssignableFrom(Primitives.wrap(data.type))) {
+ logger.error("Trying to get a value of type {} for an attribute of type {}.", type, data.type);
+ return null;
+ }
+ throw e;
}
-
- return (T) data.value;
}
/**
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.s4.base;
-
-/**
- * <p>
- * Encapsulates application-level events of type {@link Event}.
- * </p>
- * <p>
- * Indeed, events that are defined at the application level can only be handled by the classloader of the corresponding
- * application.
- * </p>
- * <p>
- * Includes routing information (application name, stream name), so that this message can be dispatched at the
- * communication level.
- * </p>
- *
- */
-public class EventMessage {
-
- private String appName;
- private String streamName;
- private byte[] serializedEvent;
-
- public EventMessage() {
- }
-
- /**
- *
- * @param appName
- * name of the application
- * @param streamName
- * name of the stream
- * @param serializedEvent
- * application-specific {@link Event} instance in serialized form
- */
- public EventMessage(String appName, String streamName, byte[] serializedEvent) {
- super();
- this.appName = appName;
- this.streamName = streamName;
- this.serializedEvent = serializedEvent;
- }
-
- public String getAppName() {
- return appName;
- }
-
- public String getStreamName() {
- return streamName;
- }
-
- public byte[] getSerializedEvent() {
- return serializedEvent;
- }
-
-}
@@ -20,21 +20,16 @@
/**
*
- * Get a byte array received by a lower level layer.
+ * Defines the communication level entry point for serialized events. The implementation is expected to open a server
+ * socket on the node's listening port, receive messages through this channel, and delegate to the application layer
+ * through the {@link Receiver} interface.
*
*/
public interface Listener {
/**
- * Perform blocking receive on the appropriate communication channel
- *
- * @return <ul>
- * <li>byte[] message returned by the channel</li>
- * <li>null if the associated blocking thread is interrupted</li>
- * </ul>
+ * Returns the id of the partition currently assigned to this node.
*/
- byte[] recv();
-
public int getPartitionId();
void close();
@@ -0,0 +1,24 @@
+package org.apache.s4.base;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Defines the entry point from the communication layer to the application layer.
+ *
+ * Events received as raw bytes through the {@link Listener} implementation use the {@link Receiver#receive(ByteBuffer)}
+ * method so that events can be deserialized (conversion from byte[] to Event objects) and enqueued for processing.
+ *
+ */
+public interface Receiver {
+
+ /**
+ * Handle a serialized message, i.e. deserialize the message and pass it to event processors.
+ */
+ void receive(ByteBuffer message);
+
+ /**
+ * Returns the partition id currently assigned to this node.
+ */
+ int getPartitionId();
+
+}
@@ -0,0 +1,32 @@
+package org.apache.s4.base;
+
+/**
+ * Defines the exit point from the event processing layer to the communication layer. The implementation is responsible
+ * for serializing events and passing serialized data to the communication layer.
+ *
+ */
+public interface Sender {
+
+ /**
+ * This method attempts to send an event to a remote partition. If the destination is local, the method does not
+ * send the event and returns false. <b>The caller is then expected to put the event in a local queue instead.</b>
+ *
+ * @param hashKey
+ * the string used to map the value of a key to a specific partition.
+ * @param event
+ * the event to be delivered to a Processing Element instance.
+ * @return true if the event was sent because the destination is <b>not</b> local.
+ *
+ */
+ boolean checkAndSendIfNotLocal(String hashKey, Event event);
+
+ /**
+ * Send an event to all the remote partitions in the cluster. The caller is expected to also put the event in a
+ * local queue.
+ *
+ * @param event
+ * the event to be delivered to Processing Element instances.
+ */
+ void sendToAllRemotePartitions(Event event);
+
+}
@@ -18,12 +18,14 @@
package org.apache.s4.base;
+import java.nio.ByteBuffer;
+
/**
* Defines serialization and deserialization methods used within the S4 platform, typically for events and PEs.
- *
+ *
*/
public interface SerializerDeserializer {
- public byte[] serialize(Object message);
+ public ByteBuffer serialize(Object message);
- public Object deserialize(byte[] rawMessage);
+ public Object deserialize(ByteBuffer rawMessage);
}
@@ -18,5 +18,8 @@
/**
* Defines some of the basic elements of the S4 platforms.
+ *
+ *
+ *
*/
package org.apache.s4.base;
Oops, something went wrong.

0 comments on commit 2a95d35

Please sign in to comment.