Skip to content
Browse files

Updated EDSL

- stream fields not arrays anymore
- added tests using EDSL
  - wordcount regression test
  - enabled CounterApp example
  • Loading branch information...
1 parent b712f9c commit db94ea8acd264a87b20caaa55a7d1f84c22ec162 @matthieumorel matthieumorel committed Jun 29, 2012
View
6 subprojects/s4-core/src/main/java/org/apache/s4/core/ProcessingElement.java
@@ -616,10 +616,10 @@ protected Object clone() {
* Helper method to be used by PE implementation classes. Sends an event to all the target streams.
*
*/
- protected <T extends Event> void emit(T event, Stream<T>[] streamArray) {
+ protected <T extends Event> void emit(T event, Stream<T>... streams) {
- for (int i = 0; i < streamArray.length; i++) {
- streamArray[i].put(event);
+ for (int i = 0; i < streams.length; i++) {
+ streams[i].put(event);
}
}
View
1 subprojects/s4-core/src/main/java/org/apache/s4/core/Stream.java
@@ -126,6 +126,7 @@ void setEventType(Class<T> type) {
* @return the stream object
*/
public Stream<T> setPE(ProcessingElement pe) {
+ this.targetPEs = new ProcessingElement[] { pe };
app.addStream(this);
return this;
}
View
13 subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountModule.java
@@ -1,13 +0,0 @@
-package org.apache.s4.wordcount;
-
-import com.google.inject.AbstractModule;
-
-public class WordCountModule extends AbstractModule {
-
- @Override
- protected void configure() {
- bind(WordCountApp.class);
-
- }
-
-}
View
8 subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountTest.java
@@ -53,14 +53,17 @@ public void prepare() throws IOException, InterruptedException, KeeperException
*/
@Test
public void testSimple() throws Exception {
+ testWordCountApp(WordCountApp.class);
+ }
+
+ protected void testWordCountApp(Class<?> appClass) throws IOException, KeeperException, InterruptedException {
final ZooKeeper zk = CommTestUtils.createZkClient();
TaskSetup taskSetup = new TaskSetup("localhost:" + CommTestUtils.ZK_PORT);
String clusterName = "clusterA";
taskSetup.clean("s4");
taskSetup.setup(clusterName, 1, 10000);
- Main.main(new String[] { "-cluster=" + clusterName, "-appClass=" + WordCountApp.class.getName(),
- "-extraModulesClasses=" + WordCountModule.class.getName() });
+ Main.main(new String[] { "-cluster=" + clusterName, "-appClass=" + appClass.getName() });
CountDownLatch signalTextProcessed = new CountDownLatch(1);
CommTestUtils.watchAndSignalCreation("/textProcessed", signalTextProcessed, zk);
@@ -76,7 +79,6 @@ public void testSimple() throws Exception {
File results = new File(CommTestUtils.DEFAULT_TEST_OUTPUT_DIR + File.separator + "wordcount");
String s = CommTestUtils.readFile(results);
Assert.assertEquals("be=2;da=2;doobie=5;not=1;or=1;to=2;", s);
-
}
@After
View
2 subprojects/s4-edsl/s4-edsl.gradle
@@ -7,7 +7,7 @@ def diezelSrcDir = "${projectDir}/src/main/diezel";
dependencies {
compile project(":s4-core")
- //testCompile project(path: ':s4-core', configuration: 'tests')
+ testCompile project(':s4-core').sourceSets.test.output
}
View
31 subprojects/s4-edsl/src/main/java/org/apache/s4/edsl/AppBuilder.java
@@ -1,6 +1,5 @@
package org.apache.s4.edsl;
-import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.util.Collection;
import java.util.Iterator;
@@ -16,6 +15,7 @@
import org.slf4j.LoggerFactory;
import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
@@ -221,7 +221,7 @@ private void setStreamField(ProcessingElement pe, Collection<StreamBuilder<? ext
for (Field field : fields) {
logger.trace("Field [{}] is of generic type [{}].", field.getName(), field.getGenericType());
- if (field.getType() == Stream[].class) {
+ if (field.getType() == Stream.class) {
logger.debug("Found stream field: {}", field.getGenericType());
/* Track what fields have streams with the same event type. */
@@ -231,12 +231,12 @@ private void setStreamField(ProcessingElement pe, Collection<StreamBuilder<? ext
}
/* Assign streams to stream fields. */
- Multimap<Field, Stream<? extends Event>> assignment = LinkedListMultimap.create();
+ Map<Field, Stream<? extends Event>> assignment = Maps.newHashMap();
for (StreamBuilder<? extends Event> sm : streams) {
Stream<? extends Event> stream = sm.stream;
Class<? extends Event> eventType = sm.type;
- String key = Stream.class.getCanonicalName() + "<" + eventType.getCanonicalName() + ">[]";
+ String key = Stream.class.getCanonicalName() + "<" + eventType.getCanonicalName() + ">";
if (typeMap.containsKey(key)) {
String fieldName;
Field field;
@@ -297,22 +297,17 @@ private void setStreamField(ProcessingElement pe, Collection<StreamBuilder<? ext
}
/* Now we construct the array and do the final assignment. */
- Map<Field, Collection<Stream<? extends Event>>> assignmentMap = assignment.asMap();
- for (Map.Entry<Field, Collection<Stream<? extends Event>>> entry : assignmentMap.entrySet()) {
+ // Map<Field, Stream<? extends Event>> assignmentMap = assignment.asMap();
+ for (Map.Entry<Field, Stream<? extends Event>> entry : assignment.entrySet()) {
Field f = entry.getKey();
- int arraySize = entry.getValue().size();
- @SuppressWarnings("unchecked")
- Stream<? extends Event> streamArray[] = (Stream<? extends Event>[]) Array.newInstance(Stream.class,
- arraySize);
- int i = 0;
- for (Stream<? extends Event> s : entry.getValue()) {
- streamArray[i++] = s;
-
- f.setAccessible(true);
- f.set(pe, streamArray);
- logger.debug("Assigned [{}] streams to field [{}].", streamArray.length, f.getName());
- }
+ // int arraySize = entry.getValue().size();
+ // Stream<? extends Event> streamArray[] = (Stream<? extends Event>[]) Array.newInstance(Stream.class,
+ // arraySize);
+ f.setAccessible(true);
+ f.set(pe, entry.getValue());
+
+ logger.debug("Assigned stream [{}] to field [{}].", entry.getValue().getName(), f.getName());
}
}
View
36 subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/wordcount/WordCountApp.java
@@ -0,0 +1,36 @@
+package org.apache.s4.edsl.wordcount;
+
+import java.io.IOException;
+
+import org.apache.s4.core.Stream;
+import org.apache.s4.edsl.BuilderS4DSL;
+import org.apache.s4.fixtures.SocketAdapter;
+import org.apache.s4.wordcount.SentenceKeyFinder;
+import org.apache.s4.wordcount.StringEvent;
+import org.apache.s4.wordcount.WordClassifierPE;
+import org.apache.s4.wordcount.WordCountEvent;
+import org.apache.s4.wordcount.WordCountKeyFinder;
+import org.apache.s4.wordcount.WordCounterPE;
+import org.apache.s4.wordcount.WordSeenEvent;
+import org.apache.s4.wordcount.WordSeenKeyFinder;
+import org.apache.s4.wordcount.WordSplitterPE;
+
+public class WordCountApp extends BuilderS4DSL {
+
+ SocketAdapter<StringEvent> socketAdapter;
+
+ protected void onInit() {
+ pe("Classifier").type(WordClassifierPE.class).pe("Counter").type(WordCounterPE.class)
+ .emit(WordCountEvent.class).withKeyFinder(WordCountKeyFinder.class).to("Classifier").pe("Splitter")
+ .type(WordSplitterPE.class).emit(WordSeenEvent.class).withKeyFinder(WordSeenKeyFinder.class)
+ .to("Counter").build();
+ Stream<StringEvent> sentenceStream = createStream("sentences stream", new SentenceKeyFinder(),
+ getPE("Splitter"));
+ try {
+ socketAdapter = new SocketAdapter<StringEvent>(sentenceStream, new SocketAdapter.SentenceEventFactory());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
View
13 subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/wordcount/WordCountTest.java
@@ -0,0 +1,13 @@
+package org.apache.s4.edsl.wordcount;
+
+import org.junit.Test;
+
+public class WordCountTest extends org.apache.s4.wordcount.WordCountTest {
+
+ @Test
+ public void testSimple() throws Exception {
+ testWordCountApp(WordCountApp.class);
+
+ }
+
+}
View
4 subprojects/s4-example/s4-example.gradle
@@ -15,12 +15,14 @@
*/
description = 'Examples of S4 applications.'
-
+
dependencies {
compile project( ":s4-base" )
compile project( ":s4-core" )
compile project( ":s4-comm" )
compile project( ":s4-edsl" )
compile libraries.ejml
compile libraries.junit
+ // TODO remove this dependency only used for initializing a zookeeper server
+ compile project(':s4-core').sourceSets.test.output
}
View
213 subprojects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/CounterApp.java
@@ -1,100 +1,117 @@
// NOTE: this is commented until we fix the dependency to the classes generated by the edsl subproject
-//package org.apache.s4.example.edsl.counter;
-//
-//import java.util.concurrent.TimeUnit;
-//
-//import org.apache.s4.base.Event;
-//
-//import com.google.inject.Guice;
-//import com.google.inject.Injector;
-//
-///**
-// * This is a sample application to test the S4 embedded domain-specific language (EDSL).
-// *
-// * <p>
-// * Grammar:
-// *
-// * <pre>
-// * (pe , type , prop* , (fireOn , afterInterval? , afterNumEvents?)? , (timer, withPeriod)? ,
-// * (cache, size , expires? )? , asSingleton? , (emit, onField?,
-// * (withKey|withKeyFinder)?, to )* )+ , build
-// * </pre>
-// *
-// * <p>
-// * See the <a href="http://code.google.com/p/diezel">Diezel</a> project for details.
-// *
-// */
-//final public class CounterApp extends BuilderS4DSL {
-//
-// public static void main(String[] args) {
-// Injector injector = Guice.createInjector(new Module());
-// CounterApp myApp = injector.getInstance(CounterApp.class);
-//
-// /* Normally. the container will handle this but this is just a test. */
-// myApp.init();
-// myApp.start();
-//
-// try {
-// Thread.sleep(1000);
-// } catch (InterruptedException e) {
-// e.printStackTrace();
-// }
-// myApp.close();
-// }
-//
-// @Override
-// protected void onInit() {
-//
-// pe("Print").type(PrintPE.class).asSingleton().
-//
-// pe("User Count").type(CounterPE.class).fireOn(Event.class).afterInterval(100, TimeUnit.MILLISECONDS)
-// .emit(CountEvent.class).withKeyFinder(CountKeyFinder.class).to("Print").
-//
-// pe("Gender Count").type(CounterPE.class).fireOn(Event.class).afterInterval(100, TimeUnit.MILLISECONDS)
-// .emit(CountEvent.class).withKeyFinder(CountKeyFinder.class).to("Print").
-//
-// pe("Age Count").type(CounterPE.class).fireOn(Event.class).afterInterval(100, TimeUnit.MILLISECONDS)
-// .emit(CountEvent.class).withKeyFinder(CountKeyFinder.class).to("Print").
-//
-// pe("Generate User Event").type(GenerateUserEventPE.class).timer().withPeriod(1, TimeUnit.MILLISECONDS)
-// .asSingleton().
-//
-// emit(UserEvent.class).withKeyFinder(UserIDKeyFinder.class).to("User Count").
-//
-// emit(UserEvent.class).withKey("gender").to("Gender Count").
-//
-// emit(UserEvent.class).withKeyFinder(AgeKeyFinder.class).to("Age Count").
-//
-// build();
-// }
-//
-// /*
-// * Create and send 200 dummy events of type UserEvent.
-// *
-// * @see io.s4.App#start()
-// */
-// @Override
-// protected void onStart() {
-//
-// }
-//
-// @Override
-// protected void onClose() {
-// System.out.println("Bye.");
-// }
-//
-// // Make hooks public for testing. Normally this is handled by the container.
-// public void init() {
-// super.init();
-// }
-//
-// public void start() {
-// super.start();
-// }
-//
-// public void close() {
-// super.close();
-// }
-//
-// }
+package org.apache.s4.example.edsl.counter;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.s4.base.Event;
+import org.apache.s4.comm.DefaultCommModule;
+import org.apache.s4.comm.tools.TaskSetup;
+import org.apache.s4.core.DefaultCoreModule;
+import org.apache.s4.edsl.BuilderS4DSL;
+import org.apache.s4.fixtures.CommTestUtils;
+import org.apache.s4.fixtures.CoreTestUtils;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.common.io.Resources;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+/**
+ * This is a sample application to test the S4 embedded domain-specific language (EDSL).
+ *
+ * <p>
+ * Grammar:
+ *
+ * <pre>
+ * (pe , type , prop* , (fireOn , afterInterval? , afterNumEvents?)? , (timer, withPeriod)? ,
+ * (cache, size , expires? )? , asSingleton? , (emit, onField?,
+ * (withKey|withKeyFinder)?, to )* )+ , build
+ * </pre>
+ *
+ * <p>
+ * See the <a href="http://code.google.com/p/diezel">Diezel</a> project for details.
+ *
+ */
+final public class CounterApp extends BuilderS4DSL {
+
+ public static void main(String[] args) {
+ Injector injector;
+ try {
+ injector = Guice.createInjector(new DefaultCommModule(Resources.getResource("default.s4.comm.properties")
+ .openStream(), "cluster1"),
+ new DefaultCoreModule(Resources.getResource("default.s4.core.properties").openStream()));
+
+ CoreTestUtils.startZookeeperServer();
+ TaskSetup taskSetup = new TaskSetup("localhost:" + CommTestUtils.ZK_PORT);
+ String clusterName = "cluster1";
+ taskSetup.clean("s4");
+ taskSetup.setup(clusterName, 1, 10000);
+
+ CounterApp myApp = injector.getInstance(CounterApp.class);
+
+ /* Normally. the container will handle this but this is just a test. */
+ myApp.init();
+ myApp.start();
+
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ myApp.close();
+ } catch (IOException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (KeeperException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ protected void onInit() {
+
+ pe("Print").type(PrintPE.class).asSingleton().
+
+ pe("User Count").type(CounterPE.class).fireOn(Event.class).afterInterval(100, TimeUnit.MILLISECONDS)
+ .emit(CountEvent.class).withKeyFinder(CountKeyFinder.class).to("Print").
+
+ pe("Gender Count").type(CounterPE.class).fireOn(Event.class).afterInterval(100, TimeUnit.MILLISECONDS)
+ .emit(CountEvent.class).withKeyFinder(CountKeyFinder.class).to("Print").
+
+ pe("Age Count").type(CounterPE.class).fireOn(Event.class).afterInterval(100, TimeUnit.MILLISECONDS)
+ .emit(CountEvent.class).withKeyFinder(CountKeyFinder.class).to("Print").
+
+ pe("Generate User Event").type(GenerateUserEventPE.class).timer().withPeriod(1, TimeUnit.MILLISECONDS)
+ .asSingleton().
+
+ emit(UserEvent.class).withKeyFinder(UserIDKeyFinder.class).to("User Count").
+
+ emit(UserEvent.class).withKey("gender").to("Gender Count").
+
+ emit(UserEvent.class).withKeyFinder(AgeKeyFinder.class).to("Age Count").
+
+ build();
+ }
+
+ /*
+ * Create and send 200 dummy events of type UserEvent.
+ *
+ * @see io.s4.App#start()
+ */
+ @Override
+ protected void onStart() {
+
+ }
+
+ @Override
+ protected void onClose() {
+ System.out.println("Bye.");
+ }
+
+}
View
23 subprojects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/CounterPE.java
@@ -1,17 +1,17 @@
/*
* Copyright (c) 2011 Yahoo! Inc. All rights reserved.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific
* language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
+ * License. See accompanying LICENSE file.
*/
package org.apache.s4.example.edsl.counter;
@@ -26,27 +26,12 @@
private static final Logger logger = LoggerFactory.getLogger(CounterPE.class);
- private Stream<CountEvent>[] countStream = null;
+ private Stream<CountEvent> countStream = null;
public CounterPE(App app) {
super(app);
}
- /**
- * @return the countStream
- */
- public Stream<CountEvent>[] getCountStream() {
- return countStream;
- }
-
- /**
- * @param countStream
- * the countStream to set
- */
- public void setCountStream(Stream<CountEvent>[] countStream) {
- this.countStream = countStream;
- }
-
private long counter = 0;
public void onEvent(Event event) {
View
16 ...ects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/GenerateUserEventPE.java
@@ -1,17 +1,17 @@
/*
* Copyright (c) 2011 Yahoo! Inc. All rights reserved.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific
* language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
+ * License. See accompanying LICENSE file.
*/
package org.apache.s4.example.edsl.counter;
@@ -32,21 +32,13 @@
static String userIds[] = { "pepe", "jose", "tito", "mr_smith", "joe" };
static int[] ages = { 25, 2, 33, 6, 67 };
static char[] genders = { 'f', 'm' };
- private Stream<UserEvent>[] targetStreams;
+ private Stream<UserEvent> targetStreams;
final private Random generator = new Random(22);
public GenerateUserEventPE(App app) {
super(app);
}
- /**
- * @param targetStreams
- * the {@link UserEvent} streams.
- */
- public void setStreams(Stream<UserEvent>... targetStreams) {
- this.targetStreams = targetStreams;
- }
-
protected void onTime() {
List<String> favorites = new ArrayList<String>();
favorites.add("dulce de leche");

0 comments on commit db94ea8

Please sign in to comment.
Something went wrong with that request. Please try again.