From d1a482a5317bdc64ba684e220b2c913c8172f3d5 Mon Sep 17 00:00:00 2001 From: Saikat Kanjilal Date: Tue, 18 Sep 2018 16:20:47 -0700 Subject: [PATCH 1/5] first versioon of samoa-heron --- samoa-heron/pom.xml | 133 +++++++++++ .../org/apache/samoa/LocalStormDoTask.java | 83 +++++++ .../samoa/topology/impl/StormBoltStream.java | 67 ++++++ .../topology/impl/StormComponentFactory.java | 90 ++++++++ .../samoa/topology/impl/StormDoTask.java | 118 ++++++++++ .../impl/StormEntranceProcessingItem.java | 212 ++++++++++++++++++ .../topology/impl/StormJarSubmitter.java | 75 +++++++ .../topology/impl/StormProcessingItem.java | 169 ++++++++++++++ .../samoa/topology/impl/StormSamoaUtils.java | 130 +++++++++++ .../samoa/topology/impl/StormSpoutStream.java | 66 ++++++ .../samoa/topology/impl/StormStream.java | 86 +++++++ .../samoa/topology/impl/StormTopology.java | 53 +++++ .../topology/impl/StormTopologyNode.java | 37 +++ .../topology/impl/StormTopologySubmitter.java | 132 +++++++++++ .../test/java/org/apache/samoa/AlgosTest.java | 92 ++++++++ .../impl/StormProcessingItemTest.java | 83 +++++++ 16 files changed, 1626 insertions(+) create mode 100644 samoa-heron/pom.xml create mode 100644 samoa-heron/src/main/java/org/apache/samoa/LocalStormDoTask.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormBoltStream.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormComponentFactory.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormDoTask.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormEntranceProcessingItem.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormJarSubmitter.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormProcessingItem.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSamoaUtils.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSpoutStream.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormStream.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopology.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologyNode.java create mode 100644 samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologySubmitter.java create mode 100644 samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java create mode 100644 samoa-heron/src/test/java/org/apache/samoa/topology/impl/StormProcessingItemTest.java diff --git a/samoa-heron/pom.xml b/samoa-heron/pom.xml new file mode 100644 index 00000000..b104da8a --- /dev/null +++ b/samoa-heron/pom.xml @@ -0,0 +1,133 @@ + + + + + 4.0.0 + + UTF-8 + + + samoa-heron + Heron bindings for SAMOA + + samoa-heron + + org.apache.samoa + samoa + 0.5.0-incubating-SNAPSHOT + + + + + clojars + http://clojars.org/repo/ + + + + + + org.apache.samoa + samoa-api + ${project.version} + + + org.apache.samoa + samoa-test + test-jar + test-jar-with-dependencies + ${project.version} + test + + + + org.apache.storm + storm-core + ${storm.version} + provided + + + org.apache.zookeeper + zookeeper + ${zookeeper.storm.version} + provided + + + org.slf4j + slf4j-log4j12 + ${slf4j-log4j12.version} + test + + + + + + + + maven-assembly-plugin + ${maven-assembly-plugin.version} + + SAMOA-Storm-${project.version} + false + false + ../target + + jar-with-dependencies + + + + ${parsedVersion.osgiVersion} + ${project.description} + ${project.version} + Yahoo Labs + SAMOA + + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + -Xmx1G + false + + + + + + ${project.basedir}/../bin + + *storm.properties + + + + + diff --git a/samoa-heron/src/main/java/org/apache/samoa/LocalStormDoTask.java b/samoa-heron/src/main/java/org/apache/samoa/LocalStormDoTask.java new file mode 100644 index 00000000..f77dbaca --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/LocalStormDoTask.java @@ -0,0 +1,83 @@ +package org.apache.samoa; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.samoa.topology.impl.StormSamoaUtils; +import org.apache.samoa.topology.impl.StormTopology; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.configuration.Configuration; + +import backtype.storm.Config; +import backtype.storm.utils.Utils; + +/** + * The main class to execute a SAMOA task in LOCAL mode in Storm. + * + * @author Arinto Murdopo + * + */ +public class LocalStormDoTask { + + private static final Logger logger = LoggerFactory.getLogger(LocalStormDoTask.class); + private static final String EXECUTION_DURATION_KEY ="samoa.storm.local.mode.execution.duration"; + private static final String SAMOA_STORM_PROPERTY_FILE_LOC ="samoa-storm.properties"; + /** + * The main method. + * + * @param args + * the arguments + */ + public static void main(String[] args) { + + List tmpArgs = new ArrayList(Arrays.asList(args)); + + int numWorker = StormSamoaUtils.numWorkers(tmpArgs); + + args = tmpArgs.toArray(new String[0]); + + // convert the arguments into Storm topology + StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); + String topologyName = stormTopo.getTopologyName(); + + Config conf = new Config(); + // conf.putAll(Utils.readStormConfig()); + conf.setDebug(false); + + // local mode + conf.setMaxTaskParallelism(numWorker); + + backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster(); + cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology()); + + // Read local mode execution duration from property file + Configuration stormConfig = StormSamoaUtils.getPropertyConfig(LocalStormDoTask.SAMOA_STORM_PROPERTY_FILE_LOC); + long executionDuration= stormConfig.getLong(LocalStormDoTask.EXECUTION_DURATION_KEY); + backtype.storm.utils.Utils.sleep(executionDuration * 1000); + + cluster.killTopology(topologyName); + cluster.shutdown(); + + } +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormBoltStream.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormBoltStream.java new file mode 100644 index 00000000..de9ef0a2 --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormBoltStream.java @@ -0,0 +1,67 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.apache.samoa.core.ContentEvent; + +import backtype.storm.task.OutputCollector; +import backtype.storm.tuple.Values; + +/** + * Storm Stream that connects into Bolt. It wraps Storm's outputCollector class + * + * @author Arinto Murdopo + * + */ +class StormBoltStream extends StormStream { + + /** + * + */ + private static final long serialVersionUID = -5712513402991550847L; + + private OutputCollector outputCollector; + + StormBoltStream(String stormComponentId) { + super(stormComponentId); + } + + @Override + public void put(ContentEvent contentEvent) { + outputCollector.emit(this.outputStreamId, new Values(contentEvent, contentEvent.getKey())); + } + + public void setCollector(OutputCollector outputCollector) { + this.outputCollector = outputCollector; + } + + // @Override + // public void setStreamId(String streamId) { + // // TODO Auto-generated method stub + // //this.outputStreamId = streamId; + // } + + @Override + public String getStreamId() { + // TODO Auto-generated method stub + return null; + } +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormComponentFactory.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormComponentFactory.java new file mode 100644 index 00000000..353cffcd --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormComponentFactory.java @@ -0,0 +1,90 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.util.HashMap; +import java.util.Map; + +import org.apache.samoa.core.EntranceProcessor; +import org.apache.samoa.core.Processor; +import org.apache.samoa.topology.ComponentFactory; +import org.apache.samoa.topology.EntranceProcessingItem; +import org.apache.samoa.topology.IProcessingItem; +import org.apache.samoa.topology.ProcessingItem; +import org.apache.samoa.topology.Stream; +import org.apache.samoa.topology.Topology; + +/** + * Component factory implementation for samoa-storm + */ +public final class StormComponentFactory implements ComponentFactory { + + private final Map processorList; + + public StormComponentFactory() { + processorList = new HashMap<>(); + } + + @Override + public ProcessingItem createPi(Processor processor) { + return new StormProcessingItem(processor, this.getComponentName(processor.getClass()), 1); + } + + @Override + public EntranceProcessingItem createEntrancePi(EntranceProcessor processor) { + return new StormEntranceProcessingItem(processor, this.getComponentName(processor.getClass())); + } + + @Override + public Stream createStream(IProcessingItem sourcePi) { + StormTopologyNode stormCompatiblePi = (StormTopologyNode) sourcePi; + return stormCompatiblePi.createStream(); + } + + @Override + public Topology createTopology(String topoName) { + return new StormTopology(topoName); + } + + private String getComponentName(Class clazz) { + StringBuilder componentName = new StringBuilder(clazz.getCanonicalName()); + String key = componentName.toString(); + Integer index; + + if (!processorList.containsKey(key)) { + index = 1; + } else { + index = processorList.get(key) + 1; + } + + processorList.put(key, index); + + componentName.append('_'); + componentName.append(index); + + return componentName.toString(); + } + + @Override + public ProcessingItem createPi(Processor processor, int parallelism) { + return new StormProcessingItem(processor, this.getComponentName(processor.getClass()), parallelism); + } +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormDoTask.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormDoTask.java new file mode 100644 index 00000000..436db592 --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormDoTask.java @@ -0,0 +1,118 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import backtype.storm.Config; +import backtype.storm.utils.Utils; + +/** + * The main class that used by samoa script to execute SAMOA task. + * + * @author Arinto Murdopo + * + */ +public class StormDoTask { + private static final Logger logger = LoggerFactory.getLogger(StormDoTask.class); + private static String localFlag = "local"; + private static String clusterFlag = "cluster"; + + /** + * The main method. + * + * @param args + * the arguments + */ + public static void main(String[] args) { + + List tmpArgs = new ArrayList(Arrays.asList(args)); + + boolean isLocal = isLocal(tmpArgs); + int numWorker = StormSamoaUtils.numWorkers(tmpArgs); + + args = tmpArgs.toArray(new String[0]); + + // convert the arguments into Storm topology + StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); + String topologyName = stormTopo.getTopologyName(); + + Config conf = new Config(); + conf.putAll(Utils.readStormConfig()); + conf.setDebug(false); + + if (isLocal) { + // local mode + conf.setMaxTaskParallelism(numWorker); + + backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster(); + cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology()); + + backtype.storm.utils.Utils.sleep(600 * 1000); + + cluster.killTopology(topologyName); + cluster.shutdown(); + + } else { + // cluster mode + conf.setNumWorkers(numWorker); + try { + backtype.storm.StormSubmitter.submitTopology(topologyName, conf, + stormTopo.getStormBuilder().createTopology()); + } catch (backtype.storm.generated.AlreadyAliveException ale) { + ale.printStackTrace(); + } catch (backtype.storm.generated.InvalidTopologyException ite) { + ite.printStackTrace(); + } + } + } + + private static boolean isLocal(List tmpArgs) { + ExecutionMode executionMode = ExecutionMode.UNDETERMINED; + + int position = tmpArgs.size() - 1; + String flag = tmpArgs.get(position); + boolean isLocal = true; + + if (flag.equals(clusterFlag)) { + executionMode = ExecutionMode.CLUSTER; + isLocal = false; + } else if (flag.equals(localFlag)) { + executionMode = ExecutionMode.LOCAL; + isLocal = true; + } + + if (executionMode != ExecutionMode.UNDETERMINED) { + tmpArgs.remove(position); + } + + return isLocal; + } + + private enum ExecutionMode { + LOCAL, CLUSTER, UNDETERMINED + }; +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormEntranceProcessingItem.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormEntranceProcessingItem.java new file mode 100644 index 00000000..1183fac0 --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormEntranceProcessingItem.java @@ -0,0 +1,212 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.util.Map; +import java.util.UUID; + +import org.apache.samoa.core.ContentEvent; +import org.apache.samoa.core.EntranceProcessor; +import org.apache.samoa.topology.AbstractEntranceProcessingItem; +import org.apache.samoa.topology.EntranceProcessingItem; +import org.apache.samoa.topology.Stream; + +import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseRichSpout; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Values; +import backtype.storm.utils.Utils; + +/** + * EntranceProcessingItem implementation for Storm. + */ +class StormEntranceProcessingItem extends AbstractEntranceProcessingItem implements StormTopologyNode { + private final StormEntranceSpout piSpout; + + StormEntranceProcessingItem(EntranceProcessor processor) { + this(processor, UUID.randomUUID().toString()); + } + + StormEntranceProcessingItem(EntranceProcessor processor, String friendlyId) { + super(processor); + this.setName(friendlyId); + this.piSpout = new StormEntranceSpout(processor); + } + + @Override + public EntranceProcessingItem setOutputStream(Stream stream) { + // piSpout.streams.add(stream); + piSpout.setOutputStream((StormStream) stream); + return this; + } + + @Override + public Stream getOutputStream() { + return piSpout.getOutputStream(); + } + + @Override + public void addToTopology(StormTopology topology, int parallelismHint) { + topology.getStormBuilder().setSpout(this.getName(), piSpout, parallelismHint); + } + + @Override + public StormStream createStream() { + return piSpout.createStream(this.getName()); + } + + @Override + public String getId() { + return this.getName(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(super.toString()); + sb.insert(0, String.format("id: %s, ", this.getName())); + return sb.toString(); + } + + /** + * Resulting Spout of StormEntranceProcessingItem + */ + final static class StormEntranceSpout extends BaseRichSpout { + + private static final long serialVersionUID = -9066409791668954099L; + + // private final Set streams; + private final EntranceProcessor entranceProcessor; + private StormStream outputStream; + + // private transient SpoutStarter spoutStarter; + // private transient Executor spoutExecutors; + // private transient LinkedBlockingQueue tupleInfoQueue; + + private SpoutOutputCollector collector; + + StormEntranceSpout(EntranceProcessor processor) { + // this.streams = new HashSet(); + this.entranceProcessor = processor; + } + + public StormStream getOutputStream() { + return outputStream; + } + + public void setOutputStream(StormStream stream) { + this.outputStream = stream; + } + + @Override + public void open(@SuppressWarnings("rawtypes") Map conf, TopologyContext context, SpoutOutputCollector collector) { + this.collector = collector; + // this.tupleInfoQueue = new LinkedBlockingQueue(); + + // Processor and this class share the same instance of stream + // for (StormSpoutStream stream : streams) { + // stream.setSpout(this); + // } + // outputStream.setSpout(this); + + this.entranceProcessor.onCreate(context.getThisTaskId()); + // this.spoutStarter = new SpoutStarter(this.starter); + + // this.spoutExecutors = Executors.newSingleThreadExecutor(); + // this.spoutExecutors.execute(spoutStarter); + } + + @Override + public void nextTuple() { + if (entranceProcessor.hasNext()) { + Values value = newValues(entranceProcessor.nextEvent()); + collector.emit(outputStream.getOutputId(), value); + } else + Utils.sleep(1000); + // StormTupleInfo tupleInfo = tupleInfoQueue.poll(50, + // TimeUnit.MILLISECONDS); + // if (tupleInfo != null) { + // Values value = newValues(tupleInfo.getContentEvent()); + // collector.emit(tupleInfo.getStormStream().getOutputId(), value); + // } + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + // for (StormStream stream : streams) { + // declarer.declareStream(stream.getOutputId(), new + // Fields(StormSamoaUtils.CONTENT_EVENT_FIELD, + // StormSamoaUtils.KEY_FIELD)); + // } + declarer.declareStream(outputStream.getOutputId(), new Fields(StormSamoaUtils.CONTENT_EVENT_FIELD, + StormSamoaUtils.KEY_FIELD)); + } + + StormStream createStream(String piId) { + // StormSpoutStream stream = new StormSpoutStream(piId); + StormStream stream = new StormBoltStream(piId); + // streams.add(stream); + return stream; + } + + // void put(StormSpoutStream stream, ContentEvent contentEvent) { + // tupleInfoQueue.add(new StormTupleInfo(stream, contentEvent)); + // } + + private Values newValues(ContentEvent contentEvent) { + return new Values(contentEvent, contentEvent.getKey()); + } + + // private final static class StormTupleInfo { + // + // private final StormStream stream; + // private final ContentEvent event; + // + // StormTupleInfo(StormStream stream, ContentEvent event) { + // this.stream = stream; + // this.event = event; + // } + // + // public StormStream getStormStream() { + // return this.stream; + // } + // + // public ContentEvent getContentEvent() { + // return this.event; + // } + // } + + // private final static class SpoutStarter implements Runnable { + // + // private final TopologyStarter topoStarter; + // + // SpoutStarter(TopologyStarter topoStarter) { + // this.topoStarter = topoStarter; + // } + // + // @Override + // public void run() { + // this.topoStarter.start(); + // } + // } + } +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormJarSubmitter.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormJarSubmitter.java new file mode 100644 index 00000000..a3de7982 --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormJarSubmitter.java @@ -0,0 +1,75 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Properties; + +import backtype.storm.Config; +import backtype.storm.StormSubmitter; +import backtype.storm.utils.Utils; + +/** + * Utility class to submit samoa-storm jar to a Storm cluster. + * + * @author Arinto Murdopo + * + */ +public class StormJarSubmitter { + + public final static String UPLOADED_JAR_LOCATION_KEY = "UploadedJarLocation"; + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + + Config config = new Config(); + config.putAll(Utils.readCommandLineOpts()); + config.putAll(Utils.readStormConfig()); + + String nimbusHost = (String) config.get(Config.NIMBUS_HOST); + int nimbusThriftPort = Utils.getInt(config + .get(Config.NIMBUS_THRIFT_PORT)); + + System.out.println("Nimbus host " + nimbusHost); + System.out.println("Nimbus thrift port " + nimbusThriftPort); + + System.out.println("uploading jar from " + args[0]); + String uploadedJarLocation = StormSubmitter.submitJar(config, args[0]); + + System.out.println("Uploaded jar file location: "); + System.out.println(uploadedJarLocation); + + Properties props = StormSamoaUtils.getProperties(); + props.setProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY, uploadedJarLocation); + + File f = new File("src/main/resources/samoa-storm-cluster.properties"); + f.createNewFile(); + + OutputStream out = new FileOutputStream(f); + props.store(out, "properties file to store uploaded jar location from StormJarSubmitter"); + } +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormProcessingItem.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormProcessingItem.java new file mode 100644 index 00000000..28c37468 --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormProcessingItem.java @@ -0,0 +1,169 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.apache.samoa.core.ContentEvent; +import org.apache.samoa.core.Processor; +import org.apache.samoa.topology.AbstractProcessingItem; +import org.apache.samoa.topology.ProcessingItem; +import org.apache.samoa.topology.Stream; +import org.apache.samoa.topology.impl.StormStream.InputStreamId; +import org.apache.samoa.utils.PartitioningScheme; + +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.BoltDeclarer; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.topology.base.BaseRichBolt; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; + +/** + * ProcessingItem implementation for Storm. + * + * @author Arinto Murdopo + * + */ +class StormProcessingItem extends AbstractProcessingItem implements StormTopologyNode { + private final ProcessingItemBolt piBolt; + private BoltDeclarer piBoltDeclarer; + + // TODO: should we put parallelism hint here? + // imo, parallelism hint only declared when we add this PI in the topology + // open for dicussion :p + + StormProcessingItem(Processor processor, int parallelismHint) { + this(processor, UUID.randomUUID().toString(), parallelismHint); + } + + StormProcessingItem(Processor processor, String friendlyId, int parallelismHint) { + super(processor, parallelismHint); + this.piBolt = new ProcessingItemBolt(processor); + this.setName(friendlyId); + } + + @Override + protected ProcessingItem addInputStream(Stream inputStream, PartitioningScheme scheme) { + StormStream stormInputStream = (StormStream) inputStream; + InputStreamId inputId = stormInputStream.getInputId(); + + switch (scheme) { + case SHUFFLE: + piBoltDeclarer.shuffleGrouping(inputId.getComponentId(), inputId.getStreamId()); + break; + case GROUP_BY_KEY: + piBoltDeclarer.fieldsGrouping( + inputId.getComponentId(), + inputId.getStreamId(), + new Fields(StormSamoaUtils.KEY_FIELD)); + break; + case BROADCAST: + piBoltDeclarer.allGrouping( + inputId.getComponentId(), + inputId.getStreamId()); + break; + } + return this; + } + + @Override + public void addToTopology(StormTopology topology, int parallelismHint) { + if (piBoltDeclarer != null) { + // throw exception that one PI only belong to one topology + } else { + TopologyBuilder stormBuilder = topology.getStormBuilder(); + this.piBoltDeclarer = stormBuilder.setBolt(this.getName(), + this.piBolt, parallelismHint); + } + } + + @Override + public StormStream createStream() { + return piBolt.createStream(this.getName()); + } + + @Override + public String getId() { + return this.getName(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(super.toString()); + sb.insert(0, String.format("id: %s, ", this.getName())); + return sb.toString(); + } + + private final static class ProcessingItemBolt extends BaseRichBolt { + + private static final long serialVersionUID = -6637673741263199198L; + + private final Set streams; + private final Processor processor; + + private OutputCollector collector; + + ProcessingItemBolt(Processor processor) { + this.streams = new HashSet(); + this.processor = processor; + } + + @Override + public void prepare(@SuppressWarnings("rawtypes") Map stormConf, TopologyContext context, + OutputCollector collector) { + this.collector = collector; + // Processor and this class share the same instance of stream + for (StormBoltStream stream : streams) { + stream.setCollector(this.collector); + } + + this.processor.onCreate(context.getThisTaskId()); + } + + @Override + public void execute(Tuple input) { + Object sentObject = input.getValue(0); + ContentEvent sentEvent = (ContentEvent) sentObject; + processor.process(sentEvent); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + for (StormStream stream : streams) { + declarer.declareStream(stream.getOutputId(), + new Fields(StormSamoaUtils.CONTENT_EVENT_FIELD, + StormSamoaUtils.KEY_FIELD)); + } + } + + StormStream createStream(String piId) { + StormBoltStream stream = new StormBoltStream(piId); + streams.add(stream); + return stream; + } + } +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSamoaUtils.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSamoaUtils.java new file mode 100644 index 00000000..7f7e5780 --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSamoaUtils.java @@ -0,0 +1,130 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import com.github.javacliparser.ClassOption; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Properties; + +import org.apache.samoa.tasks.Task; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; + +/** + * Utility class for samoa-storm project. It is used by StormDoTask to process its arguments. + * + * @author Arinto Murdopo + * + */ +public class StormSamoaUtils { + + private static final Logger logger = LoggerFactory.getLogger(StormSamoaUtils.class); + + static final String KEY_FIELD = "key"; + static final String CONTENT_EVENT_FIELD = "content_event"; + + static Properties getProperties() throws IOException { + Properties props = new Properties(); + InputStream is; + + File f = new File("src/main/resources/samoa-storm-cluster.properties"); // FIXME it does not exist anymore + is = new FileInputStream(f); + + try { + props.load(is); + } catch (IOException e1) { + System.out.println("Fail to load property file"); + return null; + } finally { + is.close(); + } + + return props; + } + + public static StormTopology argsToTopology(String[] args) { + StringBuilder cliString = new StringBuilder(); + for (String arg : args) { + cliString.append(" ").append(arg); + } + logger.debug("Command line string = {}", cliString.toString()); + + Task task = getTask(cliString.toString()); + + // TODO: remove setFactory method with DynamicBinding + task.setFactory(new StormComponentFactory()); + task.init(); + + return (StormTopology) task.getTopology(); + } + + public static int numWorkers(List tmpArgs) { + int position = tmpArgs.size() - 1; + int numWorkers; + + try { + numWorkers = Integer.parseInt(tmpArgs.get(position)); + tmpArgs.remove(position); + } catch (NumberFormatException e) { + numWorkers = 4; + } + + return numWorkers; + } + + public static Task getTask(String cliString) { + Task task = null; + try { + logger.debug("Providing task [{}]", cliString); + task = ClassOption.cliStringToObject(cliString, Task.class, null); + } catch (Exception e) { + logger.warn("Fail in initializing the task!"); + e.printStackTrace(); + } + return task; + } + + public static Configuration getPropertyConfig(String configPropertyPath){ + Configuration config = null; + try { + config = new PropertiesConfiguration(configPropertyPath); + if (null == config || config.isEmpty()) { + logger.error("Configuration is null or empty at file = {}",configPropertyPath); + throw new RuntimeException("Configuration is null or empty : " + configPropertyPath); + } + } + catch(ConfigurationException configurationException) + { + logger.error("ConfigurationException while reading property file = {}",configurationException); + throw new RuntimeException("ConfigurationException while reading property file : " + configPropertyPath); + } + return config; + } +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSpoutStream.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSpoutStream.java new file mode 100644 index 00000000..14fc4243 --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSpoutStream.java @@ -0,0 +1,66 @@ +package org.apache.samoa.topology.impl; +//package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +// +//import org.apache.samoa.core.ContentEvent; +//import org.apache.samoa.topology.impl.StormEntranceProcessingItem.StormEntranceSpout; +// +///** +// * Storm Stream that connects into Spout. It wraps the spout itself +// * @author Arinto Murdopo +// * +// */ +//final class StormSpoutStream extends StormStream{ +// +// /** +// * +// */ +// private static final long serialVersionUID = -7444653177614988650L; +// +// private StormEntranceSpout spout; +// +// StormSpoutStream(String stormComponentId) { +// super(stormComponentId); +// } +// +// @Override +// public void put(ContentEvent contentEvent) { +// spout.put(this, contentEvent); +// } +// +// void setSpout(StormEntranceSpout spout){ +// this.spout = spout; +// } +// +//// @Override +//// public void setStreamId(String stream) { +//// // TODO Auto-generated method stub +//// +//// } +// +// @Override +// public String getStreamId() { +// // TODO Auto-generated method stub +// return null; +// } +// +// } diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormStream.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormStream.java new file mode 100644 index 00000000..1c623896 --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormStream.java @@ -0,0 +1,86 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.util.UUID; + +import org.apache.samoa.core.ContentEvent; +import org.apache.samoa.topology.Stream; + +/** + * Abstract class to implement Storm Stream + * + * @author Arinto Murdopo + * + */ +abstract class StormStream implements Stream, java.io.Serializable { + + /** + * + */ + private static final long serialVersionUID = 281835563756514852L; + protected final String outputStreamId; + protected final InputStreamId inputStreamId; + + public StormStream(String stormComponentId) { + this.outputStreamId = UUID.randomUUID().toString(); + this.inputStreamId = new InputStreamId(stormComponentId, this.outputStreamId); + } + + @Override + public abstract void put(ContentEvent contentEvent); + + String getOutputId() { + return this.outputStreamId; + } + + InputStreamId getInputId() { + return this.inputStreamId; + } + + final static class InputStreamId implements java.io.Serializable { + + /** + * + */ + private static final long serialVersionUID = -7457995634133691295L; + private final String componentId; + private final String streamId; + + InputStreamId(String componentId, String streamId) { + this.componentId = componentId; + this.streamId = streamId; + } + + String getComponentId() { + return componentId; + } + + String getStreamId() { + return streamId; + } + } + + @Override + public void setBatchSize(int batchSize) { + // Ignore batch size + } +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopology.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopology.java new file mode 100644 index 00000000..51bb9097 --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopology.java @@ -0,0 +1,53 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.apache.samoa.topology.AbstractTopology; +import org.apache.samoa.topology.IProcessingItem; + +import backtype.storm.topology.TopologyBuilder; + +/** + * Adaptation of SAMOA topology in samoa-storm + * + * @author Arinto Murdopo + * + */ +public class StormTopology extends AbstractTopology { + + private TopologyBuilder builder; + + public StormTopology(String topologyName) { + super(topologyName); + this.builder = new TopologyBuilder(); + } + + @Override + public void addProcessingItem(IProcessingItem procItem, int parallelismHint) { + StormTopologyNode stormNode = (StormTopologyNode) procItem; + stormNode.addToTopology(this, parallelismHint); + super.addProcessingItem(procItem, parallelismHint); + } + + public TopologyBuilder getStormBuilder() { + return builder; + } +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologyNode.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologyNode.java new file mode 100644 index 00000000..994fb9fc --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologyNode.java @@ -0,0 +1,37 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +/** + * Interface to represent a node in samoa-storm topology. + * + * @author Arinto Murdopo + * + */ +interface StormTopologyNode { + + void addToTopology(StormTopology topology, int parallelismHint); + + StormStream createStream(); + + String getId(); + +} diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologySubmitter.java b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologySubmitter.java new file mode 100644 index 00000000..25985dfd --- /dev/null +++ b/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologySubmitter.java @@ -0,0 +1,132 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.thrift7.TException; +import org.json.simple.JSONValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import backtype.storm.Config; +import backtype.storm.generated.AlreadyAliveException; +import backtype.storm.generated.InvalidTopologyException; +import backtype.storm.utils.NimbusClient; +import backtype.storm.utils.Utils; + +/** + * Helper class to submit SAMOA task into Storm without the need of submitting the jar file. The jar file must be + * submitted first using StormJarSubmitter class. + * + * @author Arinto Murdopo + * + */ +public class StormTopologySubmitter { + + public static String YJP_OPTIONS_KEY = "YjpOptions"; + + private static Logger logger = LoggerFactory.getLogger(StormTopologySubmitter.class); + + public static void main(String[] args) throws IOException { + Properties props = StormSamoaUtils.getProperties(); + + String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY); + if (uploadedJarLocation == null) { + logger.error("Invalid properties file. It must have key {}", + StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY); + return; + } + + List tmpArgs = new ArrayList(Arrays.asList(args)); + int numWorkers = StormSamoaUtils.numWorkers(tmpArgs); + + args = tmpArgs.toArray(new String[0]); + StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); + + Config conf = new Config(); + conf.putAll(Utils.readStormConfig()); + conf.putAll(Utils.readCommandLineOpts()); + conf.setDebug(false); + conf.setNumWorkers(numWorkers); + + String profilerOption = + props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY); + if (profilerOption != null) { + String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS); + StringBuilder optionBuilder = new StringBuilder(); + if (topoWorkerChildOpts != null) { + optionBuilder.append(topoWorkerChildOpts); + optionBuilder.append(' '); + } + optionBuilder.append(profilerOption); + conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString()); + } + + Map myConfigMap = new HashMap(conf); + StringWriter out = new StringWriter(); + + try { + JSONValue.writeJSONString(myConfigMap, out); + } catch (IOException e) { + System.out.println("Error in writing JSONString"); + e.printStackTrace(); + return; + } + + Config config = new Config(); + config.putAll(Utils.readStormConfig()); + + NimbusClient nc = NimbusClient.getConfiguredClient(config); + String topologyName = stormTopo.getTopologyName(); + try { + System.out.println("Submitting topology with name: " + + topologyName); + nc.getClient().submitTopology(topologyName, uploadedJarLocation, + out.toString(), stormTopo.getStormBuilder().createTopology()); + System.out.println(topologyName + " is successfully submitted"); + + } catch (AlreadyAliveException aae) { + System.out.println("Fail to submit " + topologyName + + "\nError message: " + aae.get_msg()); + } catch (InvalidTopologyException ite) { + System.out.println("Invalid topology for " + topologyName); + ite.printStackTrace(); + } catch (TException te) { + System.out.println("Texception for " + topologyName); + te.printStackTrace(); + } + } + + private static String uploadedJarLocation(List tmpArgs) { + int position = tmpArgs.size() - 1; + String uploadedJarLocation = tmpArgs.get(position); + tmpArgs.remove(position); + return uploadedJarLocation; + } +} diff --git a/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java b/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java new file mode 100644 index 00000000..85204458 --- /dev/null +++ b/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java @@ -0,0 +1,92 @@ +package org.apache.samoa; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.apache.samoa.LocalStormDoTask; +import org.apache.samoa.TestParams; +import org.apache.samoa.TestUtils; +import org.junit.Test; + +public class AlgosTest { + + @Test(timeout = 60000) + public void testVHTWithStorm() throws Exception { + + TestParams vhtConfig = new TestParams.Builder() + .inputInstances(200_000) + .samplingSize(20_000) + .evaluationInstances(200_000) + .classifiedInstances(200_000) + .labelSamplingSize(10l) + .classificationsCorrect(55f) + .kappaStat(-0.1f) + .kappaTempStat(-0.1f) + .cliStringTemplate(TestParams.Templates.PREQEVAL_VHT_RANDOMTREE) + .resultFilePollTimeout(30) + .prePollWait(15) + .taskClassName(LocalStormDoTask.class.getName()) + .build(); + TestUtils.test(vhtConfig); + + } + + @Test(timeout = 120000) + public void testBaggingWithStorm() throws Exception { + TestParams baggingConfig = new TestParams.Builder() + .inputInstances(200_000) + .samplingSize(20_000) + .evaluationInstances(180_000) + .classifiedInstances(190_000) + .labelSamplingSize(10l) + .classificationsCorrect(60f) + .kappaStat(0f) + .kappaTempStat(0f) + .cliStringTemplate(TestParams.Templates.PREQEVAL_BAGGING_RANDOMTREE) + .resultFilePollTimeout(40) + .prePollWait(20) + .taskClassName(LocalStormDoTask.class.getName()) + .build(); + TestUtils.test(baggingConfig); + + } + + @Test(timeout = 240000) + public void testCVPReqVHTWithStorm() throws Exception { + + TestParams vhtConfig = new TestParams.Builder() + .inputInstances(200_000) + .samplingSize(20_000) + .evaluationInstances(200_000) + .classifiedInstances(200_000) + .classificationsCorrect(55f) + .kappaStat(0f) + .kappaTempStat(0f) + .cliStringTemplate(TestParams.Templates.PREQCVEVAL_VHT_RANDOMTREE) + .resultFilePollTimeout(30) + .prePollWait(15) + .taskClassName(LocalStormDoTask.class.getName()) + .labelFileCreated(false) + .build(); + TestUtils.test(vhtConfig); + + } + +} diff --git a/samoa-heron/src/test/java/org/apache/samoa/topology/impl/StormProcessingItemTest.java b/samoa-heron/src/test/java/org/apache/samoa/topology/impl/StormProcessingItemTest.java new file mode 100644 index 00000000..46739036 --- /dev/null +++ b/samoa-heron/src/test/java/org/apache/samoa/topology/impl/StormProcessingItemTest.java @@ -0,0 +1,83 @@ +package org.apache.samoa.topology.impl; + +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2015 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import static org.junit.Assert.assertEquals; +import mockit.Expectations; +import mockit.MockUp; +import mockit.Mocked; +import mockit.Tested; +import mockit.Verifications; + +import org.apache.samoa.core.Processor; +import org.apache.samoa.topology.impl.StormProcessingItem; +import org.apache.samoa.topology.impl.StormTopology; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import backtype.storm.topology.BoltDeclarer; +import backtype.storm.topology.IRichBolt; +import backtype.storm.topology.TopologyBuilder; + +public class StormProcessingItemTest { + private static final int PARRALLELISM_HINT_2 = 2; + private static final int PARRALLELISM_HINT_4 = 4; + private static final String ID = "id"; + @Tested + private StormProcessingItem pi; + @Mocked + private Processor processor; + @Mocked + private StormTopology topology; + @Mocked + private TopologyBuilder stormBuilder = new TopologyBuilder(); + + @Before + public void setUp() { + pi = new StormProcessingItem(processor, ID, PARRALLELISM_HINT_2); + } + + @Test + public void testAddToTopology() { + new Expectations() { + { + topology.getStormBuilder(); + result = stormBuilder; + + stormBuilder.setBolt(ID, (IRichBolt) any, anyInt); + result = new MockUp() { + }.getMockInstance(); + } + }; + + pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored + + new Verifications() { + { + assertEquals(pi.getProcessor(), processor); + // TODO add methods to explore a topology and verify them + assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2); + assertEquals(pi.getId(), ID); + } + }; + } +} From b0e090d56e65be75d55cba082e2a129c75dce8a5 Mon Sep 17 00:00:00 2001 From: Saikat Kanjilal Date: Thu, 11 Oct 2018 17:12:44 -0700 Subject: [PATCH 2/5] first successful build after renaming everything to Heron oriented names --- ...StormDoTask.java => LocalHeronDoTask.java} | 32 ++++++------ .../topology/impl/HeronBoltStream.java} | 6 +-- .../topology/impl/HeronComponentFactory.java} | 16 +++--- .../topology/impl/HeronDoTask.java} | 16 +++--- .../impl/HeronEntranceProcessingItem.java} | 50 +++++++++---------- .../topology/impl/HeronJarSubmitter.java} | 8 +-- .../topology/impl/HeronProcessingItem.java} | 39 +++++++-------- .../topology/impl/HeronSamoaUtils.java} | 16 +++--- .../topology/impl/HeronSpoutStream.java} | 3 +- .../topology/impl/HeronStream.java} | 6 +-- .../topology/impl/HeronTopology.java} | 12 ++--- .../topology/impl/HeronTopologyNode.java} | 8 +-- .../impl/HeronTopologySubmitter.java} | 22 ++++---- .../test/java/org/apache/samoa/AlgosTest.java | 25 ++++++---- .../impl/HeronProcessingItemTest.java} | 22 ++++---- 15 files changed, 140 insertions(+), 141 deletions(-) rename samoa-heron/src/main/java/org/apache/samoa/{LocalStormDoTask.java => LocalHeronDoTask.java} (70%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormBoltStream.java => heron/topology/impl/HeronBoltStream.java} (92%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormComponentFactory.java => heron/topology/impl/HeronComponentFactory.java} (83%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormDoTask.java => heron/topology/impl/HeronDoTask.java} (85%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormEntranceProcessingItem.java => heron/topology/impl/HeronEntranceProcessingItem.java} (79%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormJarSubmitter.java => heron/topology/impl/HeronJarSubmitter.java} (91%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormProcessingItem.java => heron/topology/impl/HeronProcessingItem.java} (78%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormSamoaUtils.java => heron/topology/impl/HeronSamoaUtils.java} (88%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormSpoutStream.java => heron/topology/impl/HeronSpoutStream.java} (95%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormStream.java => heron/topology/impl/HeronStream.java} (92%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormTopology.java => heron/topology/impl/HeronTopology.java} (79%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormTopologyNode.java => heron/topology/impl/HeronTopologyNode.java} (82%) rename samoa-heron/src/main/java/org/apache/samoa/{topology/impl/StormTopologySubmitter.java => heron/topology/impl/HeronTopologySubmitter.java} (85%) rename samoa-heron/src/test/java/org/apache/samoa/{topology/impl/StormProcessingItemTest.java => heron/topology/impl/HeronProcessingItemTest.java} (78%) diff --git a/samoa-heron/src/main/java/org/apache/samoa/LocalStormDoTask.java b/samoa-heron/src/main/java/org/apache/samoa/LocalHeronDoTask.java similarity index 70% rename from samoa-heron/src/main/java/org/apache/samoa/LocalStormDoTask.java rename to samoa-heron/src/main/java/org/apache/samoa/LocalHeronDoTask.java index f77dbaca..b4ccccbf 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/LocalStormDoTask.java +++ b/samoa-heron/src/main/java/org/apache/samoa/LocalHeronDoTask.java @@ -1,7 +1,5 @@ -package org.apache.samoa; - -/* - * #%L +package org.apache.samoa.heron; + /* #%L * SAMOA * %% * Copyright (C) 2014 - 2015 Apache Software Foundation @@ -23,8 +21,8 @@ import java.util.Arrays; import java.util.List; -import org.apache.samoa.topology.impl.StormSamoaUtils; -import org.apache.samoa.topology.impl.StormTopology; +import org.apache.samoa.heron.topology.impl.HeronSamoaUtils; +import org.apache.samoa.heron.topology.impl.HeronTopology; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.configuration.Configuration; @@ -33,16 +31,16 @@ import backtype.storm.utils.Utils; /** - * The main class to execute a SAMOA task in LOCAL mode in Storm. + * The main class to execute a SAMOA task in LOCAL mode in Heron. * * @author Arinto Murdopo * */ -public class LocalStormDoTask { +public class LocalHeronDoTask { - private static final Logger logger = LoggerFactory.getLogger(LocalStormDoTask.class); + private static final Logger logger = LoggerFactory.getLogger(LocalHeronDoTask.class); private static final String EXECUTION_DURATION_KEY ="samoa.storm.local.mode.execution.duration"; - private static final String SAMOA_STORM_PROPERTY_FILE_LOC ="samoa-storm.properties"; + private static final String SAMOA_STORM_PROPERTY_FILE_LOC ="samoa-heron.properties"; /** * The main method. * @@ -53,13 +51,13 @@ public static void main(String[] args) { List tmpArgs = new ArrayList(Arrays.asList(args)); - int numWorker = StormSamoaUtils.numWorkers(tmpArgs); + int numWorker = HeronSamoaUtils.numWorkers(tmpArgs); args = tmpArgs.toArray(new String[0]); - // convert the arguments into Storm topology - StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); - String topologyName = stormTopo.getTopologyName(); + // convert the arguments into Heron topology + HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args); + String topologyName = heronTopo.getTopologyName(); Config conf = new Config(); // conf.putAll(Utils.readStormConfig()); @@ -69,11 +67,11 @@ public static void main(String[] args) { conf.setMaxTaskParallelism(numWorker); backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster(); - cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology()); + cluster.submitTopology(topologyName, conf, heronTopo.getHeronBuilder().createTopology()); // Read local mode execution duration from property file - Configuration stormConfig = StormSamoaUtils.getPropertyConfig(LocalStormDoTask.SAMOA_STORM_PROPERTY_FILE_LOC); - long executionDuration= stormConfig.getLong(LocalStormDoTask.EXECUTION_DURATION_KEY); + Configuration stormConfig = HeronSamoaUtils.getPropertyConfig(LocalHeronDoTask.SAMOA_STORM_PROPERTY_FILE_LOC); + long executionDuration= stormConfig.getLong(LocalHeronDoTask.EXECUTION_DURATION_KEY); backtype.storm.utils.Utils.sleep(executionDuration * 1000); cluster.killTopology(topologyName); diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormBoltStream.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronBoltStream.java similarity index 92% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormBoltStream.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronBoltStream.java index de9ef0a2..aaf9a760 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormBoltStream.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronBoltStream.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -31,7 +31,7 @@ * @author Arinto Murdopo * */ -class StormBoltStream extends StormStream { +class HeronBoltStream extends HeronStream { /** * @@ -40,7 +40,7 @@ class StormBoltStream extends StormStream { private OutputCollector outputCollector; - StormBoltStream(String stormComponentId) { + HeronBoltStream(String stormComponentId) { super(stormComponentId); } diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormComponentFactory.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronComponentFactory.java similarity index 83% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormComponentFactory.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronComponentFactory.java index 353cffcd..968ef4d6 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormComponentFactory.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronComponentFactory.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -35,33 +35,33 @@ /** * Component factory implementation for samoa-storm */ -public final class StormComponentFactory implements ComponentFactory { +public final class HeronComponentFactory implements ComponentFactory { private final Map processorList; - public StormComponentFactory() { + public HeronComponentFactory() { processorList = new HashMap<>(); } @Override public ProcessingItem createPi(Processor processor) { - return new StormProcessingItem(processor, this.getComponentName(processor.getClass()), 1); + return new HeronProcessingItem(processor, this.getComponentName(processor.getClass()), 1); } @Override public EntranceProcessingItem createEntrancePi(EntranceProcessor processor) { - return new StormEntranceProcessingItem(processor, this.getComponentName(processor.getClass())); + return new HeronEntranceProcessingItem(processor, this.getComponentName(processor.getClass())); } @Override public Stream createStream(IProcessingItem sourcePi) { - StormTopologyNode stormCompatiblePi = (StormTopologyNode) sourcePi; + HeronTopologyNode stormCompatiblePi = (HeronTopologyNode) sourcePi; return stormCompatiblePi.createStream(); } @Override public Topology createTopology(String topoName) { - return new StormTopology(topoName); + return new HeronTopology(topoName); } private String getComponentName(Class clazz) { @@ -85,6 +85,6 @@ private String getComponentName(Class clazz) { @Override public ProcessingItem createPi(Processor processor, int parallelism) { - return new StormProcessingItem(processor, this.getComponentName(processor.getClass()), parallelism); + return new HeronProcessingItem(processor, this.getComponentName(processor.getClass()), parallelism); } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormDoTask.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronDoTask.java similarity index 85% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormDoTask.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronDoTask.java index 436db592..2fe74820 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormDoTask.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronDoTask.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -36,8 +36,8 @@ * @author Arinto Murdopo * */ -public class StormDoTask { - private static final Logger logger = LoggerFactory.getLogger(StormDoTask.class); +public class HeronDoTask { + private static final Logger logger = LoggerFactory.getLogger(HeronDoTask.class); private static String localFlag = "local"; private static String clusterFlag = "cluster"; @@ -52,13 +52,13 @@ public static void main(String[] args) { List tmpArgs = new ArrayList(Arrays.asList(args)); boolean isLocal = isLocal(tmpArgs); - int numWorker = StormSamoaUtils.numWorkers(tmpArgs); + int numWorker = HeronSamoaUtils.numWorkers(tmpArgs); args = tmpArgs.toArray(new String[0]); // convert the arguments into Storm topology - StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); - String topologyName = stormTopo.getTopologyName(); + HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args); + String topologyName = heronTopo.getTopologyName(); Config conf = new Config(); conf.putAll(Utils.readStormConfig()); @@ -69,7 +69,7 @@ public static void main(String[] args) { conf.setMaxTaskParallelism(numWorker); backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster(); - cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology()); + cluster.submitTopology(topologyName, conf, heronTopo.getHeronBuilder().createTopology()); backtype.storm.utils.Utils.sleep(600 * 1000); @@ -81,7 +81,7 @@ public static void main(String[] args) { conf.setNumWorkers(numWorker); try { backtype.storm.StormSubmitter.submitTopology(topologyName, conf, - stormTopo.getStormBuilder().createTopology()); + heronTopo.getHeronBuilder().createTopology()); } catch (backtype.storm.generated.AlreadyAliveException ale) { ale.printStackTrace(); } catch (backtype.storm.generated.InvalidTopologyException ite) { diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormEntranceProcessingItem.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronEntranceProcessingItem.java similarity index 79% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormEntranceProcessingItem.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronEntranceProcessingItem.java index 1183fac0..c4ae6f35 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormEntranceProcessingItem.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronEntranceProcessingItem.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -38,25 +38,25 @@ import backtype.storm.utils.Utils; /** - * EntranceProcessingItem implementation for Storm. + * EntranceProcessingItem implementation for Heron. */ -class StormEntranceProcessingItem extends AbstractEntranceProcessingItem implements StormTopologyNode { - private final StormEntranceSpout piSpout; +class HeronEntranceProcessingItem extends AbstractEntranceProcessingItem implements HeronTopologyNode { + private final HeronEntranceSpout piSpout; - StormEntranceProcessingItem(EntranceProcessor processor) { + HeronEntranceProcessingItem(EntranceProcessor processor) { this(processor, UUID.randomUUID().toString()); } - StormEntranceProcessingItem(EntranceProcessor processor, String friendlyId) { + HeronEntranceProcessingItem(EntranceProcessor processor, String friendlyId) { super(processor); this.setName(friendlyId); - this.piSpout = new StormEntranceSpout(processor); + this.piSpout = new HeronEntranceSpout(processor); } @Override public EntranceProcessingItem setOutputStream(Stream stream) { // piSpout.streams.add(stream); - piSpout.setOutputStream((StormStream) stream); + piSpout.setOutputStream((HeronStream) stream); return this; } @@ -66,12 +66,12 @@ public Stream getOutputStream() { } @Override - public void addToTopology(StormTopology topology, int parallelismHint) { - topology.getStormBuilder().setSpout(this.getName(), piSpout, parallelismHint); + public void addToTopology(HeronTopology topology, int parallelismHint) { + topology.getHeronBuilder().setSpout(this.getName(), piSpout, parallelismHint); } @Override - public StormStream createStream() { + public HeronStream createStream() { return piSpout.createStream(this.getName()); } @@ -90,13 +90,13 @@ public String toString() { /** * Resulting Spout of StormEntranceProcessingItem */ - final static class StormEntranceSpout extends BaseRichSpout { + final static class HeronEntranceSpout extends BaseRichSpout { private static final long serialVersionUID = -9066409791668954099L; // private final Set streams; private final EntranceProcessor entranceProcessor; - private StormStream outputStream; + private HeronStream outputStream; // private transient SpoutStarter spoutStarter; // private transient Executor spoutExecutors; @@ -104,16 +104,16 @@ final static class StormEntranceSpout extends BaseRichSpout { private SpoutOutputCollector collector; - StormEntranceSpout(EntranceProcessor processor) { + HeronEntranceSpout(EntranceProcessor processor) { // this.streams = new HashSet(); this.entranceProcessor = processor; } - public StormStream getOutputStream() { + public HeronStream getOutputStream() { return outputStream; } - public void setOutputStream(StormStream stream) { + public void setOutputStream(HeronStream stream) { this.outputStream = stream; } @@ -146,24 +146,24 @@ public void nextTuple() { // TimeUnit.MILLISECONDS); // if (tupleInfo != null) { // Values value = newValues(tupleInfo.getContentEvent()); - // collector.emit(tupleInfo.getStormStream().getOutputId(), value); + // collector.emit(tupleInfo.getHeronStream().getOutputId(), value); // } } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - // for (StormStream stream : streams) { + // for (HeronStream stream : streams) { // declarer.declareStream(stream.getOutputId(), new // Fields(StormSamoaUtils.CONTENT_EVENT_FIELD, // StormSamoaUtils.KEY_FIELD)); // } - declarer.declareStream(outputStream.getOutputId(), new Fields(StormSamoaUtils.CONTENT_EVENT_FIELD, - StormSamoaUtils.KEY_FIELD)); + declarer.declareStream(outputStream.getOutputId(), new Fields(HeronSamoaUtils.CONTENT_EVENT_FIELD, + HeronSamoaUtils.KEY_FIELD)); } - StormStream createStream(String piId) { + HeronStream createStream(String piId) { // StormSpoutStream stream = new StormSpoutStream(piId); - StormStream stream = new StormBoltStream(piId); + HeronStream stream = new HeronBoltStream(piId); // streams.add(stream); return stream; } @@ -178,15 +178,15 @@ private Values newValues(ContentEvent contentEvent) { // private final static class StormTupleInfo { // - // private final StormStream stream; + // private final HeronStream stream; // private final ContentEvent event; // - // StormTupleInfo(StormStream stream, ContentEvent event) { + // StormTupleInfo(HeronStream stream, ContentEvent event) { // this.stream = stream; // this.event = event; // } // - // public StormStream getStormStream() { + // public HeronStream getHeronStream() { // return this.stream; // } // diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormJarSubmitter.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronJarSubmitter.java similarity index 91% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormJarSubmitter.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronJarSubmitter.java index a3de7982..aa5197d1 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormJarSubmitter.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronJarSubmitter.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -36,7 +36,7 @@ * @author Arinto Murdopo * */ -public class StormJarSubmitter { +public class HeronJarSubmitter { public final static String UPLOADED_JAR_LOCATION_KEY = "UploadedJarLocation"; @@ -63,8 +63,8 @@ public static void main(String[] args) throws IOException { System.out.println("Uploaded jar file location: "); System.out.println(uploadedJarLocation); - Properties props = StormSamoaUtils.getProperties(); - props.setProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY, uploadedJarLocation); + Properties props = HeronSamoaUtils.getProperties(); + props.setProperty(HeronJarSubmitter.UPLOADED_JAR_LOCATION_KEY, uploadedJarLocation); File f = new File("src/main/resources/samoa-storm-cluster.properties"); f.createNewFile(); diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormProcessingItem.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronProcessingItem.java similarity index 78% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormProcessingItem.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronProcessingItem.java index 28c37468..87bb4a21 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormProcessingItem.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronProcessingItem.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -30,7 +30,7 @@ import org.apache.samoa.topology.AbstractProcessingItem; import org.apache.samoa.topology.ProcessingItem; import org.apache.samoa.topology.Stream; -import org.apache.samoa.topology.impl.StormStream.InputStreamId; +import org.apache.samoa.heron.topology.impl.HeronStream.InputStreamId; import org.apache.samoa.utils.PartitioningScheme; import backtype.storm.task.OutputCollector; @@ -43,12 +43,12 @@ import backtype.storm.tuple.Tuple; /** - * ProcessingItem implementation for Storm. + * ProcessingItem implementation for Heron. * * @author Arinto Murdopo * */ -class StormProcessingItem extends AbstractProcessingItem implements StormTopologyNode { +public class HeronProcessingItem extends AbstractProcessingItem implements HeronTopologyNode { private final ProcessingItemBolt piBolt; private BoltDeclarer piBoltDeclarer; @@ -56,11 +56,11 @@ class StormProcessingItem extends AbstractProcessingItem implements StormTopolog // imo, parallelism hint only declared when we add this PI in the topology // open for dicussion :p - StormProcessingItem(Processor processor, int parallelismHint) { + HeronProcessingItem(Processor processor, int parallelismHint) { this(processor, UUID.randomUUID().toString(), parallelismHint); } - StormProcessingItem(Processor processor, String friendlyId, int parallelismHint) { + HeronProcessingItem(Processor processor, String friendlyId, int parallelismHint) { super(processor, parallelismHint); this.piBolt = new ProcessingItemBolt(processor); this.setName(friendlyId); @@ -68,7 +68,7 @@ class StormProcessingItem extends AbstractProcessingItem implements StormTopolog @Override protected ProcessingItem addInputStream(Stream inputStream, PartitioningScheme scheme) { - StormStream stormInputStream = (StormStream) inputStream; + HeronStream stormInputStream = (HeronStream) inputStream; InputStreamId inputId = stormInputStream.getInputId(); switch (scheme) { @@ -79,7 +79,7 @@ protected ProcessingItem addInputStream(Stream inputStream, PartitioningScheme s piBoltDeclarer.fieldsGrouping( inputId.getComponentId(), inputId.getStreamId(), - new Fields(StormSamoaUtils.KEY_FIELD)); + new Fields(HeronSamoaUtils.KEY_FIELD)); break; case BROADCAST: piBoltDeclarer.allGrouping( @@ -91,18 +91,18 @@ protected ProcessingItem addInputStream(Stream inputStream, PartitioningScheme s } @Override - public void addToTopology(StormTopology topology, int parallelismHint) { + public void addToTopology(HeronTopology topology, int parallelismHint) { if (piBoltDeclarer != null) { // throw exception that one PI only belong to one topology } else { - TopologyBuilder stormBuilder = topology.getStormBuilder(); - this.piBoltDeclarer = stormBuilder.setBolt(this.getName(), + TopologyBuilder heronBuilder = topology.getHeronBuilder(); + this.piBoltDeclarer = heronBuilder.setBolt(this.getName(), this.piBolt, parallelismHint); } } @Override - public StormStream createStream() { + public HeronStream createStream() { return piBolt.createStream(this.getName()); } @@ -122,13 +122,13 @@ private final static class ProcessingItemBolt extends BaseRichBolt { private static final long serialVersionUID = -6637673741263199198L; - private final Set streams; + private final Set streams; private final Processor processor; private OutputCollector collector; ProcessingItemBolt(Processor processor) { - this.streams = new HashSet(); + this.streams = new HashSet(); this.processor = processor; } @@ -137,7 +137,7 @@ public void prepare(@SuppressWarnings("rawtypes") Map stormConf, TopologyContext OutputCollector collector) { this.collector = collector; // Processor and this class share the same instance of stream - for (StormBoltStream stream : streams) { + for (HeronBoltStream stream : streams) { stream.setCollector(this.collector); } @@ -153,15 +153,14 @@ public void execute(Tuple input) { @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - for (StormStream stream : streams) { + for (HeronStream stream : streams) { declarer.declareStream(stream.getOutputId(), - new Fields(StormSamoaUtils.CONTENT_EVENT_FIELD, - StormSamoaUtils.KEY_FIELD)); + new Fields(HeronSamoaUtils.CONTENT_EVENT_FIELD, + HeronSamoaUtils.KEY_FIELD)); } } - StormStream createStream(String piId) { - StormBoltStream stream = new StormBoltStream(piId); + HeronStream createStream(String piId) { HeronBoltStream stream = new HeronBoltStream(piId); streams.add(stream); return stream; } diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSamoaUtils.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSamoaUtils.java similarity index 88% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSamoaUtils.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSamoaUtils.java index 7f7e5780..0781dedc 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSamoaUtils.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSamoaUtils.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -38,14 +38,14 @@ import org.apache.commons.configuration.PropertiesConfiguration; /** - * Utility class for samoa-storm project. It is used by StormDoTask to process its arguments. + * Utility class for samoa-heron project. It is used by HeronDoTask to process its arguments. * * @author Arinto Murdopo * */ -public class StormSamoaUtils { +public class HeronSamoaUtils { - private static final Logger logger = LoggerFactory.getLogger(StormSamoaUtils.class); + private static final Logger logger = LoggerFactory.getLogger(HeronSamoaUtils.class); static final String KEY_FIELD = "key"; static final String CONTENT_EVENT_FIELD = "content_event"; @@ -54,7 +54,7 @@ static Properties getProperties() throws IOException { Properties props = new Properties(); InputStream is; - File f = new File("src/main/resources/samoa-storm-cluster.properties"); // FIXME it does not exist anymore + File f = new File("src/main/resources/samoa-heron-cluster.properties"); // FIXME it does not exist anymore is = new FileInputStream(f); try { @@ -69,7 +69,7 @@ static Properties getProperties() throws IOException { return props; } - public static StormTopology argsToTopology(String[] args) { + public static HeronTopology argsToTopology(String[] args) { StringBuilder cliString = new StringBuilder(); for (String arg : args) { cliString.append(" ").append(arg); @@ -79,10 +79,10 @@ public static StormTopology argsToTopology(String[] args) { Task task = getTask(cliString.toString()); // TODO: remove setFactory method with DynamicBinding - task.setFactory(new StormComponentFactory()); + task.setFactory(new HeronComponentFactory()); task.init(); - return (StormTopology) task.getTopology(); + return (HeronTopology) task.getTopology(); } public static int numWorkers(List tmpArgs) { diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSpoutStream.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSpoutStream.java similarity index 95% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSpoutStream.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSpoutStream.java index 14fc4243..8e5cbdf2 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormSpoutStream.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSpoutStream.java @@ -1,5 +1,4 @@ -package org.apache.samoa.topology.impl; -//package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormStream.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronStream.java similarity index 92% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormStream.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronStream.java index 1c623896..6af12eb0 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormStream.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronStream.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -31,7 +31,7 @@ * @author Arinto Murdopo * */ -abstract class StormStream implements Stream, java.io.Serializable { +abstract class HeronStream implements Stream, java.io.Serializable { /** * @@ -40,7 +40,7 @@ abstract class StormStream implements Stream, java.io.Serializable { protected final String outputStreamId; protected final InputStreamId inputStreamId; - public StormStream(String stormComponentId) { + public HeronStream(String stormComponentId) { this.outputStreamId = UUID.randomUUID().toString(); this.inputStreamId = new InputStreamId(stormComponentId, this.outputStreamId); } diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopology.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopology.java similarity index 79% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopology.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopology.java index 51bb9097..d5923cbd 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopology.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopology.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -26,28 +26,28 @@ import backtype.storm.topology.TopologyBuilder; /** - * Adaptation of SAMOA topology in samoa-storm + * Adaptation of SAMOA topology in samoa-heron * * @author Arinto Murdopo * */ -public class StormTopology extends AbstractTopology { +public class HeronTopology extends AbstractTopology { private TopologyBuilder builder; - public StormTopology(String topologyName) { + public HeronTopology(String topologyName) { super(topologyName); this.builder = new TopologyBuilder(); } @Override public void addProcessingItem(IProcessingItem procItem, int parallelismHint) { - StormTopologyNode stormNode = (StormTopologyNode) procItem; + HeronTopologyNode stormNode = (HeronTopologyNode) procItem; stormNode.addToTopology(this, parallelismHint); super.addProcessingItem(procItem, parallelismHint); } - public TopologyBuilder getStormBuilder() { + public TopologyBuilder getHeronBuilder() { return builder; } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologyNode.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologyNode.java similarity index 82% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologyNode.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologyNode.java index 994fb9fc..af8f7e96 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologyNode.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologyNode.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -26,11 +26,11 @@ * @author Arinto Murdopo * */ -interface StormTopologyNode { +interface HeronTopologyNode { - void addToTopology(StormTopology topology, int parallelismHint); + void addToTopology(HeronTopology topology, int parallelismHint); - StormStream createStream(); + HeronStream createStream(); String getId(); diff --git a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologySubmitter.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologySubmitter.java similarity index 85% rename from samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologySubmitter.java rename to samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologySubmitter.java index 25985dfd..4157498d 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/topology/impl/StormTopologySubmitter.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologySubmitter.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -47,27 +47,27 @@ * @author Arinto Murdopo * */ -public class StormTopologySubmitter { +public class HeronTopologySubmitter { public static String YJP_OPTIONS_KEY = "YjpOptions"; - private static Logger logger = LoggerFactory.getLogger(StormTopologySubmitter.class); + private static Logger logger = LoggerFactory.getLogger(HeronTopologySubmitter.class); public static void main(String[] args) throws IOException { - Properties props = StormSamoaUtils.getProperties(); + Properties props = HeronSamoaUtils.getProperties(); - String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY); + String uploadedJarLocation = props.getProperty(HeronJarSubmitter.UPLOADED_JAR_LOCATION_KEY); if (uploadedJarLocation == null) { logger.error("Invalid properties file. It must have key {}", - StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY); + HeronJarSubmitter.UPLOADED_JAR_LOCATION_KEY); return; } List tmpArgs = new ArrayList(Arrays.asList(args)); - int numWorkers = StormSamoaUtils.numWorkers(tmpArgs); + int numWorkers = HeronSamoaUtils.numWorkers(tmpArgs); args = tmpArgs.toArray(new String[0]); - StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); + HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args); Config conf = new Config(); conf.putAll(Utils.readStormConfig()); @@ -76,7 +76,7 @@ public static void main(String[] args) throws IOException { conf.setNumWorkers(numWorkers); String profilerOption = - props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY); + props.getProperty(HeronTopologySubmitter.YJP_OPTIONS_KEY); if (profilerOption != null) { String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS); StringBuilder optionBuilder = new StringBuilder(); @@ -103,12 +103,12 @@ public static void main(String[] args) throws IOException { config.putAll(Utils.readStormConfig()); NimbusClient nc = NimbusClient.getConfiguredClient(config); - String topologyName = stormTopo.getTopologyName(); + String topologyName = heronTopo.getTopologyName(); try { System.out.println("Submitting topology with name: " + topologyName); nc.getClient().submitTopology(topologyName, uploadedJarLocation, - out.toString(), stormTopo.getStormBuilder().createTopology()); + out.toString(), heronTopo.getHeronBuilder().createTopology()); System.out.println(topologyName + " is successfully submitted"); } catch (AlreadyAliveException aae) { diff --git a/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java b/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java index 85204458..becba367 100644 --- a/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java +++ b/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java @@ -1,4 +1,4 @@ -package org.apache.samoa; +package org.apache.samoa.heron; /* * #%L @@ -20,56 +20,59 @@ * #L% */ -import org.apache.samoa.LocalStormDoTask; +import org.apache.samoa.heron.LocalHeronDoTask; import org.apache.samoa.TestParams; import org.apache.samoa.TestUtils; import org.junit.Test; +import org.junit.Ignore; public class AlgosTest { @Test(timeout = 60000) - public void testVHTWithStorm() throws Exception { + public void testVHTWithHeron() throws Exception { TestParams vhtConfig = new TestParams.Builder() .inputInstances(200_000) .samplingSize(20_000) .evaluationInstances(200_000) .classifiedInstances(200_000) - .labelSamplingSize(10l) + //.labelSamplingSize(10l) .classificationsCorrect(55f) .kappaStat(-0.1f) .kappaTempStat(-0.1f) .cliStringTemplate(TestParams.Templates.PREQEVAL_VHT_RANDOMTREE) .resultFilePollTimeout(30) .prePollWait(15) - .taskClassName(LocalStormDoTask.class.getName()) + .taskClassName(LocalHeronDoTask.class.getName()) .build(); TestUtils.test(vhtConfig); } @Test(timeout = 120000) - public void testBaggingWithStorm() throws Exception { + @Ignore + public void testBaggingWithHeron() throws Exception { TestParams baggingConfig = new TestParams.Builder() .inputInstances(200_000) .samplingSize(20_000) .evaluationInstances(180_000) .classifiedInstances(190_000) - .labelSamplingSize(10l) + //.labelSamplingSize(10l) .classificationsCorrect(60f) .kappaStat(0f) .kappaTempStat(0f) .cliStringTemplate(TestParams.Templates.PREQEVAL_BAGGING_RANDOMTREE) .resultFilePollTimeout(40) .prePollWait(20) - .taskClassName(LocalStormDoTask.class.getName()) + .taskClassName(LocalHeronDoTask.class.getName()) .build(); TestUtils.test(baggingConfig); } @Test(timeout = 240000) - public void testCVPReqVHTWithStorm() throws Exception { + @Ignore + public void testCVPReqVHTWithHeron() throws Exception { TestParams vhtConfig = new TestParams.Builder() .inputInstances(200_000) @@ -82,8 +85,8 @@ public void testCVPReqVHTWithStorm() throws Exception { .cliStringTemplate(TestParams.Templates.PREQCVEVAL_VHT_RANDOMTREE) .resultFilePollTimeout(30) .prePollWait(15) - .taskClassName(LocalStormDoTask.class.getName()) - .labelFileCreated(false) + .taskClassName(LocalHeronDoTask.class.getName()) + //.labelFileCreated(false) .build(); TestUtils.test(vhtConfig); diff --git a/samoa-heron/src/test/java/org/apache/samoa/topology/impl/StormProcessingItemTest.java b/samoa-heron/src/test/java/org/apache/samoa/heron/topology/impl/HeronProcessingItemTest.java similarity index 78% rename from samoa-heron/src/test/java/org/apache/samoa/topology/impl/StormProcessingItemTest.java rename to samoa-heron/src/test/java/org/apache/samoa/heron/topology/impl/HeronProcessingItemTest.java index 46739036..8e3978e2 100644 --- a/samoa-heron/src/test/java/org/apache/samoa/topology/impl/StormProcessingItemTest.java +++ b/samoa-heron/src/test/java/org/apache/samoa/heron/topology/impl/HeronProcessingItemTest.java @@ -1,4 +1,4 @@ -package org.apache.samoa.topology.impl; +package org.apache.samoa.heron.topology.impl; /* * #%L @@ -28,42 +28,42 @@ import mockit.Verifications; import org.apache.samoa.core.Processor; -import org.apache.samoa.topology.impl.StormProcessingItem; -import org.apache.samoa.topology.impl.StormTopology; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.apache.samoa.heron.topology.impl.HeronProcessingItem; + import backtype.storm.topology.BoltDeclarer; import backtype.storm.topology.IRichBolt; import backtype.storm.topology.TopologyBuilder; -public class StormProcessingItemTest { +public class HeronProcessingItemTest { private static final int PARRALLELISM_HINT_2 = 2; private static final int PARRALLELISM_HINT_4 = 4; private static final String ID = "id"; @Tested - private StormProcessingItem pi; + private HeronProcessingItem pi; @Mocked private Processor processor; @Mocked - private StormTopology topology; + private HeronTopology topology; @Mocked - private TopologyBuilder stormBuilder = new TopologyBuilder(); + private TopologyBuilder heronBuilder = new TopologyBuilder(); @Before public void setUp() { - pi = new StormProcessingItem(processor, ID, PARRALLELISM_HINT_2); + pi = new HeronProcessingItem(processor, ID, PARRALLELISM_HINT_2); } @Test public void testAddToTopology() { new Expectations() { { - topology.getStormBuilder(); - result = stormBuilder; + topology.getHeronBuilder(); + result = heronBuilder; - stormBuilder.setBolt(ID, (IRichBolt) any, anyInt); + heronBuilder.setBolt(ID, (IRichBolt) any, anyInt); result = new MockUp() { }.getMockInstance(); } From 1939e73b65209ab05b19a170610633dd845889d8 Mon Sep 17 00:00:00 2001 From: Ning Wang Date: Wed, 3 Oct 2018 23:41:09 -0700 Subject: [PATCH 3/5] Update POM files for Heron --- pom.xml | 12 ++++++++++++ samoa-heron/pom.xml | 27 +++++++++++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 90d6a5f1..aa5c2aba 100644 --- a/pom.xml +++ b/pom.xml @@ -102,6 +102,15 @@ samoa-test + + heron + + samoa-instances + samoa-api + samoa-heron + samoa-test + + all @@ -114,6 +123,7 @@ samoa-flink samoa-samza samoa-test + samoa-heron @@ -144,6 +154,8 @@ 0.9.4 3.4.6 + 0.17.8 + 3.4.6 1.7.7 diff --git a/samoa-heron/pom.xml b/samoa-heron/pom.xml index b104da8a..faf37513 100644 --- a/samoa-heron/pom.xml +++ b/samoa-heron/pom.xml @@ -3,7 +3,7 @@ #%L SAMOA %% - Copyright (C) 2014 - 2015 Apache Software Foundation + Copyright (C) 2018 Apache Software Foundation %% Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -36,9 +36,16 @@ - - clojars - http://clojars.org/repo/ + + mvnrepository + Mvn repository + https://mvnrepository.com/artifact/ + + false + + + true + @@ -58,15 +65,15 @@ - org.apache.storm - storm-core - ${storm.version} + com.twitter.heron + heron-storm + ${heron.version} provided org.apache.zookeeper zookeeper - ${zookeeper.storm.version} + ${zookeeper.heron.version} provided @@ -84,7 +91,7 @@ maven-assembly-plugin ${maven-assembly-plugin.version} - SAMOA-Storm-${project.version} + SAMOA-heron-${project.version} false false ../target @@ -125,7 +132,7 @@ ${project.basedir}/../bin - *storm.properties + *heron.properties From 1efb54dcd8e78c013ebbc653f6f90ee01518d2e7 Mon Sep 17 00:00:00 2001 From: Saikat Kanjilal Date: Tue, 16 Oct 2018 14:47:17 -0700 Subject: [PATCH 4/5] incorporated pom file changes in upper level project and samoa-heron --- pom.xml | 20 +- .../java/org/apache/samoa/moa/core/Vote.java | 20 ++ samoa-heron/pom.xml | 231 +++++++------ .../org/apache/samoa/LocalHeronDoTask.java | 99 +++--- .../heron/topology/impl/HeronBoltStream.java | 61 ++-- .../topology/impl/HeronComponentFactory.java | 94 +++--- .../heron/topology/impl/HeronDoTask.java | 156 ++++----- .../impl/HeronEntranceProcessingItem.java | 316 +++++++++--------- .../topology/impl/HeronJarSubmitter.java | 47 ++- .../topology/impl/HeronProcessingItem.java | 222 ++++++------ .../heron/topology/impl/HeronSamoaUtils.java | 139 ++++---- .../heron/topology/impl/HeronSpoutStream.java | 4 +- .../heron/topology/impl/HeronStream.java | 85 +++-- .../heron/topology/impl/HeronTopology.java | 38 +-- .../topology/impl/HeronTopologyNode.java | 13 +- .../topology/impl/HeronTopologySubmitter.java | 165 +++++---- .../test/java/org/apache/samoa/AlgosTest.java | 118 +++---- .../impl/HeronProcessingItemTest.java | 77 ++--- 18 files changed, 972 insertions(+), 933 deletions(-) diff --git a/pom.xml b/pom.xml index 90d6a5f1..172b2c7f 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,8 @@ #L% --> - + 4.0.0 Apache SAMOA @@ -75,6 +76,15 @@ samoa-test + + heron + + samoa-instances + samoa-api + samoa-heron + samoa-test + + apex @@ -114,6 +124,7 @@ samoa-flink samoa-samza samoa-test + samoa-heron @@ -144,6 +155,8 @@ 0.9.4 3.4.6 + 0.17.8 + 3.4.6 1.7.7 @@ -198,7 +211,8 @@ true
Apache SAMOA ${project.version}
Scalable Advanced Massive Online Analysis, - ${project.version}
+ ${project.version} + Apache SAMOA API ${project.version} http://samoa.incubator.apache.org/docs/api/ @@ -270,7 +284,7 @@ - + diff --git a/samoa-api/src/main/java/org/apache/samoa/moa/core/Vote.java b/samoa-api/src/main/java/org/apache/samoa/moa/core/Vote.java index 24ea3f31..c1b5541d 100644 --- a/samoa-api/src/main/java/org/apache/samoa/moa/core/Vote.java +++ b/samoa-api/src/main/java/org/apache/samoa/moa/core/Vote.java @@ -1,5 +1,25 @@ package org.apache.samoa.moa.core; +/* + * #%L + * SAMOA + * %% + * Copyright (C) 2014 - 2018 Apache Software Foundation + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + import java.io.Serializable; /* diff --git a/samoa-heron/pom.xml b/samoa-heron/pom.xml index b104da8a..21c85122 100644 --- a/samoa-heron/pom.xml +++ b/samoa-heron/pom.xml @@ -3,7 +3,7 @@ #%L SAMOA %% - Copyright (C) 2014 - 2015 Apache Software Foundation + Copyright (C) 2018 Apache Software Foundation %% Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,115 +19,132 @@ #L% --> - - 4.0.0 - - UTF-8 - + + 4.0.0 + + UTF-8 + - samoa-heron - Heron bindings for SAMOA + samoa-heron + Heron bindings for SAMOA - samoa-heron - - org.apache.samoa - samoa - 0.5.0-incubating-SNAPSHOT - + samoa-heron + + org.apache.samoa + samoa + 0.5.0-incubating-SNAPSHOT + - - - clojars - http://clojars.org/repo/ - - + + + mvnrepository + Maven repository + https://mvnrepository.com/artifact/ + + false + + + true + + + - - - org.apache.samoa - samoa-api - ${project.version} - - - org.apache.samoa - samoa-test - test-jar - test-jar-with-dependencies - ${project.version} - test - + + + org.apache.samoa + samoa-api + ${project.version} + + + org.apache.samoa + samoa-test + test-jar + test-jar-with-dependencies + ${project.version} + test + + + com.twitter.heron + heron-storm + ${heron.version} + provided + + + org.apache.zookeeper + zookeeper + ${zookeeper.heron.version} + provided + + + org.slf4j + slf4j-log4j12 + ${slf4j-log4j12.version} + test + + + com.googlecode.json-simple + json-simple + 1.1 + + + org.apache.thrift + libthrift + 0.11.0 + + - - org.apache.storm - storm-core - ${storm.version} - provided - - - org.apache.zookeeper - zookeeper - ${zookeeper.storm.version} - provided - - - org.slf4j - slf4j-log4j12 - ${slf4j-log4j12.version} - test - - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - SAMOA-Storm-${project.version} - false - false - ../target - - jar-with-dependencies - - - - ${parsedVersion.osgiVersion} - ${project.description} - ${project.version} - Yahoo Labs - SAMOA - - - - - - make-assembly - package - - single - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx1G - false - - - - - - ${project.basedir}/../bin - - *storm.properties - - - - + + + + + maven-assembly-plugin + ${maven-assembly-plugin.version} + + SAMOA-heron-${project.version} + false + false + ../target + + jar-with-dependencies + + + + ${parsedVersion.osgiVersion} + ${project.description} + ${project.version} + Yahoo Labs + SAMOA + + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + -Xmx1G + false + + + + + + ${project.basedir}/../bin + + *heron.properties + + + + diff --git a/samoa-heron/src/main/java/org/apache/samoa/LocalHeronDoTask.java b/samoa-heron/src/main/java/org/apache/samoa/LocalHeronDoTask.java index b4ccccbf..3dbe19dd 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/LocalHeronDoTask.java +++ b/samoa-heron/src/main/java/org/apache/samoa/LocalHeronDoTask.java @@ -1,5 +1,6 @@ -package org.apache.samoa.heron; - /* #%L +package org.apache.samoa.heron.topology; +/* + * #%L * SAMOA * %% * Copyright (C) 2014 - 2015 Apache Software Foundation @@ -7,9 +8,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +18,7 @@ * limitations under the License. * #L% */ + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -26,56 +28,57 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.configuration.Configuration; +import org.apache.storm.Config; +import org.apache.storm.utils.Utils; +import org.apache.storm.LocalCluster; +import org.apache.storm.generated.AlreadyAliveException; +import org.apache.storm.generated.InvalidTopologyException; +import org.apache.storm.generated.NotAliveException; -import backtype.storm.Config; -import backtype.storm.utils.Utils; /** * The main class to execute a SAMOA task in LOCAL mode in Heron. - * + * * @author Arinto Murdopo - * */ public class LocalHeronDoTask { + private static final Logger logger = LoggerFactory.getLogger(LocalHeronDoTask.class); + private static final String EXECUTION_DURATION_KEY = "samoa.storm.local.mode.execution.duration"; + private static final String SAMOA_STORM_PROPERTY_FILE_LOC = "samoa-heron.properties"; - private static final Logger logger = LoggerFactory.getLogger(LocalHeronDoTask.class); - private static final String EXECUTION_DURATION_KEY ="samoa.storm.local.mode.execution.duration"; - private static final String SAMOA_STORM_PROPERTY_FILE_LOC ="samoa-heron.properties"; - /** - * The main method. - * - * @param args - * the arguments - */ - public static void main(String[] args) { - - List tmpArgs = new ArrayList(Arrays.asList(args)); - - int numWorker = HeronSamoaUtils.numWorkers(tmpArgs); - - args = tmpArgs.toArray(new String[0]); - - // convert the arguments into Heron topology - HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args); - String topologyName = heronTopo.getTopologyName(); - - Config conf = new Config(); - // conf.putAll(Utils.readStormConfig()); - conf.setDebug(false); - - // local mode - conf.setMaxTaskParallelism(numWorker); - - backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster(); - cluster.submitTopology(topologyName, conf, heronTopo.getHeronBuilder().createTopology()); - - // Read local mode execution duration from property file - Configuration stormConfig = HeronSamoaUtils.getPropertyConfig(LocalHeronDoTask.SAMOA_STORM_PROPERTY_FILE_LOC); - long executionDuration= stormConfig.getLong(LocalHeronDoTask.EXECUTION_DURATION_KEY); - backtype.storm.utils.Utils.sleep(executionDuration * 1000); - - cluster.killTopology(topologyName); - cluster.shutdown(); + /** + * The main method. + * + * @param args the arguments + */ + public static void main(String[] args) { + List tmpArgs = new ArrayList(Arrays.asList(args)); + int numWorker = HeronSamoaUtils.numWorkers(tmpArgs); + args = tmpArgs.toArray(new String[0]); + // convert the arguments into Storm topology + HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args); + String topologyName = heronTopo.getTopologyName(); + Config conf = new Config(); + // conf.putAll(Utils.readStormConfig()); + conf.setDebug(false); + // local mode + conf.setMaxTaskParallelism(numWorker); + LocalCluster cluster = new LocalCluster(); + try { + cluster.submitTopology(topologyName, conf, heronTopo.getHeronBuilder().createTopology()); + // Read local mode execution duration from property file + Configuration heronConfig = HeronSamoaUtils.getPropertyConfig(LocalHeronDoTask.SAMOA_STORM_PROPERTY_FILE_LOC); + long executionDuration = heronConfig.getLong(LocalHeronDoTask.EXECUTION_DURATION_KEY); + backtype.storm.utils.Utils.sleep(executionDuration * 1000); + cluster.killTopology(topologyName); + cluster.shutdown(); + } catch (AlreadyAliveException aae) { + aae.printStackTrace(); + } catch (InvalidTopologyException ite) { + ite.printStackTrace(); + } catch (NotAliveException nae) { + nae.printStackTrace(); + } - } -} + } +} \ No newline at end of file diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronBoltStream.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronBoltStream.java index aaf9a760..5d135034 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronBoltStream.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronBoltStream.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,46 +22,45 @@ import org.apache.samoa.core.ContentEvent; -import backtype.storm.task.OutputCollector; -import backtype.storm.tuple.Values; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.tuple.Values; /** * Storm Stream that connects into Bolt. It wraps Storm's outputCollector class - * + * * @author Arinto Murdopo - * */ class HeronBoltStream extends HeronStream { - /** - * - */ - private static final long serialVersionUID = -5712513402991550847L; + /** + * + */ + private static final long serialVersionUID = -5712513402991550847L; - private OutputCollector outputCollector; + private OutputCollector outputCollector; - HeronBoltStream(String stormComponentId) { - super(stormComponentId); - } + HeronBoltStream(String stormComponentId) { + super(stormComponentId); + } - @Override - public void put(ContentEvent contentEvent) { - outputCollector.emit(this.outputStreamId, new Values(contentEvent, contentEvent.getKey())); - } + @Override + public void put(ContentEvent contentEvent) { + outputCollector.emit(this.outputStreamId, new Values(contentEvent, contentEvent.getKey())); + } - public void setCollector(OutputCollector outputCollector) { - this.outputCollector = outputCollector; - } + public void setCollector(OutputCollector outputCollector) { + this.outputCollector = outputCollector; + } - // @Override - // public void setStreamId(String streamId) { - // // TODO Auto-generated method stub - // //this.outputStreamId = streamId; - // } + // @Override + // public void setStreamId(String streamId) { + // // TODO Auto-generated method stub + // //this.outputStreamId = streamId; + // } - @Override - public String getStreamId() { - // TODO Auto-generated method stub - return null; - } + @Override + public String getStreamId() { + // TODO Auto-generated method stub + return null; + } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronComponentFactory.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronComponentFactory.java index 968ef4d6..bf457480 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronComponentFactory.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronComponentFactory.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,54 +37,54 @@ */ public final class HeronComponentFactory implements ComponentFactory { - private final Map processorList; - - public HeronComponentFactory() { - processorList = new HashMap<>(); - } - - @Override - public ProcessingItem createPi(Processor processor) { - return new HeronProcessingItem(processor, this.getComponentName(processor.getClass()), 1); - } - - @Override - public EntranceProcessingItem createEntrancePi(EntranceProcessor processor) { - return new HeronEntranceProcessingItem(processor, this.getComponentName(processor.getClass())); - } - - @Override - public Stream createStream(IProcessingItem sourcePi) { - HeronTopologyNode stormCompatiblePi = (HeronTopologyNode) sourcePi; - return stormCompatiblePi.createStream(); - } - - @Override - public Topology createTopology(String topoName) { - return new HeronTopology(topoName); - } - - private String getComponentName(Class clazz) { - StringBuilder componentName = new StringBuilder(clazz.getCanonicalName()); - String key = componentName.toString(); - Integer index; - - if (!processorList.containsKey(key)) { - index = 1; - } else { - index = processorList.get(key) + 1; + private final Map processorList; + + public HeronComponentFactory() { + processorList = new HashMap<>(); + } + + @Override + public ProcessingItem createPi(Processor processor) { + return new HeronProcessingItem(processor, this.getComponentName(processor.getClass()), 1); } - processorList.put(key, index); + @Override + public EntranceProcessingItem createEntrancePi(EntranceProcessor processor) { + return new HeronEntranceProcessingItem(processor, this.getComponentName(processor.getClass())); + } - componentName.append('_'); - componentName.append(index); + @Override + public Stream createStream(IProcessingItem sourcePi) { + HeronTopologyNode stormCompatiblePi = (HeronTopologyNode) sourcePi; + return stormCompatiblePi.createStream(); + } + + @Override + public Topology createTopology(String topoName) { + return new HeronTopology(topoName); + } - return componentName.toString(); - } + private String getComponentName(Class clazz) { + StringBuilder componentName = new StringBuilder(clazz.getCanonicalName()); + String key = componentName.toString(); + Integer index; - @Override - public ProcessingItem createPi(Processor processor, int parallelism) { - return new HeronProcessingItem(processor, this.getComponentName(processor.getClass()), parallelism); - } + if (!processorList.containsKey(key)) { + index = 1; + } else { + index = processorList.get(key) + 1; + } + + processorList.put(key, index); + + componentName.append('_'); + componentName.append(index); + + return componentName.toString(); + } + + @Override + public ProcessingItem createPi(Processor processor, int parallelism) { + return new HeronProcessingItem(processor, this.getComponentName(processor.getClass()), parallelism); + } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronDoTask.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronDoTask.java index 2fe74820..9e5e28e6 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronDoTask.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronDoTask.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,92 +27,96 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import backtype.storm.Config; -import backtype.storm.utils.Utils; +import org.apache.storm.Config; +import org.apache.storm.utils.Utils; +import org.apache.storm.StormSubmitter; +import org.apache.storm.LocalCluster; +import org.apache.storm.generated.AlreadyAliveException; +import org.apache.storm.generated.InvalidTopologyException; +import org.apache.storm.generated.NotAliveException; /** * The main class that used by samoa script to execute SAMOA task. - * + * * @author Arinto Murdopo - * */ public class HeronDoTask { - private static final Logger logger = LoggerFactory.getLogger(HeronDoTask.class); - private static String localFlag = "local"; - private static String clusterFlag = "cluster"; - - /** - * The main method. - * - * @param args - * the arguments - */ - public static void main(String[] args) { - - List tmpArgs = new ArrayList(Arrays.asList(args)); - - boolean isLocal = isLocal(tmpArgs); - int numWorker = HeronSamoaUtils.numWorkers(tmpArgs); - - args = tmpArgs.toArray(new String[0]); - - // convert the arguments into Storm topology - HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args); - String topologyName = heronTopo.getTopologyName(); - - Config conf = new Config(); - conf.putAll(Utils.readStormConfig()); - conf.setDebug(false); - - if (isLocal) { - // local mode - conf.setMaxTaskParallelism(numWorker); - - backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster(); - cluster.submitTopology(topologyName, conf, heronTopo.getHeronBuilder().createTopology()); - - backtype.storm.utils.Utils.sleep(600 * 1000); - - cluster.killTopology(topologyName); - cluster.shutdown(); - - } else { - // cluster mode - conf.setNumWorkers(numWorker); - try { - backtype.storm.StormSubmitter.submitTopology(topologyName, conf, - heronTopo.getHeronBuilder().createTopology()); - } catch (backtype.storm.generated.AlreadyAliveException ale) { - ale.printStackTrace(); - } catch (backtype.storm.generated.InvalidTopologyException ite) { - ite.printStackTrace(); - } + private static final Logger logger = LoggerFactory.getLogger(HeronDoTask.class); + private static String localFlag = "local"; + private static String clusterFlag = "cluster"; + + /** + * The main method. + * + * @param args the arguments + */ + public static void main(String[] args) throws AlreadyAliveException, NotAliveException { + + List tmpArgs = new ArrayList(Arrays.asList(args)); + + boolean isLocal = isLocal(tmpArgs); + int numWorker = HeronSamoaUtils.numWorkers(tmpArgs); + + args = tmpArgs.toArray(new String[0]); + + // convert the arguments into Storm topology + HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args); + String topologyName = heronTopo.getTopologyName(); + + Config conf = new Config(); + //conf.putAll(Utils.readStormConfig()); + conf.setDebug(false); + + try { + if (isLocal) { + // local mode + conf.setMaxTaskParallelism(numWorker); + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology(topologyName, conf, heronTopo.getHeronBuilder().createTopology()); + backtype.storm.utils.Utils.sleep(600 * 1000); + cluster.killTopology(topologyName); + cluster.shutdown(); + } else { + // cluster mode + conf.setNumWorkers(numWorker); + StormSubmitter.submitTopology(topologyName, conf, + heronTopo.getHeronBuilder().createTopology()); + + } + } catch (AlreadyAliveException aae) { + aae.printStackTrace(); + } catch (InvalidTopologyException ite) { + ite.printStackTrace(); + } catch (NotAliveException nae) { + nae.printStackTrace(); + } } - } - private static boolean isLocal(List tmpArgs) { - ExecutionMode executionMode = ExecutionMode.UNDETERMINED; + private static boolean isLocal(List tmpArgs) { + ExecutionMode executionMode = ExecutionMode.UNDETERMINED; - int position = tmpArgs.size() - 1; - String flag = tmpArgs.get(position); - boolean isLocal = true; + int position = tmpArgs.size() - 1; + String flag = tmpArgs.get(position); + boolean isLocal = true; - if (flag.equals(clusterFlag)) { - executionMode = ExecutionMode.CLUSTER; - isLocal = false; - } else if (flag.equals(localFlag)) { - executionMode = ExecutionMode.LOCAL; - isLocal = true; - } + if (flag.equals(clusterFlag)) { + executionMode = ExecutionMode.CLUSTER; + isLocal = false; + } else if (flag.equals(localFlag)) { + executionMode = ExecutionMode.LOCAL; + isLocal = true; + } + + if (executionMode != ExecutionMode.UNDETERMINED) { + tmpArgs.remove(position); + } - if (executionMode != ExecutionMode.UNDETERMINED) { - tmpArgs.remove(position); + return isLocal; } - return isLocal; - } + private enum ExecutionMode { + LOCAL, CLUSTER, UNDETERMINED + } - private enum ExecutionMode { - LOCAL, CLUSTER, UNDETERMINED - }; + ; } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronEntranceProcessingItem.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronEntranceProcessingItem.java index c4ae6f35..6a2a8460 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronEntranceProcessingItem.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronEntranceProcessingItem.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,184 +29,184 @@ import org.apache.samoa.topology.EntranceProcessingItem; import org.apache.samoa.topology.Stream; -import backtype.storm.spout.SpoutOutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichSpout; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Values; -import backtype.storm.utils.Utils; +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichSpout; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; +import org.apache.storm.utils.Utils; /** * EntranceProcessingItem implementation for Heron. */ class HeronEntranceProcessingItem extends AbstractEntranceProcessingItem implements HeronTopologyNode { - private final HeronEntranceSpout piSpout; - - HeronEntranceProcessingItem(EntranceProcessor processor) { - this(processor, UUID.randomUUID().toString()); - } - - HeronEntranceProcessingItem(EntranceProcessor processor, String friendlyId) { - super(processor); - this.setName(friendlyId); - this.piSpout = new HeronEntranceSpout(processor); - } - - @Override - public EntranceProcessingItem setOutputStream(Stream stream) { - // piSpout.streams.add(stream); - piSpout.setOutputStream((HeronStream) stream); - return this; - } - - @Override - public Stream getOutputStream() { - return piSpout.getOutputStream(); - } - - @Override - public void addToTopology(HeronTopology topology, int parallelismHint) { - topology.getHeronBuilder().setSpout(this.getName(), piSpout, parallelismHint); - } - - @Override - public HeronStream createStream() { - return piSpout.createStream(this.getName()); - } - - @Override - public String getId() { - return this.getName(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(super.toString()); - sb.insert(0, String.format("id: %s, ", this.getName())); - return sb.toString(); - } - - /** - * Resulting Spout of StormEntranceProcessingItem - */ - final static class HeronEntranceSpout extends BaseRichSpout { - - private static final long serialVersionUID = -9066409791668954099L; - - // private final Set streams; - private final EntranceProcessor entranceProcessor; - private HeronStream outputStream; - - // private transient SpoutStarter spoutStarter; - // private transient Executor spoutExecutors; - // private transient LinkedBlockingQueue tupleInfoQueue; - - private SpoutOutputCollector collector; - - HeronEntranceSpout(EntranceProcessor processor) { - // this.streams = new HashSet(); - this.entranceProcessor = processor; - } + private final HeronEntranceSpout piSpout; - public HeronStream getOutputStream() { - return outputStream; + HeronEntranceProcessingItem(EntranceProcessor processor) { + this(processor, UUID.randomUUID().toString()); } - public void setOutputStream(HeronStream stream) { - this.outputStream = stream; + HeronEntranceProcessingItem(EntranceProcessor processor, String friendlyId) { + super(processor); + this.setName(friendlyId); + this.piSpout = new HeronEntranceSpout(processor); } @Override - public void open(@SuppressWarnings("rawtypes") Map conf, TopologyContext context, SpoutOutputCollector collector) { - this.collector = collector; - // this.tupleInfoQueue = new LinkedBlockingQueue(); - - // Processor and this class share the same instance of stream - // for (StormSpoutStream stream : streams) { - // stream.setSpout(this); - // } - // outputStream.setSpout(this); - - this.entranceProcessor.onCreate(context.getThisTaskId()); - // this.spoutStarter = new SpoutStarter(this.starter); - - // this.spoutExecutors = Executors.newSingleThreadExecutor(); - // this.spoutExecutors.execute(spoutStarter); + public EntranceProcessingItem setOutputStream(Stream stream) { + // piSpout.streams.add(stream); + piSpout.setOutputStream((HeronStream) stream); + return this; } @Override - public void nextTuple() { - if (entranceProcessor.hasNext()) { - Values value = newValues(entranceProcessor.nextEvent()); - collector.emit(outputStream.getOutputId(), value); - } else - Utils.sleep(1000); - // StormTupleInfo tupleInfo = tupleInfoQueue.poll(50, - // TimeUnit.MILLISECONDS); - // if (tupleInfo != null) { - // Values value = newValues(tupleInfo.getContentEvent()); - // collector.emit(tupleInfo.getHeronStream().getOutputId(), value); - // } + public Stream getOutputStream() { + return piSpout.getOutputStream(); } @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - // for (HeronStream stream : streams) { - // declarer.declareStream(stream.getOutputId(), new - // Fields(StormSamoaUtils.CONTENT_EVENT_FIELD, - // StormSamoaUtils.KEY_FIELD)); - // } - declarer.declareStream(outputStream.getOutputId(), new Fields(HeronSamoaUtils.CONTENT_EVENT_FIELD, - HeronSamoaUtils.KEY_FIELD)); + public void addToTopology(HeronTopology topology, int parallelismHint) { + topology.getHeronBuilder().setSpout(this.getName(), piSpout, parallelismHint); } - HeronStream createStream(String piId) { - // StormSpoutStream stream = new StormSpoutStream(piId); - HeronStream stream = new HeronBoltStream(piId); - // streams.add(stream); - return stream; + @Override + public HeronStream createStream() { + return piSpout.createStream(this.getName()); } - // void put(StormSpoutStream stream, ContentEvent contentEvent) { - // tupleInfoQueue.add(new StormTupleInfo(stream, contentEvent)); - // } + @Override + public String getId() { + return this.getName(); + } - private Values newValues(ContentEvent contentEvent) { - return new Values(contentEvent, contentEvent.getKey()); + @Override + public String toString() { + StringBuilder sb = new StringBuilder(super.toString()); + sb.insert(0, String.format("id: %s, ", this.getName())); + return sb.toString(); } - // private final static class StormTupleInfo { - // - // private final HeronStream stream; - // private final ContentEvent event; - // - // StormTupleInfo(HeronStream stream, ContentEvent event) { - // this.stream = stream; - // this.event = event; - // } - // - // public HeronStream getHeronStream() { - // return this.stream; - // } - // - // public ContentEvent getContentEvent() { - // return this.event; - // } - // } - - // private final static class SpoutStarter implements Runnable { - // - // private final TopologyStarter topoStarter; - // - // SpoutStarter(TopologyStarter topoStarter) { - // this.topoStarter = topoStarter; - // } - // - // @Override - // public void run() { - // this.topoStarter.start(); - // } - // } - } + /** + * Resulting Spout of StormEntranceProcessingItem + */ + final static class HeronEntranceSpout extends BaseRichSpout { + + private static final long serialVersionUID = -9066409791668954099L; + + // private final Set streams; + private final EntranceProcessor entranceProcessor; + private HeronStream outputStream; + + // private transient SpoutStarter spoutStarter; + // private transient Executor spoutExecutors; + // private transient LinkedBlockingQueue tupleInfoQueue; + + private SpoutOutputCollector collector; + + HeronEntranceSpout(EntranceProcessor processor) { + // this.streams = new HashSet(); + this.entranceProcessor = processor; + } + + public HeronStream getOutputStream() { + return outputStream; + } + + public void setOutputStream(HeronStream stream) { + this.outputStream = stream; + } + + @Override + public void open(@SuppressWarnings("rawtypes") Map conf, TopologyContext context, SpoutOutputCollector collector) { + this.collector = collector; + // this.tupleInfoQueue = new LinkedBlockingQueue(); + + // Processor and this class share the same instance of stream + // for (StormSpoutStream stream : streams) { + // stream.setSpout(this); + // } + // outputStream.setSpout(this); + + this.entranceProcessor.onCreate(context.getThisTaskId()); + // this.spoutStarter = new SpoutStarter(this.starter); + + // this.spoutExecutors = Executors.newSingleThreadExecutor(); + // this.spoutExecutors.execute(spoutStarter); + } + + @Override + public void nextTuple() { + if (entranceProcessor.hasNext()) { + Values value = newValues(entranceProcessor.nextEvent()); + collector.emit(outputStream.getOutputId(), value); + } else + Utils.sleep(1000); + // StormTupleInfo tupleInfo = tupleInfoQueue.poll(50, + // TimeUnit.MILLISECONDS); + // if (tupleInfo != null) { + // Values value = newValues(tupleInfo.getContentEvent()); + // collector.emit(tupleInfo.getHeronStream().getOutputId(), value); + // } + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + // for (HeronStream stream : streams) { + // declarer.declareStream(stream.getOutputId(), new + // Fields(StormSamoaUtils.CONTENT_EVENT_FIELD, + // StormSamoaUtils.KEY_FIELD)); + // } + declarer.declareStream(outputStream.getOutputId(), new Fields(HeronSamoaUtils.CONTENT_EVENT_FIELD, + HeronSamoaUtils.KEY_FIELD)); + } + + HeronStream createStream(String piId) { + // StormSpoutStream stream = new StormSpoutStream(piId); + HeronStream stream = new HeronBoltStream(piId); + // streams.add(stream); + return stream; + } + + // void put(StormSpoutStream stream, ContentEvent contentEvent) { + // tupleInfoQueue.add(new StormTupleInfo(stream, contentEvent)); + // } + + private Values newValues(ContentEvent contentEvent) { + return new Values(contentEvent, contentEvent.getKey()); + } + + // private final static class StormTupleInfo { + // + // private final HeronStream stream; + // private final ContentEvent event; + // + // StormTupleInfo(HeronStream stream, ContentEvent event) { + // this.stream = stream; + // this.event = event; + // } + // + // public HeronStream getHeronStream() { + // return this.stream; + // } + // + // public ContentEvent getContentEvent() { + // return this.event; + // } + // } + + // private final static class SpoutStarter implements Runnable { + // + // private final TopologyStarter topoStarter; + // + // SpoutStarter(TopologyStarter topoStarter) { + // this.topoStarter = topoStarter; + // } + // + // @Override + // public void run() { + // this.topoStarter.start(); + // } + // } + } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronJarSubmitter.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronJarSubmitter.java index aa5197d1..a5ff51c4 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronJarSubmitter.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronJarSubmitter.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,37 +25,34 @@ import java.io.IOException; import java.io.OutputStream; import java.util.Properties; +import java.util.Map; +import java.util.HashMap; -import backtype.storm.Config; -import backtype.storm.StormSubmitter; -import backtype.storm.utils.Utils; +import org.apache.storm.Config; +import org.apache.storm.StormSubmitter; +import org.apache.storm.utils.Utils; /** - * Utility class to submit samoa-storm jar to a Storm cluster. - * + * Utility class to submit samoa-storm jar to a Heron cluster. + * * @author Arinto Murdopo - * */ public class HeronJarSubmitter { - public final static String UPLOADED_JAR_LOCATION_KEY = "UploadedJarLocation"; + public final static String UPLOADED_JAR_LOCATION_KEY = "UploadedJarLocation"; - /** - * @param args - * @throws IOException - */ - public static void main(String[] args) throws IOException { + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { - Config config = new Config(); - config.putAll(Utils.readCommandLineOpts()); - config.putAll(Utils.readStormConfig()); - String nimbusHost = (String) config.get(Config.NIMBUS_HOST); - int nimbusThriftPort = Utils.getInt(config - .get(Config.NIMBUS_THRIFT_PORT)); + //TODO refactor this whole code to use submitTopology instead + //as submitJar is not longer available in heron-storm + /*Map config = Utils.readStormConfig(); + //config.putAll(Utils.readCommandLineOpts()); - System.out.println("Nimbus host " + nimbusHost); - System.out.println("Nimbus thrift port " + nimbusThriftPort); System.out.println("uploading jar from " + args[0]); String uploadedJarLocation = StormSubmitter.submitJar(config, args[0]); @@ -66,10 +63,10 @@ public static void main(String[] args) throws IOException { Properties props = HeronSamoaUtils.getProperties(); props.setProperty(HeronJarSubmitter.UPLOADED_JAR_LOCATION_KEY, uploadedJarLocation); - File f = new File("src/main/resources/samoa-storm-cluster.properties"); + File f = new File("src/main/resources/samoa-heron-cluster.properties"); f.createNewFile(); OutputStream out = new FileOutputStream(f); - props.store(out, "properties file to store uploaded jar location from StormJarSubmitter"); - } + props.store(out, "properties file to store uploaded jar location from HeronJarSubmitter");*/ + } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronProcessingItem.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronProcessingItem.java index 87bb4a21..bf01f108 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronProcessingItem.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronProcessingItem.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,136 +33,136 @@ import org.apache.samoa.heron.topology.impl.HeronStream.InputStreamId; import org.apache.samoa.utils.PartitioningScheme; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.BoltDeclarer; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.TopologyBuilder; -import backtype.storm.topology.base.BaseRichBolt; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BoltDeclarer; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.topology.base.BaseRichBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; /** * ProcessingItem implementation for Heron. - * + * * @author Arinto Murdopo - * */ public class HeronProcessingItem extends AbstractProcessingItem implements HeronTopologyNode { - private final ProcessingItemBolt piBolt; - private BoltDeclarer piBoltDeclarer; - - // TODO: should we put parallelism hint here? - // imo, parallelism hint only declared when we add this PI in the topology - // open for dicussion :p - - HeronProcessingItem(Processor processor, int parallelismHint) { - this(processor, UUID.randomUUID().toString(), parallelismHint); - } - - HeronProcessingItem(Processor processor, String friendlyId, int parallelismHint) { - super(processor, parallelismHint); - this.piBolt = new ProcessingItemBolt(processor); - this.setName(friendlyId); - } - - @Override - protected ProcessingItem addInputStream(Stream inputStream, PartitioningScheme scheme) { - HeronStream stormInputStream = (HeronStream) inputStream; - InputStreamId inputId = stormInputStream.getInputId(); - - switch (scheme) { - case SHUFFLE: - piBoltDeclarer.shuffleGrouping(inputId.getComponentId(), inputId.getStreamId()); - break; - case GROUP_BY_KEY: - piBoltDeclarer.fieldsGrouping( - inputId.getComponentId(), - inputId.getStreamId(), - new Fields(HeronSamoaUtils.KEY_FIELD)); - break; - case BROADCAST: - piBoltDeclarer.allGrouping( - inputId.getComponentId(), - inputId.getStreamId()); - break; - } - return this; - } - - @Override - public void addToTopology(HeronTopology topology, int parallelismHint) { - if (piBoltDeclarer != null) { - // throw exception that one PI only belong to one topology - } else { - TopologyBuilder heronBuilder = topology.getHeronBuilder(); - this.piBoltDeclarer = heronBuilder.setBolt(this.getName(), - this.piBolt, parallelismHint); - } - } - - @Override - public HeronStream createStream() { - return piBolt.createStream(this.getName()); - } - - @Override - public String getId() { - return this.getName(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(super.toString()); - sb.insert(0, String.format("id: %s, ", this.getName())); - return sb.toString(); - } + private final ProcessingItemBolt piBolt; + private BoltDeclarer piBoltDeclarer; - private final static class ProcessingItemBolt extends BaseRichBolt { + // TODO: should we put parallelism hint here? + // imo, parallelism hint only declared when we add this PI in the topology + // open for dicussion :p - private static final long serialVersionUID = -6637673741263199198L; + HeronProcessingItem(Processor processor, int parallelismHint) { + this(processor, UUID.randomUUID().toString(), parallelismHint); + } - private final Set streams; - private final Processor processor; + HeronProcessingItem(Processor processor, String friendlyId, int parallelismHint) { + super(processor, parallelismHint); + this.piBolt = new ProcessingItemBolt(processor); + this.setName(friendlyId); + } - private OutputCollector collector; + @Override + protected ProcessingItem addInputStream(Stream inputStream, PartitioningScheme scheme) { + HeronStream stormInputStream = (HeronStream) inputStream; + InputStreamId inputId = stormInputStream.getInputId(); + + switch (scheme) { + case SHUFFLE: + piBoltDeclarer.shuffleGrouping(inputId.getComponentId(), inputId.getStreamId()); + break; + case GROUP_BY_KEY: + piBoltDeclarer.fieldsGrouping( + inputId.getComponentId(), + inputId.getStreamId(), + new Fields(HeronSamoaUtils.KEY_FIELD)); + break; + case BROADCAST: + piBoltDeclarer.allGrouping( + inputId.getComponentId(), + inputId.getStreamId()); + break; + } + return this; + } - ProcessingItemBolt(Processor processor) { - this.streams = new HashSet(); - this.processor = processor; + @Override + public void addToTopology(HeronTopology topology, int parallelismHint) { + if (piBoltDeclarer != null) { + // throw exception that one PI only belong to one topology + } else { + TopologyBuilder heronBuilder = topology.getHeronBuilder(); + this.piBoltDeclarer = heronBuilder.setBolt(this.getName(), + this.piBolt, parallelismHint); + } } @Override - public void prepare(@SuppressWarnings("rawtypes") Map stormConf, TopologyContext context, - OutputCollector collector) { - this.collector = collector; - // Processor and this class share the same instance of stream - for (HeronBoltStream stream : streams) { - stream.setCollector(this.collector); - } - - this.processor.onCreate(context.getThisTaskId()); + public HeronStream createStream() { + return piBolt.createStream(this.getName()); } @Override - public void execute(Tuple input) { - Object sentObject = input.getValue(0); - ContentEvent sentEvent = (ContentEvent) sentObject; - processor.process(sentEvent); + public String getId() { + return this.getName(); } @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - for (HeronStream stream : streams) { - declarer.declareStream(stream.getOutputId(), - new Fields(HeronSamoaUtils.CONTENT_EVENT_FIELD, - HeronSamoaUtils.KEY_FIELD)); - } + public String toString() { + StringBuilder sb = new StringBuilder(super.toString()); + sb.insert(0, String.format("id: %s, ", this.getName())); + return sb.toString(); } - HeronStream createStream(String piId) { HeronBoltStream stream = new HeronBoltStream(piId); - streams.add(stream); - return stream; + private final static class ProcessingItemBolt extends BaseRichBolt { + + private static final long serialVersionUID = -6637673741263199198L; + + private final Set streams; + private final Processor processor; + + private OutputCollector collector; + + ProcessingItemBolt(Processor processor) { + this.streams = new HashSet(); + this.processor = processor; + } + + @Override + public void prepare(@SuppressWarnings("rawtypes") Map stormConf, TopologyContext context, + OutputCollector collector) { + this.collector = collector; + // Processor and this class share the same instance of stream + for (HeronBoltStream stream : streams) { + stream.setCollector(this.collector); + } + + this.processor.onCreate(context.getThisTaskId()); + } + + @Override + public void execute(Tuple input) { + Object sentObject = input.getValue(0); + ContentEvent sentEvent = (ContentEvent) sentObject; + processor.process(sentEvent); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + for (HeronStream stream : streams) { + declarer.declareStream(stream.getOutputId(), + new Fields(HeronSamoaUtils.CONTENT_EVENT_FIELD, + HeronSamoaUtils.KEY_FIELD)); + } + } + + HeronStream createStream(String piId) { + HeronBoltStream stream = new HeronBoltStream(piId); + streams.add(stream); + return stream; + } } - } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSamoaUtils.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSamoaUtils.java index 0781dedc..a0e1ff2f 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSamoaUtils.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSamoaUtils.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,92 +39,89 @@ /** * Utility class for samoa-heron project. It is used by HeronDoTask to process its arguments. - * + * * @author Arinto Murdopo - * */ public class HeronSamoaUtils { - private static final Logger logger = LoggerFactory.getLogger(HeronSamoaUtils.class); + private static final Logger logger = LoggerFactory.getLogger(HeronSamoaUtils.class); - static final String KEY_FIELD = "key"; - static final String CONTENT_EVENT_FIELD = "content_event"; + static final String KEY_FIELD = "key"; + static final String CONTENT_EVENT_FIELD = "content_event"; - static Properties getProperties() throws IOException { - Properties props = new Properties(); - InputStream is; + static Properties getProperties() throws IOException { + Properties props = new Properties(); + InputStream is; - File f = new File("src/main/resources/samoa-heron-cluster.properties"); // FIXME it does not exist anymore - is = new FileInputStream(f); + File f = new File("src/main/resources/samoa-heron-cluster.properties"); // FIXME it does not exist anymore + is = new FileInputStream(f); - try { - props.load(is); - } catch (IOException e1) { - System.out.println("Fail to load property file"); - return null; - } finally { - is.close(); + try { + props.load(is); + } catch (IOException e1) { + System.out.println("Fail to load property file"); + return null; + } finally { + is.close(); + } + + return props; } - return props; - } + public static HeronTopology argsToTopology(String[] args) { + StringBuilder cliString = new StringBuilder(); + for (String arg : args) { + cliString.append(" ").append(arg); + } + logger.debug("Command line string = {}", cliString.toString()); - public static HeronTopology argsToTopology(String[] args) { - StringBuilder cliString = new StringBuilder(); - for (String arg : args) { - cliString.append(" ").append(arg); - } - logger.debug("Command line string = {}", cliString.toString()); + Task task = getTask(cliString.toString()); - Task task = getTask(cliString.toString()); + // TODO: remove setFactory method with DynamicBinding + task.setFactory(new HeronComponentFactory()); + task.init(); - // TODO: remove setFactory method with DynamicBinding - task.setFactory(new HeronComponentFactory()); - task.init(); + return (HeronTopology) task.getTopology(); + } - return (HeronTopology) task.getTopology(); - } + public static int numWorkers(List tmpArgs) { + int position = tmpArgs.size() - 1; + int numWorkers; - public static int numWorkers(List tmpArgs) { - int position = tmpArgs.size() - 1; - int numWorkers; + try { + numWorkers = Integer.parseInt(tmpArgs.get(position)); + tmpArgs.remove(position); + } catch (NumberFormatException e) { + numWorkers = 4; + } + + return numWorkers; + } - try { - numWorkers = Integer.parseInt(tmpArgs.get(position)); - tmpArgs.remove(position); - } catch (NumberFormatException e) { - numWorkers = 4; + public static Task getTask(String cliString) { + Task task = null; + try { + logger.debug("Providing task [{}]", cliString); + task = ClassOption.cliStringToObject(cliString, Task.class, null); + } catch (Exception e) { + logger.warn("Fail in initializing the task!"); + e.printStackTrace(); + } + return task; } - return numWorkers; - } - - public static Task getTask(String cliString) { - Task task = null; - try { - logger.debug("Providing task [{}]", cliString); - task = ClassOption.cliStringToObject(cliString, Task.class, null); - } catch (Exception e) { - logger.warn("Fail in initializing the task!"); - e.printStackTrace(); + public static Configuration getPropertyConfig(String configPropertyPath) { + Configuration config = null; + try { + config = new PropertiesConfiguration(configPropertyPath); + if (null == config || config.isEmpty()) { + logger.error("Configuration is null or empty at file = {}", configPropertyPath); + throw new RuntimeException("Configuration is null or empty : " + configPropertyPath); + } + } catch (ConfigurationException configurationException) { + logger.error("ConfigurationException while reading property file = {}", configurationException); + throw new RuntimeException("ConfigurationException while reading property file : " + configPropertyPath); + } + return config; } - return task; - } - - public static Configuration getPropertyConfig(String configPropertyPath){ - Configuration config = null; - try { - config = new PropertiesConfiguration(configPropertyPath); - if (null == config || config.isEmpty()) { - logger.error("Configuration is null or empty at file = {}",configPropertyPath); - throw new RuntimeException("Configuration is null or empty : " + configPropertyPath); - } - } - catch(ConfigurationException configurationException) - { - logger.error("ConfigurationException while reading property file = {}",configurationException); - throw new RuntimeException("ConfigurationException while reading property file : " + configPropertyPath); - } - return config; - } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSpoutStream.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSpoutStream.java index 8e5cbdf2..e2f1e2be 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSpoutStream.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronSpoutStream.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronStream.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronStream.java index 6af12eb0..49e66845 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronStream.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronStream.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,60 +27,59 @@ /** * Abstract class to implement Storm Stream - * + * * @author Arinto Murdopo - * */ abstract class HeronStream implements Stream, java.io.Serializable { - /** - * - */ - private static final long serialVersionUID = 281835563756514852L; - protected final String outputStreamId; - protected final InputStreamId inputStreamId; + /** + * + */ + private static final long serialVersionUID = 281835563756514852L; + protected final String outputStreamId; + protected final InputStreamId inputStreamId; + + public HeronStream(String stormComponentId) { + this.outputStreamId = UUID.randomUUID().toString(); + this.inputStreamId = new InputStreamId(stormComponentId, this.outputStreamId); + } - public HeronStream(String stormComponentId) { - this.outputStreamId = UUID.randomUUID().toString(); - this.inputStreamId = new InputStreamId(stormComponentId, this.outputStreamId); - } + @Override + public abstract void put(ContentEvent contentEvent); - @Override - public abstract void put(ContentEvent contentEvent); + String getOutputId() { + return this.outputStreamId; + } - String getOutputId() { - return this.outputStreamId; - } + InputStreamId getInputId() { + return this.inputStreamId; + } - InputStreamId getInputId() { - return this.inputStreamId; - } + final static class InputStreamId implements java.io.Serializable { - final static class InputStreamId implements java.io.Serializable { + /** + * + */ + private static final long serialVersionUID = -7457995634133691295L; + private final String componentId; + private final String streamId; - /** - * - */ - private static final long serialVersionUID = -7457995634133691295L; - private final String componentId; - private final String streamId; + InputStreamId(String componentId, String streamId) { + this.componentId = componentId; + this.streamId = streamId; + } - InputStreamId(String componentId, String streamId) { - this.componentId = componentId; - this.streamId = streamId; - } + String getComponentId() { + return componentId; + } - String getComponentId() { - return componentId; + String getStreamId() { + return streamId; + } } - String getStreamId() { - return streamId; + @Override + public void setBatchSize(int batchSize) { + // Ignore batch size } - } - - @Override - public void setBatchSize(int batchSize) { - // Ignore batch size - } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopology.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopology.java index d5923cbd..2eb3aedb 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopology.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopology.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,32 +22,30 @@ import org.apache.samoa.topology.AbstractTopology; import org.apache.samoa.topology.IProcessingItem; - -import backtype.storm.topology.TopologyBuilder; +import org.apache.storm.topology.TopologyBuilder; /** * Adaptation of SAMOA topology in samoa-heron - * + * * @author Arinto Murdopo - * */ public class HeronTopology extends AbstractTopology { - private TopologyBuilder builder; + private TopologyBuilder builder; - public HeronTopology(String topologyName) { - super(topologyName); - this.builder = new TopologyBuilder(); - } + public HeronTopology(String topologyName) { + super(topologyName); + this.builder = new TopologyBuilder(); + } - @Override - public void addProcessingItem(IProcessingItem procItem, int parallelismHint) { - HeronTopologyNode stormNode = (HeronTopologyNode) procItem; - stormNode.addToTopology(this, parallelismHint); - super.addProcessingItem(procItem, parallelismHint); - } + @Override + public void addProcessingItem(IProcessingItem procItem, int parallelismHint) { + HeronTopologyNode heronNode = (HeronTopologyNode) procItem; + heronNode.addToTopology(this, parallelismHint); + super.addProcessingItem(procItem, parallelismHint); + } - public TopologyBuilder getHeronBuilder() { - return builder; - } + public TopologyBuilder getHeronBuilder() { + return builder; + } } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologyNode.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologyNode.java index af8f7e96..376c048a 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologyNode.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologyNode.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,16 +22,15 @@ /** * Interface to represent a node in samoa-storm topology. - * + * * @author Arinto Murdopo - * */ interface HeronTopologyNode { - void addToTopology(HeronTopology topology, int parallelismHint); + void addToTopology(HeronTopology topology, int parallelismHint); - HeronStream createStream(); + HeronStream createStream(); - String getId(); + String getId(); } diff --git a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologySubmitter.java b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologySubmitter.java index 4157498d..0d97ad9a 100644 --- a/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologySubmitter.java +++ b/samoa-heron/src/main/java/org/apache/samoa/heron/topology/impl/HeronTopologySubmitter.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,104 +29,95 @@ import java.util.Map; import java.util.Properties; -import org.apache.thrift7.TException; +import org.apache.thrift.TException; import org.json.simple.JSONValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import backtype.storm.Config; -import backtype.storm.generated.AlreadyAliveException; -import backtype.storm.generated.InvalidTopologyException; -import backtype.storm.utils.NimbusClient; -import backtype.storm.utils.Utils; +import org.apache.storm.StormSubmitter; +import org.apache.storm.Config; +import org.apache.storm.generated.AlreadyAliveException; +import org.apache.storm.generated.InvalidTopologyException; +import org.apache.storm.utils.Utils; /** * Helper class to submit SAMOA task into Storm without the need of submitting the jar file. The jar file must be * submitted first using StormJarSubmitter class. - * + * * @author Arinto Murdopo - * */ public class HeronTopologySubmitter { - public static String YJP_OPTIONS_KEY = "YjpOptions"; - - private static Logger logger = LoggerFactory.getLogger(HeronTopologySubmitter.class); - - public static void main(String[] args) throws IOException { - Properties props = HeronSamoaUtils.getProperties(); - - String uploadedJarLocation = props.getProperty(HeronJarSubmitter.UPLOADED_JAR_LOCATION_KEY); - if (uploadedJarLocation == null) { - logger.error("Invalid properties file. It must have key {}", - HeronJarSubmitter.UPLOADED_JAR_LOCATION_KEY); - return; - } - - List tmpArgs = new ArrayList(Arrays.asList(args)); - int numWorkers = HeronSamoaUtils.numWorkers(tmpArgs); - - args = tmpArgs.toArray(new String[0]); - HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args); - - Config conf = new Config(); - conf.putAll(Utils.readStormConfig()); - conf.putAll(Utils.readCommandLineOpts()); - conf.setDebug(false); - conf.setNumWorkers(numWorkers); - - String profilerOption = - props.getProperty(HeronTopologySubmitter.YJP_OPTIONS_KEY); - if (profilerOption != null) { - String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS); - StringBuilder optionBuilder = new StringBuilder(); - if (topoWorkerChildOpts != null) { - optionBuilder.append(topoWorkerChildOpts); - optionBuilder.append(' '); - } - optionBuilder.append(profilerOption); - conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString()); - } - - Map myConfigMap = new HashMap(conf); - StringWriter out = new StringWriter(); - - try { - JSONValue.writeJSONString(myConfigMap, out); - } catch (IOException e) { - System.out.println("Error in writing JSONString"); - e.printStackTrace(); - return; + public static String YJP_OPTIONS_KEY = "YjpOptions"; + + private static Logger logger = LoggerFactory.getLogger(HeronTopologySubmitter.class); + + public static void main(String[] args) throws IOException { + Properties props = HeronSamoaUtils.getProperties(); + + String uploadedJarLocation = props.getProperty(HeronJarSubmitter.UPLOADED_JAR_LOCATION_KEY); + if (uploadedJarLocation == null) { + logger.error("Invalid properties file. It must have key {}", + HeronJarSubmitter.UPLOADED_JAR_LOCATION_KEY); + return; + } + + List tmpArgs = new ArrayList(Arrays.asList(args)); + int numWorkers = HeronSamoaUtils.numWorkers(tmpArgs); + + args = tmpArgs.toArray(new String[0]); + HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args); + + Config conf = new Config(); + //conf.putAll(Utils.readStormConfig()); + conf.putAll(Utils.readCommandLineOpts()); + conf.setDebug(false); + conf.setNumWorkers(numWorkers); + + String profilerOption = + props.getProperty(HeronTopologySubmitter.YJP_OPTIONS_KEY); + if (profilerOption != null) { + String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS); + StringBuilder optionBuilder = new StringBuilder(); + if (topoWorkerChildOpts != null) { + optionBuilder.append(topoWorkerChildOpts); + optionBuilder.append(' '); + } + optionBuilder.append(profilerOption); + conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString()); + } + + Map myConfigMap = new HashMap(conf); + StringWriter out = new StringWriter(); + String topologyName = heronTopo.getTopologyName(); + + try { + JSONValue.writeJSONString(myConfigMap, out); + + + Config config = new Config(); + + + System.out.println("Submitting topology with name: " + + topologyName); + StormSubmitter.submitTopology(topologyName, conf, heronTopo.getHeronBuilder().createTopology()); + System.out.println(topologyName + " is successfully submitted"); + } catch (IOException e) { + System.out.println("Error in writing JSONString"); + e.printStackTrace(); + return; + } catch (AlreadyAliveException aae) { + aae.printStackTrace(); + } catch (InvalidTopologyException ite) { + System.out.println("Invalid topology for " + topologyName); + ite.printStackTrace(); + } } - Config config = new Config(); - config.putAll(Utils.readStormConfig()); - - NimbusClient nc = NimbusClient.getConfiguredClient(config); - String topologyName = heronTopo.getTopologyName(); - try { - System.out.println("Submitting topology with name: " - + topologyName); - nc.getClient().submitTopology(topologyName, uploadedJarLocation, - out.toString(), heronTopo.getHeronBuilder().createTopology()); - System.out.println(topologyName + " is successfully submitted"); - - } catch (AlreadyAliveException aae) { - System.out.println("Fail to submit " + topologyName - + "\nError message: " + aae.get_msg()); - } catch (InvalidTopologyException ite) { - System.out.println("Invalid topology for " + topologyName); - ite.printStackTrace(); - } catch (TException te) { - System.out.println("Texception for " + topologyName); - te.printStackTrace(); + private static String uploadedJarLocation(List tmpArgs) { + int position = tmpArgs.size() - 1; + String uploadedJarLocation = tmpArgs.get(position); + tmpArgs.remove(position); + return uploadedJarLocation; } - } - - private static String uploadedJarLocation(List tmpArgs) { - int position = tmpArgs.size() - 1; - String uploadedJarLocation = tmpArgs.get(position); - tmpArgs.remove(position); - return uploadedJarLocation; - } } diff --git a/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java b/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java index becba367..91b493f7 100644 --- a/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java +++ b/samoa-heron/src/test/java/org/apache/samoa/AlgosTest.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,7 +20,7 @@ * #L% */ -import org.apache.samoa.heron.LocalHeronDoTask; +import org.apache.samoa.heron.topology.LocalHeronDoTask; import org.apache.samoa.TestParams; import org.apache.samoa.TestUtils; import org.junit.Test; @@ -28,68 +28,68 @@ public class AlgosTest { - @Test(timeout = 60000) - public void testVHTWithHeron() throws Exception { + @Test(timeout = 60000) + public void testVHTWithHeron() throws Exception { - TestParams vhtConfig = new TestParams.Builder() - .inputInstances(200_000) - .samplingSize(20_000) - .evaluationInstances(200_000) - .classifiedInstances(200_000) - //.labelSamplingSize(10l) - .classificationsCorrect(55f) - .kappaStat(-0.1f) - .kappaTempStat(-0.1f) - .cliStringTemplate(TestParams.Templates.PREQEVAL_VHT_RANDOMTREE) - .resultFilePollTimeout(30) - .prePollWait(15) - .taskClassName(LocalHeronDoTask.class.getName()) - .build(); - TestUtils.test(vhtConfig); + TestParams vhtConfig = new TestParams.Builder() + .inputInstances(200_000) + .samplingSize(20_000) + .evaluationInstances(200_000) + .classifiedInstances(200_000) + .labelSamplingSize(10l) + .classificationsCorrect(55f) + .kappaStat(-0.1f) + .kappaTempStat(-0.1f) + .cliStringTemplate(TestParams.Templates.PREQEVAL_VHT_RANDOMTREE) + .resultFilePollTimeout(30) + .prePollWait(15) + .taskClassName(LocalHeronDoTask.class.getName()) + .build(); + TestUtils.test(vhtConfig); - } + } - @Test(timeout = 120000) - @Ignore - public void testBaggingWithHeron() throws Exception { - TestParams baggingConfig = new TestParams.Builder() - .inputInstances(200_000) - .samplingSize(20_000) - .evaluationInstances(180_000) - .classifiedInstances(190_000) - //.labelSamplingSize(10l) - .classificationsCorrect(60f) - .kappaStat(0f) - .kappaTempStat(0f) - .cliStringTemplate(TestParams.Templates.PREQEVAL_BAGGING_RANDOMTREE) - .resultFilePollTimeout(40) - .prePollWait(20) - .taskClassName(LocalHeronDoTask.class.getName()) - .build(); - TestUtils.test(baggingConfig); + @Test(timeout = 120000) + @Ignore + public void testBaggingWithHeron() throws Exception { + TestParams baggingConfig = new TestParams.Builder() + .inputInstances(200_000) + .samplingSize(20_000) + .evaluationInstances(180_000) + .classifiedInstances(190_000) + .labelSamplingSize(10l) + .classificationsCorrect(60f) + .kappaStat(0f) + .kappaTempStat(0f) + .cliStringTemplate(TestParams.Templates.PREQEVAL_BAGGING_RANDOMTREE) + .resultFilePollTimeout(40) + .prePollWait(20) + .taskClassName(LocalHeronDoTask.class.getName()) + .build(); + TestUtils.test(baggingConfig); - } + } - @Test(timeout = 240000) - @Ignore - public void testCVPReqVHTWithHeron() throws Exception { + @Test(timeout = 240000) + @Ignore + public void testCVPReqVHTWithHeron() throws Exception { - TestParams vhtConfig = new TestParams.Builder() - .inputInstances(200_000) - .samplingSize(20_000) - .evaluationInstances(200_000) - .classifiedInstances(200_000) - .classificationsCorrect(55f) - .kappaStat(0f) - .kappaTempStat(0f) - .cliStringTemplate(TestParams.Templates.PREQCVEVAL_VHT_RANDOMTREE) - .resultFilePollTimeout(30) - .prePollWait(15) - .taskClassName(LocalHeronDoTask.class.getName()) - //.labelFileCreated(false) - .build(); - TestUtils.test(vhtConfig); + TestParams vhtConfig = new TestParams.Builder() + .inputInstances(200_000) + .samplingSize(20_000) + .evaluationInstances(200_000) + .classifiedInstances(200_000) + .classificationsCorrect(55f) + .kappaStat(0f) + .kappaTempStat(0f) + .cliStringTemplate(TestParams.Templates.PREQCVEVAL_VHT_RANDOMTREE) + .resultFilePollTimeout(30) + .prePollWait(15) + .taskClassName(LocalHeronDoTask.class.getName()) + .labelFileCreated(false) + .build(); + TestUtils.test(vhtConfig); - } + } } diff --git a/samoa-heron/src/test/java/org/apache/samoa/heron/topology/impl/HeronProcessingItemTest.java b/samoa-heron/src/test/java/org/apache/samoa/heron/topology/impl/HeronProcessingItemTest.java index 8e3978e2..290ff151 100644 --- a/samoa-heron/src/test/java/org/apache/samoa/heron/topology/impl/HeronProcessingItemTest.java +++ b/samoa-heron/src/test/java/org/apache/samoa/heron/topology/impl/HeronProcessingItemTest.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,6 +21,7 @@ */ import static org.junit.Assert.assertEquals; + import mockit.Expectations; import mockit.MockUp; import mockit.Mocked; @@ -39,45 +40,45 @@ import backtype.storm.topology.TopologyBuilder; public class HeronProcessingItemTest { - private static final int PARRALLELISM_HINT_2 = 2; - private static final int PARRALLELISM_HINT_4 = 4; - private static final String ID = "id"; - @Tested - private HeronProcessingItem pi; - @Mocked - private Processor processor; - @Mocked - private HeronTopology topology; - @Mocked - private TopologyBuilder heronBuilder = new TopologyBuilder(); + private static final int PARRALLELISM_HINT_2 = 2; + private static final int PARRALLELISM_HINT_4 = 4; + private static final String ID = "id"; + @Tested + private HeronProcessingItem pi; + @Mocked + private Processor processor; + @Mocked + private HeronTopology topology; + @Mocked + private TopologyBuilder heronBuilder = new TopologyBuilder(); - @Before - public void setUp() { - pi = new HeronProcessingItem(processor, ID, PARRALLELISM_HINT_2); - } + @Before + public void setUp() { + pi = new HeronProcessingItem(processor, ID, PARRALLELISM_HINT_2); + } - @Test - public void testAddToTopology() { - new Expectations() { - { - topology.getHeronBuilder(); - result = heronBuilder; + @Test + public void testAddToTopology() { + new Expectations() { + { + topology.getHeronBuilder(); + result = heronBuilder; - heronBuilder.setBolt(ID, (IRichBolt) any, anyInt); - result = new MockUp() { - }.getMockInstance(); - } - }; + heronBuilder.setBolt(ID, (IRichBolt) any, anyInt); + result = new MockUp() { + }.getMockInstance(); + } + }; - pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored + pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored - new Verifications() { - { - assertEquals(pi.getProcessor(), processor); - // TODO add methods to explore a topology and verify them - assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2); - assertEquals(pi.getId(), ID); - } - }; - } + new Verifications() { + { + assertEquals(pi.getProcessor(), processor); + // TODO add methods to explore a topology and verify them + assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2); + assertEquals(pi.getId(), ID); + } + }; + } } From 5725584da367af194e63b2ba7bc2bd0f5597e914 Mon Sep 17 00:00:00 2001 From: Saikat Kanjilal Date: Thu, 25 Oct 2018 11:20:20 -0700 Subject: [PATCH 5/5] removed duplicate profiles for heron --- pom.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pom.xml b/pom.xml index 25697ae9..f200cc8c 100644 --- a/pom.xml +++ b/pom.xml @@ -76,15 +76,6 @@ samoa-test - - heron - - samoa-instances - samoa-api - samoa-heron - samoa-test - - apex