From f9bdaee668a127792b2ab99598795ceab46993f4 Mon Sep 17 00:00:00 2001 From: Pramod Immaneni Date: Fri, 25 Nov 2016 04:41:31 -0800 Subject: [PATCH] Example application showing how to use the launcher API in your project --- examples/launcher/README.md | 9 + examples/launcher/pom.xml | 184 ++++++++++++++++++ .../apex/examples/launcher/Application.java | 55 ++++++ .../launcher/RandomNumberGenerator.java | 62 ++++++ .../main/resources/META-INF/properties.xml | 44 +++++ .../examples/launcher/ApplicationTest.java | 50 +++++ .../src/test/resources/log4j.properties | 22 +++ examples/pom.xml | 42 ++++ pom.xml | 1 + 9 files changed, 469 insertions(+) create mode 100644 examples/launcher/README.md create mode 100644 examples/launcher/pom.xml create mode 100644 examples/launcher/src/main/java/org/apache/apex/examples/launcher/Application.java create mode 100644 examples/launcher/src/main/java/org/apache/apex/examples/launcher/RandomNumberGenerator.java create mode 100644 examples/launcher/src/main/resources/META-INF/properties.xml create mode 100644 examples/launcher/src/test/java/org/apache/apex/examples/launcher/ApplicationTest.java create mode 100644 examples/launcher/src/test/resources/log4j.properties create mode 100644 examples/pom.xml diff --git a/examples/launcher/README.md b/examples/launcher/README.md new file mode 100644 index 0000000000..fd4866dc95 --- /dev/null +++ b/examples/launcher/README.md @@ -0,0 +1,9 @@ +This example application shows you how to use the launcher API to launch an Apex application on a YARN cluster. It +contains a simple Apex application along with the bootstrap code to launch it. + +The application _main_ function in _Application.java contains the launch code. The _pom.xml_ build file +has the necessary configuration to run the application. To launch the application, run the following maven command + +```sh +mvn -Plaunch-app +``` diff --git a/examples/launcher/pom.xml b/examples/launcher/pom.xml new file mode 100644 index 0000000000..e89e03324e --- /dev/null +++ b/examples/launcher/pom.xml @@ -0,0 +1,184 @@ + + + + 4.0.0 + + apex-examples + org.apache.apex + 3.6.0-SNAPSHOT + + + org.apache.apex + 1.0-SNAPSHOT + launcher-example + jar + + + Launcher API Example + A simple example that shows how to launch an application using the API directly + + + lib/*.jar + + + + + + org.apache.maven.plugins + maven-eclipse-plugin + 2.9 + + true + + + + maven-compiler-plugin + 3.3 + + UTF-8 + 1.7 + 1.7 + true + false + true + true + + + + maven-dependency-plugin + 2.8 + + + copy-dependencies + prepare-package + + copy-dependencies + + + target/deps + runtime + + + + + + + + + + + + + launch-app + + true + + + package + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + + build-classpath + + package + + + + app.runtime.path + org.apache.hadoop + slf4j-log4j* + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.5.0 + + + + exec + + package + + + + hadoop + + 1 + ${app.runtime.path}:${project.build.directory}/${project.artifactId}-${project.version}.jar + + + org.apache.apex.examples.launcher.Application + + + + + + + + + + + + + org.apache.apex + malhar-library + 3.6.0 + + + + * + * + + + + + org.apache.apex + apex-common + ${apex.version} + provided + + + junit + junit + 4.10 + test + + + org.apache.apex + apex-engine + ${apex.version} + test + + + + diff --git a/examples/launcher/src/main/java/org/apache/apex/examples/launcher/Application.java b/examples/launcher/src/main/java/org/apache/apex/examples/launcher/Application.java new file mode 100644 index 0000000000..b99adfe8cb --- /dev/null +++ b/examples/launcher/src/main/java/org/apache/apex/examples/launcher/Application.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.apex.examples.launcher; + +import org.apache.apex.api.Launcher; +import org.apache.apex.api.YarnAppLauncher; +import org.apache.hadoop.conf.Configuration; + +import com.datatorrent.api.DAG; +import com.datatorrent.api.StreamingApplication; +import com.datatorrent.api.annotation.ApplicationAnnotation; +import com.datatorrent.lib.io.ConsoleOutputOperator; + +@ApplicationAnnotation(name="TestApplication") +public class Application implements StreamingApplication +{ + + @Override + public void populateDAG(DAG dag, Configuration conf) + { + // Sample DAG with 2 operators + // Replace this code with the DAG you want to build + + RandomNumberGenerator randomGenerator = dag.addOperator("randomGenerator", RandomNumberGenerator.class); + randomGenerator.setNumTuples(500); + + ConsoleOutputOperator cons = dag.addOperator("console", new ConsoleOutputOperator()); + + dag.addStream("randomData", randomGenerator.out, cons.input); + } + + public static void main(String[] args) { + Configuration conf = new Configuration(true); + conf.addResource("/META-INF/properties.xml"); + YarnAppLauncher launcher = Launcher.getLauncher(Launcher.LaunchMode.YARN); + //Launcher launcher = Launcher.getLauncher(Launcher.LaunchMode.LOCAL); + launcher.launchApp(new Application(), conf); + } +} diff --git a/examples/launcher/src/main/java/org/apache/apex/examples/launcher/RandomNumberGenerator.java b/examples/launcher/src/main/java/org/apache/apex/examples/launcher/RandomNumberGenerator.java new file mode 100644 index 0000000000..189fdd5590 --- /dev/null +++ b/examples/launcher/src/main/java/org/apache/apex/examples/launcher/RandomNumberGenerator.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.apex.examples.launcher; + +import com.datatorrent.api.DefaultOutputPort; +import com.datatorrent.api.InputOperator; +import com.datatorrent.common.util.BaseOperator; + +/** + * This is a simple operator that emits random number. + */ +public class RandomNumberGenerator extends BaseOperator implements InputOperator +{ + private int numTuples = 100; + private transient int count = 0; + + public final transient DefaultOutputPort out = new DefaultOutputPort(); + + @Override + public void beginWindow(long windowId) + { + count = 0; + } + + @Override + public void emitTuples() + { + if (count++ < numTuples) { + out.emit(Math.random()); + } + } + + public int getNumTuples() + { + return numTuples; + } + + /** + * Sets the number of tuples to be emitted every window. + * @param numTuples number of tuples + */ + public void setNumTuples(int numTuples) + { + this.numTuples = numTuples; + } +} diff --git a/examples/launcher/src/main/resources/META-INF/properties.xml b/examples/launcher/src/main/resources/META-INF/properties.xml new file mode 100644 index 0000000000..9ee9e9d7d1 --- /dev/null +++ b/examples/launcher/src/main/resources/META-INF/properties.xml @@ -0,0 +1,44 @@ + + + + + + + dt.application.TestApplication.operator.randomGenerator.prop.numTuples + 1000 + + + dt.application.TestApplication.operator.console.prop.stringFormat + hello world: %s + + + diff --git a/examples/launcher/src/test/java/org/apache/apex/examples/launcher/ApplicationTest.java b/examples/launcher/src/test/java/org/apache/apex/examples/launcher/ApplicationTest.java new file mode 100644 index 0000000000..bc159bb08f --- /dev/null +++ b/examples/launcher/src/test/java/org/apache/apex/examples/launcher/ApplicationTest.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.apex.examples.launcher; + +import java.io.IOException; + +import javax.validation.ConstraintViolationException; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.apex.api.EmbeddedAppLauncher; +import org.apache.apex.api.Launcher; +import org.apache.hadoop.conf.Configuration; + +/** + * Test the DAG declaration in local mode. + */ +public class ApplicationTest { + + @Test + public void testApplication() throws IOException, Exception { + try { + EmbeddedAppLauncher launcher = Launcher.getLauncher(Launcher.LaunchMode.EMBEDDED); + Configuration conf = new Configuration(false); + conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml")); + // runs for 10 seconds and quits + launcher.runApp(new Application(), conf, 10000); + } catch (ConstraintViolationException e) { + Assert.fail("constraint violations: " + e.getConstraintViolations()); + } + } + +} diff --git a/examples/launcher/src/test/resources/log4j.properties b/examples/launcher/src/test/resources/log4j.properties new file mode 100644 index 0000000000..98544e8346 --- /dev/null +++ b/examples/launcher/src/test/resources/log4j.properties @@ -0,0 +1,22 @@ +log4j.rootLogger=DEBUG,CONSOLE + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n + +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n +log4j.appender.RFA.File=/tmp/app.log + +# to enable, add SYSLOG to rootLogger +log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender +log4j.appender.SYSLOG.syslogHost=127.0.0.1 +log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout +log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n +log4j.appender.SYSLOG.Facility=LOCAL1 + +#log4j.logger.org.apache.commons.beanutils=warn +log4j.logger.com.datatorrent=debug +log4j.logger.org.apache.apex=debug +log4j.logger.org=info diff --git a/examples/pom.xml b/examples/pom.xml new file mode 100644 index 0000000000..7a2d982947 --- /dev/null +++ b/examples/pom.xml @@ -0,0 +1,42 @@ + + + + + apex + org.apache.apex + 3.6.0-SNAPSHOT + + 4.0.0 + + apex-examples + pom + + Apache Apex Examples + + + 3.6.0-SNAPSHOT + + + + launcher + + diff --git a/pom.xml b/pom.xml index fec5cecd34..b19f7314e1 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,7 @@ engine apex-app-archetype apex-conf-archetype + examples