From 778db32756c4019887d380b9419e035bead139cc Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 14 Jun 2017 19:43:49 +0900 Subject: [PATCH 01/52] Added getters --- bin/interpreter.sh | 13 +- zeppelin-interpreter/pom.xml | 5 + .../remote/RemoteInterpreterServer.java | 107 ++++++++++++--- .../remote/RemoteInterpreterUtils.java | 80 ++++++++++- .../remote/RemoteInterpreterServerTest.java | 4 +- .../RemoteInterpreterManagedProcess.java | 129 +++++++++++++----- .../remote/RemoteInterpreterProcessTest.java | 6 +- 7 files changed, 282 insertions(+), 62 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 29d02218d80..33f2738def0 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -23,7 +23,7 @@ function usage() { echo "usage) $0 -p -d -l -g " } -while getopts "hp:d:l:v:u:g:" o; do +while getopts "hc:p:d:l:v:u:g:" o; do case ${o} in h) usage @@ -32,8 +32,11 @@ while getopts "hp:d:l:v:u:g:" o; do d) INTERPRETER_DIR=${OPTARG} ;; + c) + CALLBACK_HOST=${OPTARG} # This will be used callback host + ;; p) - PORT=${OPTARG} + PORT=${OPTARG} # This will be used callback port ;; l) LOCAL_INTERPRETER_REPO=${OPTARG} @@ -202,12 +205,12 @@ fi if [[ -n "${SPARK_SUBMIT}" ]]; then if [[ -n "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" != "false" ]]; then - INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT}` else - INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT}` fi else - INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} ` + INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ` fi if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ -n "${suid}" || -z "${SPARK_SUBMIT}" ]]; then diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 109099cfc7b..1f31ccec931 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -219,6 +219,11 @@ guava + + io.netty + netty-all + + junit junit diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index f501014bf23..cb3cb60f4fe 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -17,13 +17,22 @@ package org.apache.zeppelin.interpreter.remote; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; import java.nio.ByteBuffer; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -31,26 +40,57 @@ import org.apache.thrift.server.TThreadPoolServer; import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.zeppelin.dep.DependencyResolver; -import org.apache.zeppelin.display.*; -import org.apache.zeppelin.helium.*; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.helium.Application; +import org.apache.zeppelin.helium.ApplicationContext; +import org.apache.zeppelin.helium.ApplicationException; +import org.apache.zeppelin.helium.ApplicationLoader; +import org.apache.zeppelin.helium.HeliumAppAngularObjectRegistry; +import org.apache.zeppelin.helium.HeliumPackage; +import org.apache.zeppelin.interpreter.Constants; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterHookListener; +import org.apache.zeppelin.interpreter.InterpreterHookRegistry; +import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterOutputListener; +import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.thrift.*; -import org.apache.zeppelin.resource.*; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.apache.zeppelin.interpreter.RemoteWorksController; +import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.interpreter.thrift.RemoteApplicationResult; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; +import org.apache.zeppelin.interpreter.thrift.ZeppelinServerResourceParagraphRunner; +import org.apache.zeppelin.resource.DistributedResourcePool; +import org.apache.zeppelin.resource.Resource; +import org.apache.zeppelin.resource.ResourcePool; +import org.apache.zeppelin.resource.ResourceSet; +import org.apache.zeppelin.resource.WellKnownResourceName; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; import org.apache.zeppelin.scheduler.JobProgressPoller; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.user.AuthenticationInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; /** * Entry point for Interpreter process. @@ -70,6 +110,9 @@ public class RemoteInterpreterServer Gson gson = new Gson(); RemoteInterpreterService.Processor processor; + private String callbackHost; + private int callbackPort; + private String host; private int port; private TThreadPoolServer server; @@ -91,7 +134,15 @@ public RemoteInterpreterServer(int port) throws TTransportException { this.port = port; processor = new RemoteInterpreterService.Processor<>(this); - TServerSocket serverTransport = new TServerSocket(port); + TServerSocket serverTransport; + if (null == callbackHost) { + // Dev Interpreter + serverTransport = new TServerSocket(port); + } else { + this.port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + this.host = RemoteInterpreterUtils.findAvailableHostname(); + serverTransport = new TServerSocket(this.port); + } server = new TThreadPoolServer( new TThreadPoolServer.Args(serverTransport).processor(processor)); remoteWorksResponsePool = Collections.synchronizedMap(new HashMap()); @@ -100,6 +151,26 @@ public RemoteInterpreterServer(int port) throws TTransportException { @Override public void run() { + if (null != callbackHost) { + new Thread(new Runnable() { + boolean interrupted = false; + @Override + public void run() { + while (!interrupted && !server.isServing()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + interrupted = true; + } + } + + if (!interrupted) { + RemoteInterpreterUtils + .registerInterpreter(callbackHost, callbackPort, host + ":" + port); + } + } + }).start(); + } logger.info("Starting remote interpreter server on port {}", port); server.serve(); } @@ -146,13 +217,15 @@ public boolean isRunning() { public static void main(String[] args) - throws TTransportException, InterruptedException { - + throws TTransportException, InterruptedException, IOException { + String callbackHost = null; int port = Constants.ZEPPELIN_INTERPRETER_DEFAUlT_PORT; if (args.length > 0) { - port = Integer.parseInt(args[0]); + callbackHost = args[0]; + port = Integer.parseInt(args[1]); } - RemoteInterpreterServer remoteInterpreterServer = new RemoteInterpreterServer(port); + RemoteInterpreterServer remoteInterpreterServer = + new RemoteInterpreterServer(callbackHost, port); remoteInterpreterServer.start(); remoteInterpreterServer.join(); System.exit(0); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java index 4ee6690f7c4..dc3810dc8d9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java @@ -17,14 +17,32 @@ package org.apache.zeppelin.interpreter.remote; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.util.CharsetUtil; import java.io.IOException; import java.net.ConnectException; +import java.net.Inet4Address; +import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.InterfaceAddress; +import java.net.NetworkInterface; import java.net.ServerSocket; import java.net.Socket; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.util.Collections; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @@ -40,6 +58,24 @@ public static int findRandomAvailablePortOnAllLocalInterfaces() throws IOExcepti return port; } + public static String findAvailableHostname() throws UnknownHostException, SocketException { + InetAddress address = InetAddress.getLocalHost(); + if (address.isLoopbackAddress()) { + for (NetworkInterface networkInterface : Collections + .list(NetworkInterface.getNetworkInterfaces())) { + if (!networkInterface.isLoopback()) { + for (InterfaceAddress interfaceAddress : networkInterface.getInterfaceAddresses()) { + InetAddress a = interfaceAddress.getAddress(); + if (a instanceof Inet4Address) { + return a.getHostAddress(); + } + } + } + } + } + return address.getHostName(); + } + public static boolean checkIfRemoteEndpointAccessible(String host, int port) { try { Socket discover = new Socket(); @@ -80,4 +116,42 @@ public static boolean isEnvString(String key) { return key.matches("^[A-Z_0-9]*"); } + + public static void registerInterpreter(String callbackHost, int callbackPort, final String msg) { + LOGGER.info("callbackHost: {}, callbackPort: {}, msg: {}", callbackHost, callbackPort, msg); + EventLoopGroup workerGroup = new NioEventLoopGroup(); + try { + Bootstrap b = new Bootstrap(); + b.group(workerGroup); + b.channel(NioSocketChannel.class); + b.option(ChannelOption.SO_KEEPALIVE, true); + b.handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast(new ChannelInboundHandlerAdapter(){ + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + LOGGER.info("Send message {}", msg); + ctx.writeAndFlush(Unpooled.copiedBuffer(msg, CharsetUtil.UTF_8)); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + cause.printStackTrace(); + ctx.close(); + } + }); + } + }); + + ChannelFuture f = b.connect(callbackHost, callbackPort).sync(); + + // Wait until the connection is closed. + f.channel().closeFuture().sync(); + } catch (InterruptedException e) { + // + } finally { + workerGroup.shutdownGracefully(); + } + } } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java index a4b3a2573ba..6efe4560777 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java @@ -42,7 +42,7 @@ public void tearDown() throws Exception { @Test public void testStartStop() throws InterruptedException, IOException, TException { - RemoteInterpreterServer server = new RemoteInterpreterServer( + RemoteInterpreterServer server = new RemoteInterpreterServer("localhost", RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces()); assertEquals(false, server.isRunning()); @@ -90,7 +90,7 @@ public void run() { @Test public void testStartStopWithQueuedEvents() throws InterruptedException, IOException, TException { - RemoteInterpreterServer server = new RemoteInterpreterServer( + RemoteInterpreterServer server = new RemoteInterpreterServer("localhost", RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces()); assertEquals(false, server.isRunning()); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index 1fb9b90771c..084aa803c69 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -17,17 +17,38 @@ package org.apache.zeppelin.interpreter.remote; -import org.apache.commons.exec.*; -import org.apache.commons.exec.environment.EnvironmentUtils; -import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import io.netty.util.CharsetUtil; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteResultHandler; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.LogOutputStream; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.exec.environment.EnvironmentUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.InterpreterException; /** * This class manages start / stop of remote interpreter process @@ -37,10 +58,12 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess private static final Logger logger = LoggerFactory.getLogger( RemoteInterpreterManagedProcess.class); private final String interpreterRunner; + private final CountDownLatch hostPortLatch; private DefaultExecutor executor; private ExecuteWatchdog watchdog; boolean running = false; + private String host = null; private int port = -1; private final String interpreterDir; private final String localRepoDir; @@ -48,6 +71,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess private Map env; + public RemoteInterpreterManagedProcess( String intpRunner, String intpDir, @@ -64,6 +88,7 @@ public RemoteInterpreterManagedProcess( this.interpreterDir = intpDir; this.localRepoDir = localRepoDir; this.interpreterGroupName = interpreterGroupName; + this.hostPortLatch = new CountDownLatch(1); } RemoteInterpreterManagedProcess(String intpRunner, @@ -80,6 +105,7 @@ public RemoteInterpreterManagedProcess( this.interpreterDir = intpDir; this.localRepoDir = localRepoDir; this.interpreterGroupName = interpreterGroupName; + this.hostPortLatch = new CountDownLatch(1); } @Override @@ -95,8 +121,11 @@ public int getPort() { @Override public void start(String userName, Boolean isUserImpersonate) { // start server process + final String callbackHost; + final int callbackPort; try { - port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + callbackHost = RemoteInterpreterUtils.findAvailableHostname(); + callbackPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); } catch (IOException e1) { throw new InterpreterException(e1); } @@ -104,8 +133,10 @@ public void start(String userName, Boolean isUserImpersonate) { CommandLine cmdLine = CommandLine.parse(interpreterRunner); cmdLine.addArgument("-d", false); cmdLine.addArgument(interpreterDir, false); + cmdLine.addArgument("-c", false); + cmdLine.addArgument(callbackHost, false); cmdLine.addArgument("-p", false); - cmdLine.addArgument(Integer.toString(port), false); + cmdLine.addArgument(Integer.toString(callbackPort), false); if (isUserImpersonate && !userName.equals("anonymous")) { cmdLine.addArgument("-u", false); cmdLine.addArgument(userName, false); @@ -137,34 +168,68 @@ public void start(String userName, Boolean isUserImpersonate) { throw new InterpreterException(e); } - - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < getConnectTimeout()) { - if (!running) { + // Start netty server to receive hostPort information from RemoteInterpreterServer; + new Thread(new Runnable() { + @Override + public void run() { + EventLoopGroup bossGroup = new NioEventLoopGroup(); + EventLoopGroup workerGroup = new NioEventLoopGroup(); try { - cmdOut.flush(); - } catch (IOException e) { - // nothing to do + ServerBootstrap b = new ServerBootstrap(); + b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class) + .handler(new LoggingHandler(LogLevel.DEBUG)) + .childHandler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast(new SimpleChannelInboundHandler() { + @Override + public void channelRegistered(ChannelHandlerContext ctx) throws Exception { + logger.info("{}", ctx); + } + + @Override + public void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) { + logger.info("msg: {}", msg.toString(CharsetUtil.UTF_8)); + String[] hostPort = msg.toString(CharsetUtil.UTF_8).split(":"); + host = hostPort[0]; + port = Integer.parseInt(hostPort[1]); + hostPortLatch.countDown(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + logger.error("Netty error", cause); + ctx.close(); + } + }); + } + }); + + logger.info("Netty server starts"); + // Bind and start to accept incoming connections. + ChannelFuture f = b.bind(callbackPort).sync(); + + // Wait until the server socket is closed. + // In this example, this does not happen, but you can do that to gracefully + // shut down your server. + f.channel().closeFuture().sync(); + } catch (InterruptedException e) { + logger.error("Netty server error while binding", e); + } finally { + workerGroup.shutdownGracefully(); + bossGroup.shutdownGracefully(); } - throw new InterpreterException(new String(cmdOut.toByteArray())); } + }).start(); - try { - if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) { - break; - } else { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - logger.error("Exception in RemoteInterpreterProcess while synchronized reference " + - "Thread.sleep", e); - } - } - } catch (Exception e) { - if (logger.isDebugEnabled()) { - logger.debug("Remote interpreter not yet accessible at localhost:" + port); - } + try { + hostPortLatch.await(getConnectTimeout(), TimeUnit.MILLISECONDS); + // Check if not running + if (null == host || -1 == port) { + throw new InterpreterException("CAnnot run interpreter"); } + } catch (InterruptedException e) { + logger.error("Remote interpreter is not accessible"); } processOutput.setOutputStream(null); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index b85d7ef2fb0..ffbeac89191 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -21,11 +21,11 @@ import static org.junit.Assert.assertFalse; import static org.mockito.Mockito.*; +import java.io.IOException; import java.util.HashMap; import java.util.Properties; import org.apache.thrift.TException; -import org.apache.thrift.transport.TTransportException; import org.apache.zeppelin.interpreter.Constants; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -78,8 +78,8 @@ public void testClientFactory() throws Exception { } @Test - public void testStartStopRemoteInterpreter() throws TException, InterruptedException { - RemoteInterpreterServer server = new RemoteInterpreterServer(3678); + public void testStartStopRemoteInterpreter() throws TException, InterruptedException, IOException { + RemoteInterpreterServer server = new RemoteInterpreterServer(null, 3678); server.start(); boolean running = false; long startTime = System.currentTimeMillis(); From b530d7cb47e3baf7c6539ee6b49beec08b15e7e1 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 14 Jun 2017 22:23:10 +0900 Subject: [PATCH 02/52] changed constructor for ZeppelinDevServer --- .../java/org/apache/zeppelin/helium/ZeppelinDevServer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java index 21ce283abba..3a15e8be0dd 100644 --- a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java +++ b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java @@ -38,8 +38,8 @@ public class ZeppelinDevServer extends private DevInterpreter interpreter = null; private InterpreterOutput out; - public ZeppelinDevServer(int port) throws TException { - super(port); + public ZeppelinDevServer(int port) throws TException, IOException { + super(null, port); } @Override From 645fbca8814bd47b00fa7bf499dd74a0fe673b2b Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sat, 17 Jun 2017 09:44:23 +0900 Subject: [PATCH 03/52] Shaded netty-all dependencies into zeppelin-interpreter --- zeppelin-interpreter/pom.xml | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 1f31ccec931..cf0b255fff9 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -236,4 +236,63 @@ test + + + + + org.apache.maven.plugins + maven-shade-plugin + + false + + + io.netty:netty-all + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + io.netty + ${zeppelin.shade.package}.io.netty + + io.netty.** + + + + + + + package + + shade + + + + + + maven-dependency-plugin + + + copy-dependencies + process-test-resources + + copy-dependencies + + + io.netty + + + + + + From dfd20fd0525752abe8b1a96b4ace3312d875b3b4 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sat, 17 Jun 2017 19:47:01 +0900 Subject: [PATCH 04/52] Changed netty to thrift --- zeppelin-interpreter/pom.xml | 64 -- .../remote/RemoteInterpreterServer.java | 15 +- .../remote/RemoteInterpreterUtils.java | 65 +- .../interpreter/thrift/CallbackInfo.java | 518 +++++++++++ .../RemoteInterpreterCallbackService.java | 879 ++++++++++++++++++ .../thrift/RemoteInterpreterService.thrift | 9 + .../RemoteInterpreterManagedProcess.java | 129 ++- 7 files changed, 1495 insertions(+), 184 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/CallbackInfo.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterCallbackService.java diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index cf0b255fff9..109099cfc7b 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -219,11 +219,6 @@ guava - - io.netty - netty-all - - junit junit @@ -236,63 +231,4 @@ test - - - - - org.apache.maven.plugins - maven-shade-plugin - - false - - - io.netty:netty-all - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - io.netty - ${zeppelin.shade.package}.io.netty - - io.netty.** - - - - - - - package - - shade - - - - - - maven-dependency-plugin - - - copy-dependencies - process-test-resources - - copy-dependencies - - - io.netty - - - - - - diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index cb3cb60f4fe..738aed55015 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -72,6 +72,7 @@ import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.RemoteWorksController; import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource; +import org.apache.zeppelin.interpreter.thrift.CallbackInfo; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.interpreter.thrift.RemoteApplicationResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; @@ -165,8 +166,18 @@ public void run() { } if (!interrupted) { - RemoteInterpreterUtils - .registerInterpreter(callbackHost, callbackPort, host + ":" + port); + CallbackInfo callbackInfo = new CallbackInfo(host, port); + try { + RemoteInterpreterUtils + .registerInterpreter(callbackHost, callbackPort, callbackInfo); + } catch (TException e) { + logger.error("Error while registering interpreter: {}", callbackInfo, e); + try { + shutdown(); + } catch (TException e1) { + logger.warn("Exception occurs while shutting down", e1); + } + } } } }).start(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java index dc3810dc8d9..d5e0cf14667 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java @@ -17,18 +17,6 @@ package org.apache.zeppelin.interpreter.remote; -import io.netty.bootstrap.Bootstrap; -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioSocketChannel; -import io.netty.util.CharsetUtil; import java.io.IOException; import java.net.ConnectException; import java.net.Inet4Address; @@ -41,9 +29,17 @@ import java.net.SocketException; import java.net.UnknownHostException; import java.util.Collections; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.zeppelin.interpreter.thrift.CallbackInfo; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterCallbackService; + /** * */ @@ -117,41 +113,16 @@ public static boolean isEnvString(String key) { return key.matches("^[A-Z_0-9]*"); } - public static void registerInterpreter(String callbackHost, int callbackPort, final String msg) { - LOGGER.info("callbackHost: {}, callbackPort: {}, msg: {}", callbackHost, callbackPort, msg); - EventLoopGroup workerGroup = new NioEventLoopGroup(); - try { - Bootstrap b = new Bootstrap(); - b.group(workerGroup); - b.channel(NioSocketChannel.class); - b.option(ChannelOption.SO_KEEPALIVE, true); - b.handler(new ChannelInitializer() { - @Override - public void initChannel(SocketChannel ch) throws Exception { - ch.pipeline().addLast(new ChannelInboundHandlerAdapter(){ - @Override - public void channelActive(ChannelHandlerContext ctx) throws Exception { - LOGGER.info("Send message {}", msg); - ctx.writeAndFlush(Unpooled.copiedBuffer(msg, CharsetUtil.UTF_8)); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - cause.printStackTrace(); - ctx.close(); - } - }); - } - }); - - ChannelFuture f = b.connect(callbackHost, callbackPort).sync(); - - // Wait until the connection is closed. - f.channel().closeFuture().sync(); - } catch (InterruptedException e) { - // - } finally { - workerGroup.shutdownGracefully(); + public static void registerInterpreter(String callbackHost, int callbackPort, + final CallbackInfo callbackInfo) throws TException { + LOGGER.info("callbackHost: {}, callbackPort: {}, callbackInfo: {}", callbackHost, callbackPort, + callbackInfo); + try (TTransport transport = new TSocket(callbackHost, callbackPort)) { + transport.open(); + TProtocol protocol = new TBinaryProtocol(transport); + RemoteInterpreterCallbackService.Client client = new RemoteInterpreterCallbackService.Client( + protocol); + client.callback(callbackInfo); } } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/CallbackInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/CallbackInfo.java new file mode 100644 index 00000000000..b0c7e9acb57 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/CallbackInfo.java @@ -0,0 +1,518 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Autogenerated by Thrift Compiler (0.9.2) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.zeppelin.interpreter.thrift; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.server.AbstractNonblockingServer.*; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import javax.annotation.Generated; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-6-17") +public class CallbackInfo implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CallbackInfo"); + + private static final org.apache.thrift.protocol.TField HOST_FIELD_DESC = new org.apache.thrift.protocol.TField("host", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField PORT_FIELD_DESC = new org.apache.thrift.protocol.TField("port", org.apache.thrift.protocol.TType.I32, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new CallbackInfoStandardSchemeFactory()); + schemes.put(TupleScheme.class, new CallbackInfoTupleSchemeFactory()); + } + + public String host; // required + public int port; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + HOST((short)1, "host"), + PORT((short)2, "port"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // HOST + return HOST; + case 2: // PORT + return PORT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __PORT_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.HOST, new org.apache.thrift.meta_data.FieldMetaData("host", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.PORT, new org.apache.thrift.meta_data.FieldMetaData("port", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CallbackInfo.class, metaDataMap); + } + + public CallbackInfo() { + } + + public CallbackInfo( + String host, + int port) + { + this(); + this.host = host; + this.port = port; + setPortIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public CallbackInfo(CallbackInfo other) { + __isset_bitfield = other.__isset_bitfield; + if (other.isSetHost()) { + this.host = other.host; + } + this.port = other.port; + } + + public CallbackInfo deepCopy() { + return new CallbackInfo(this); + } + + @Override + public void clear() { + this.host = null; + setPortIsSet(false); + this.port = 0; + } + + public String getHost() { + return this.host; + } + + public CallbackInfo setHost(String host) { + this.host = host; + return this; + } + + public void unsetHost() { + this.host = null; + } + + /** Returns true if field host is set (has been assigned a value) and false otherwise */ + public boolean isSetHost() { + return this.host != null; + } + + public void setHostIsSet(boolean value) { + if (!value) { + this.host = null; + } + } + + public int getPort() { + return this.port; + } + + public CallbackInfo setPort(int port) { + this.port = port; + setPortIsSet(true); + return this; + } + + public void unsetPort() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __PORT_ISSET_ID); + } + + /** Returns true if field port is set (has been assigned a value) and false otherwise */ + public boolean isSetPort() { + return EncodingUtils.testBit(__isset_bitfield, __PORT_ISSET_ID); + } + + public void setPortIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __PORT_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case HOST: + if (value == null) { + unsetHost(); + } else { + setHost((String)value); + } + break; + + case PORT: + if (value == null) { + unsetPort(); + } else { + setPort((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case HOST: + return getHost(); + + case PORT: + return Integer.valueOf(getPort()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case HOST: + return isSetHost(); + case PORT: + return isSetPort(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof CallbackInfo) + return this.equals((CallbackInfo)that); + return false; + } + + public boolean equals(CallbackInfo that) { + if (that == null) + return false; + + boolean this_present_host = true && this.isSetHost(); + boolean that_present_host = true && that.isSetHost(); + if (this_present_host || that_present_host) { + if (!(this_present_host && that_present_host)) + return false; + if (!this.host.equals(that.host)) + return false; + } + + boolean this_present_port = true; + boolean that_present_port = true; + if (this_present_port || that_present_port) { + if (!(this_present_port && that_present_port)) + return false; + if (this.port != that.port) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_host = true && (isSetHost()); + list.add(present_host); + if (present_host) + list.add(host); + + boolean present_port = true; + list.add(present_port); + if (present_port) + list.add(port); + + return list.hashCode(); + } + + @Override + public int compareTo(CallbackInfo other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetHost()).compareTo(other.isSetHost()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetHost()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.host, other.host); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetPort()).compareTo(other.isSetPort()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPort()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.port, other.port); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("CallbackInfo("); + boolean first = true; + + sb.append("host:"); + if (this.host == null) { + sb.append("null"); + } else { + sb.append(this.host); + } + first = false; + if (!first) sb.append(", "); + sb.append("port:"); + sb.append(this.port); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class CallbackInfoStandardSchemeFactory implements SchemeFactory { + public CallbackInfoStandardScheme getScheme() { + return new CallbackInfoStandardScheme(); + } + } + + private static class CallbackInfoStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, CallbackInfo struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // HOST + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.host = iprot.readString(); + struct.setHostIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // PORT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.port = iprot.readI32(); + struct.setPortIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, CallbackInfo struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.host != null) { + oprot.writeFieldBegin(HOST_FIELD_DESC); + oprot.writeString(struct.host); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(PORT_FIELD_DESC); + oprot.writeI32(struct.port); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class CallbackInfoTupleSchemeFactory implements SchemeFactory { + public CallbackInfoTupleScheme getScheme() { + return new CallbackInfoTupleScheme(); + } + } + + private static class CallbackInfoTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, CallbackInfo struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetHost()) { + optionals.set(0); + } + if (struct.isSetPort()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetHost()) { + oprot.writeString(struct.host); + } + if (struct.isSetPort()) { + oprot.writeI32(struct.port); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, CallbackInfo struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.host = iprot.readString(); + struct.setHostIsSet(true); + } + if (incoming.get(1)) { + struct.port = iprot.readI32(); + struct.setPortIsSet(true); + } + } + } + +} + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterCallbackService.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterCallbackService.java new file mode 100644 index 00000000000..6ef08f6b0e8 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterCallbackService.java @@ -0,0 +1,879 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Autogenerated by Thrift Compiler (0.9.2) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.zeppelin.interpreter.thrift; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.server.AbstractNonblockingServer.*; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import javax.annotation.Generated; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-6-17") +public class RemoteInterpreterCallbackService { + + public interface Iface { + + public void callback(CallbackInfo callbackInfo) throws org.apache.thrift.TException; + + } + + public interface AsyncIface { + + public void callback(CallbackInfo callbackInfo, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + } + + public static class Client extends org.apache.thrift.TServiceClient implements Iface { + public static class Factory implements org.apache.thrift.TServiceClientFactory { + public Factory() {} + public Client getClient(org.apache.thrift.protocol.TProtocol prot) { + return new Client(prot); + } + public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { + return new Client(iprot, oprot); + } + } + + public Client(org.apache.thrift.protocol.TProtocol prot) + { + super(prot, prot); + } + + public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { + super(iprot, oprot); + } + + public void callback(CallbackInfo callbackInfo) throws org.apache.thrift.TException + { + send_callback(callbackInfo); + recv_callback(); + } + + public void send_callback(CallbackInfo callbackInfo) throws org.apache.thrift.TException + { + callback_args args = new callback_args(); + args.setCallbackInfo(callbackInfo); + sendBase("callback", args); + } + + public void recv_callback() throws org.apache.thrift.TException + { + callback_result result = new callback_result(); + receiveBase(result, "callback"); + return; + } + + } + public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { + public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { + private org.apache.thrift.async.TAsyncClientManager clientManager; + private org.apache.thrift.protocol.TProtocolFactory protocolFactory; + public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) { + this.clientManager = clientManager; + this.protocolFactory = protocolFactory; + } + public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) { + return new AsyncClient(protocolFactory, clientManager, transport); + } + } + + public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) { + super(protocolFactory, clientManager, transport); + } + + public void callback(CallbackInfo callbackInfo, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + callback_call method_call = new callback_call(callbackInfo, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class callback_call extends org.apache.thrift.async.TAsyncMethodCall { + private CallbackInfo callbackInfo; + public callback_call(CallbackInfo callbackInfo, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.callbackInfo = callbackInfo; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("callback", org.apache.thrift.protocol.TMessageType.CALL, 0)); + callback_args args = new callback_args(); + args.setCallbackInfo(callbackInfo); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_callback(); + } + } + + } + + public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName()); + public Processor(I iface) { + super(iface, getProcessMap(new HashMap>())); + } + + protected Processor(I iface, Map> processMap) { + super(iface, getProcessMap(processMap)); + } + + private static Map> getProcessMap(Map> processMap) { + processMap.put("callback", new callback()); + return processMap; + } + + public static class callback extends org.apache.thrift.ProcessFunction { + public callback() { + super("callback"); + } + + public callback_args getEmptyArgsInstance() { + return new callback_args(); + } + + protected boolean isOneway() { + return false; + } + + public callback_result getResult(I iface, callback_args args) throws org.apache.thrift.TException { + callback_result result = new callback_result(); + iface.callback(args.callbackInfo); + return result; + } + } + + } + + public static class AsyncProcessor extends org.apache.thrift.TBaseAsyncProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(AsyncProcessor.class.getName()); + public AsyncProcessor(I iface) { + super(iface, getProcessMap(new HashMap>())); + } + + protected AsyncProcessor(I iface, Map> processMap) { + super(iface, getProcessMap(processMap)); + } + + private static Map> getProcessMap(Map> processMap) { + processMap.put("callback", new callback()); + return processMap; + } + + public static class callback extends org.apache.thrift.AsyncProcessFunction { + public callback() { + super("callback"); + } + + public callback_args getEmptyArgsInstance() { + return new callback_args(); + } + + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(Void o) { + callback_result result = new callback_result(); + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); + } + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + callback_result result = new callback_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); + } + }; + } + + protected boolean isOneway() { + return false; + } + + public void start(I iface, callback_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.callback(args.callbackInfo,resultHandler); + } + } + + } + + public static class callback_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("callback_args"); + + private static final org.apache.thrift.protocol.TField CALLBACK_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("callbackInfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new callback_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new callback_argsTupleSchemeFactory()); + } + + public CallbackInfo callbackInfo; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + CALLBACK_INFO((short)1, "callbackInfo"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // CALLBACK_INFO + return CALLBACK_INFO; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.CALLBACK_INFO, new org.apache.thrift.meta_data.FieldMetaData("callbackInfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CallbackInfo.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(callback_args.class, metaDataMap); + } + + public callback_args() { + } + + public callback_args( + CallbackInfo callbackInfo) + { + this(); + this.callbackInfo = callbackInfo; + } + + /** + * Performs a deep copy on other. + */ + public callback_args(callback_args other) { + if (other.isSetCallbackInfo()) { + this.callbackInfo = new CallbackInfo(other.callbackInfo); + } + } + + public callback_args deepCopy() { + return new callback_args(this); + } + + @Override + public void clear() { + this.callbackInfo = null; + } + + public CallbackInfo getCallbackInfo() { + return this.callbackInfo; + } + + public callback_args setCallbackInfo(CallbackInfo callbackInfo) { + this.callbackInfo = callbackInfo; + return this; + } + + public void unsetCallbackInfo() { + this.callbackInfo = null; + } + + /** Returns true if field callbackInfo is set (has been assigned a value) and false otherwise */ + public boolean isSetCallbackInfo() { + return this.callbackInfo != null; + } + + public void setCallbackInfoIsSet(boolean value) { + if (!value) { + this.callbackInfo = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case CALLBACK_INFO: + if (value == null) { + unsetCallbackInfo(); + } else { + setCallbackInfo((CallbackInfo)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case CALLBACK_INFO: + return getCallbackInfo(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case CALLBACK_INFO: + return isSetCallbackInfo(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof callback_args) + return this.equals((callback_args)that); + return false; + } + + public boolean equals(callback_args that) { + if (that == null) + return false; + + boolean this_present_callbackInfo = true && this.isSetCallbackInfo(); + boolean that_present_callbackInfo = true && that.isSetCallbackInfo(); + if (this_present_callbackInfo || that_present_callbackInfo) { + if (!(this_present_callbackInfo && that_present_callbackInfo)) + return false; + if (!this.callbackInfo.equals(that.callbackInfo)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_callbackInfo = true && (isSetCallbackInfo()); + list.add(present_callbackInfo); + if (present_callbackInfo) + list.add(callbackInfo); + + return list.hashCode(); + } + + @Override + public int compareTo(callback_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetCallbackInfo()).compareTo(other.isSetCallbackInfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCallbackInfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.callbackInfo, other.callbackInfo); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("callback_args("); + boolean first = true; + + sb.append("callbackInfo:"); + if (this.callbackInfo == null) { + sb.append("null"); + } else { + sb.append(this.callbackInfo); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (callbackInfo != null) { + callbackInfo.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class callback_argsStandardSchemeFactory implements SchemeFactory { + public callback_argsStandardScheme getScheme() { + return new callback_argsStandardScheme(); + } + } + + private static class callback_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, callback_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // CALLBACK_INFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.callbackInfo = new CallbackInfo(); + struct.callbackInfo.read(iprot); + struct.setCallbackInfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, callback_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.callbackInfo != null) { + oprot.writeFieldBegin(CALLBACK_INFO_FIELD_DESC); + struct.callbackInfo.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class callback_argsTupleSchemeFactory implements SchemeFactory { + public callback_argsTupleScheme getScheme() { + return new callback_argsTupleScheme(); + } + } + + private static class callback_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, callback_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetCallbackInfo()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetCallbackInfo()) { + struct.callbackInfo.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, callback_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.callbackInfo = new CallbackInfo(); + struct.callbackInfo.read(iprot); + struct.setCallbackInfoIsSet(true); + } + } + } + + } + + public static class callback_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("callback_result"); + + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new callback_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new callback_resultTupleSchemeFactory()); + } + + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { +; + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(callback_result.class, metaDataMap); + } + + public callback_result() { + } + + /** + * Performs a deep copy on other. + */ + public callback_result(callback_result other) { + } + + public callback_result deepCopy() { + return new callback_result(this); + } + + @Override + public void clear() { + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof callback_result) + return this.equals((callback_result)that); + return false; + } + + public boolean equals(callback_result that) { + if (that == null) + return false; + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + return list.hashCode(); + } + + @Override + public int compareTo(callback_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("callback_result("); + boolean first = true; + + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class callback_resultStandardSchemeFactory implements SchemeFactory { + public callback_resultStandardScheme getScheme() { + return new callback_resultStandardScheme(); + } + } + + private static class callback_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, callback_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, callback_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class callback_resultTupleSchemeFactory implements SchemeFactory { + public callback_resultTupleScheme getScheme() { + return new callback_resultTupleScheme(); + } + } + + private static class callback_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, callback_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, callback_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + } + } + + } + +} diff --git a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift index f2eb13f23d8..f20fb9038e6 100644 --- a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift +++ b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift @@ -88,6 +88,11 @@ struct InterpreterCompletion { 3: string meta } +struct CallbackInfo { + 1: string host, + 2: i32 port +} + service RemoteInterpreterService { void createInterpreter(1: string intpGroupId, 2: string sessionKey, 3: string className, 4: map properties, 5: string userName); @@ -131,3 +136,7 @@ service RemoteInterpreterService { void onReceivedZeppelinResource(1: string object); } + +service RemoteInterpreterCallbackService { + void callback(1: CallbackInfo callbackInfo); +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index 084aa803c69..8e2f2660c12 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -17,19 +17,6 @@ package org.apache.zeppelin.interpreter.remote; -import io.netty.bootstrap.ServerBootstrap; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; -import io.netty.util.CharsetUtil; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -44,11 +31,18 @@ import org.apache.commons.exec.LogOutputStream; import org.apache.commons.exec.PumpStreamHandler; import org.apache.commons.exec.environment.EnvironmentUtils; +import org.apache.thrift.TException; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadPoolServer; +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.thrift.CallbackInfo; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterCallbackService; /** * This class manages start / stop of remote interpreter process @@ -58,11 +52,12 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess private static final Logger logger = LoggerFactory.getLogger( RemoteInterpreterManagedProcess.class); private final String interpreterRunner; - private final CountDownLatch hostPortLatch; + private CountDownLatch hostPortLatch; private DefaultExecutor executor; private ExecuteWatchdog watchdog; boolean running = false; + TServer callbackServer; private String host = null; private int port = -1; private final String interpreterDir; @@ -71,7 +66,6 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess private Map env; - public RemoteInterpreterManagedProcess( String intpRunner, String intpDir, @@ -130,6 +124,48 @@ public void start(String userName, Boolean isUserImpersonate) { throw new InterpreterException(e1); } + logger.info("Thrift server for callback will start. Port: {}", callbackPort); + try { + callbackServer = new TThreadPoolServer( + new TThreadPoolServer.Args(new TServerSocket(callbackPort)).processor( + new RemoteInterpreterCallbackService.Processor<>( + new RemoteInterpreterCallbackService.Iface() { + @Override + public void callback(CallbackInfo callbackInfo) throws TException { + logger.info("Registered: {}", callbackInfo); + host = callbackInfo.getHost(); + port = callbackInfo.getPort(); + hostPortLatch.countDown(); + } + }))); + // Start thrift server to receive callbackInfo from RemoteInterpreterServer; + new Thread(new Runnable() { + @Override + public void run() { + callbackServer.serve(); + } + }).start(); + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + if (callbackServer.isServing()) { + callbackServer.stop(); + } + } + })); + + while (!callbackServer.isServing()) { + logger.debug("callbackServer is not serving"); + Thread.sleep(500); + } + logger.debug("callbackServer is serving now"); + } catch (TTransportException e) { + logger.error("callback server error.", e); + } catch (InterruptedException e) { + logger.warn("", e); + } + CommandLine cmdLine = CommandLine.parse(interpreterRunner); cmdLine.addArgument("-d", false); cmdLine.addArgument(interpreterDir, false); @@ -168,65 +204,13 @@ public void start(String userName, Boolean isUserImpersonate) { throw new InterpreterException(e); } - // Start netty server to receive hostPort information from RemoteInterpreterServer; - new Thread(new Runnable() { - @Override - public void run() { - EventLoopGroup bossGroup = new NioEventLoopGroup(); - EventLoopGroup workerGroup = new NioEventLoopGroup(); - try { - ServerBootstrap b = new ServerBootstrap(); - b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class) - .handler(new LoggingHandler(LogLevel.DEBUG)) - .childHandler(new ChannelInitializer() { - @Override - public void initChannel(SocketChannel ch) throws Exception { - ch.pipeline().addLast(new SimpleChannelInboundHandler() { - @Override - public void channelRegistered(ChannelHandlerContext ctx) throws Exception { - logger.info("{}", ctx); - } - - @Override - public void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) { - logger.info("msg: {}", msg.toString(CharsetUtil.UTF_8)); - String[] hostPort = msg.toString(CharsetUtil.UTF_8).split(":"); - host = hostPort[0]; - port = Integer.parseInt(hostPort[1]); - hostPortLatch.countDown(); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - logger.error("Netty error", cause); - ctx.close(); - } - }); - } - }); - - logger.info("Netty server starts"); - // Bind and start to accept incoming connections. - ChannelFuture f = b.bind(callbackPort).sync(); - - // Wait until the server socket is closed. - // In this example, this does not happen, but you can do that to gracefully - // shut down your server. - f.channel().closeFuture().sync(); - } catch (InterruptedException e) { - logger.error("Netty server error while binding", e); - } finally { - workerGroup.shutdownGracefully(); - bossGroup.shutdownGracefully(); - } - } - }).start(); - try { - hostPortLatch.await(getConnectTimeout(), TimeUnit.MILLISECONDS); + hostPortLatch.await(getConnectTimeout() * 2, TimeUnit.MILLISECONDS); // Check if not running if (null == host || -1 == port) { - throw new InterpreterException("CAnnot run interpreter"); + hostPortLatch = new CountDownLatch(1); + callbackServer.stop(); + throw new InterpreterException("Cannot run interpreter"); } } catch (InterruptedException e) { logger.error("Remote interpreter is not accessible"); @@ -235,6 +219,9 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { } public void stop() { + if (callbackServer.isServing()) { + callbackServer.stop(); + } if (isRunning()) { logger.info("kill interpreter process"); watchdog.destroyProcess(); From 50f573a5097413265f6fc005b9b427503982c04e Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sat, 17 Jun 2017 22:22:53 +0900 Subject: [PATCH 05/52] Changed style --- .../RemoteInterpreterManagedProcess.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index 8e2f2660c12..58af4bf0de1 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -127,17 +127,17 @@ public void start(String userName, Boolean isUserImpersonate) { logger.info("Thrift server for callback will start. Port: {}", callbackPort); try { callbackServer = new TThreadPoolServer( - new TThreadPoolServer.Args(new TServerSocket(callbackPort)).processor( - new RemoteInterpreterCallbackService.Processor<>( - new RemoteInterpreterCallbackService.Iface() { - @Override - public void callback(CallbackInfo callbackInfo) throws TException { - logger.info("Registered: {}", callbackInfo); - host = callbackInfo.getHost(); - port = callbackInfo.getPort(); - hostPortLatch.countDown(); - } - }))); + new TThreadPoolServer.Args(new TServerSocket(callbackPort)).processor( + new RemoteInterpreterCallbackService.Processor<>( + new RemoteInterpreterCallbackService.Iface() { + @Override + public void callback(CallbackInfo callbackInfo) throws TException { + logger.info("Registered: {}", callbackInfo); + host = callbackInfo.getHost(); + port = callbackInfo.getPort(); + hostPortLatch.countDown(); + } + }))); // Start thrift server to receive callbackInfo from RemoteInterpreterServer; new Thread(new Runnable() { @Override From 67fea015c366bf421f8e0571c0b9c28576e9ef71 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 16 Apr 2017 00:39:37 +0900 Subject: [PATCH 06/52] First step to implement from scratch --- pom.xml | 34 + zeppelin-cluster/yarn/pom.xml | 110 ++++ .../cluster/yarn/ApplicationMaster.java | 594 ++++++++++++++++++ .../zeppelin/cluster/yarn/DSConstants.java | 28 + .../zeppelin/cluster/yarn/SimpleYarnApp.java | 178 ++++++ .../yarn/src/main/resources/log4j.properties | 20 + 6 files changed, 964 insertions(+) create mode 100644 zeppelin-cluster/yarn/pom.xml create mode 100644 zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java create mode 100644 zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java create mode 100644 zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java create mode 100644 zeppelin-cluster/yarn/src/main/resources/log4j.properties diff --git a/pom.xml b/pom.xml index 182b508b65d..98e28011308 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,8 @@ 1.3.1 1.3.2 + 2.7.3 + 4.12 1.10.19 @@ -133,6 +135,7 @@ 1.7 1.3.0 2.8.2 + 2.3 64m 512m @@ -263,6 +266,30 @@ ${shiro.version} + + org.apache.hadoop + hadoop-yarn-client + ${hadoop.version} + + + + org.apache.hadoop + hadoop-yarn-common + ${hadoop.version} + + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop.version} + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + junit @@ -1090,6 +1117,13 @@ + + + yarn + + zeppelin-cluster/yarn + + diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml new file mode 100644 index 00000000000..4afdcd57e35 --- /dev/null +++ b/zeppelin-cluster/yarn/pom.xml @@ -0,0 +1,110 @@ + + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.8.0-SNAPSHOT + ../.. + + + zeppelin-cluster-yarn + jar + 0.8.0-SNAPSHOT + Zeppelin Cluster: Yarn + + + + + org.codehaus.jackson + jackson-core-asl + 1.9.13 + + + org.codehaus.jackson + jackson-mapper-asl + 1.9.13 + + + org.codehaus.jackson + jackson-xc + 1.9.13 + + + org.codehaus.jackson + jackson-jaxrs + 1.9.13 + + + + + + + org.apache.hadoop + hadoop-yarn-client + + + + org.apache.hadoop + hadoop-yarn-common + + + + org.apache.hadoop + hadoop-yarn-api + + + + org.apache.hadoop + hadoop-common + + + + + diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java new file mode 100644 index 00000000000..7adebd3aa67 --- /dev/null +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java @@ -0,0 +1,594 @@ +package org.apache.zeppelin.cluster.yarn; + +import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.nio.ByteBuffer; +import java.security.PrivilegedExceptionAction; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Vector; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; +import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.client.api.TimelineClient; +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.yarn.client.api.async.NMClientAsync; +import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class ApplicationMaster { + private static final Logger logger = LoggerFactory.getLogger(ApplicationMaster.class); + + + /** + * + */ + public static enum DSEvent { + DS_APP_ATTEMPT_START, DS_APP_ATTEMPT_END, DS_CONTAINER_START, DS_CONTAINER_END + } + + + /** + * + */ + public static enum DSEntity { + DS_APP_ATTEMPT, DS_CONTAINER + } + + + private final Configuration conf; + + private ByteBuffer allTokens; + + private UserGroupInformation appSubmitterUgi; + + private ApplicationAttemptId appAttemptId; + + private AMRMClientAsync amRMClient; + + private NMClientAsync nmClientAsync; + + private String domainId; + + TimelineClient timelineClient; + + private NMCallbackHandler nmCallbackHandler; + + private String appMasterHostName = ""; + + private int appMasterRpcPort = -1; + + private String appMasterTrackingUrl = ""; + + private int containerMemory = 128; + + private int containerVCores = 1; + + private int numTotalContainers = 1; + + private int requestPriority = 0; + + private boolean done = false; + + private AtomicInteger numRequestedContainers = new AtomicInteger(); + + private AtomicInteger numAllocatedContainers = new AtomicInteger(); + + private AtomicInteger numCompletedContainers = new AtomicInteger(); + + private AtomicInteger numFailedContainers = new AtomicInteger(); + + private List launchThreads = new ArrayList<>(); + + public static void main(String[] args) { + boolean result = false; + try { + ApplicationMaster appMaster = new ApplicationMaster(); + boolean doRun = appMaster.init(args); + if (!doRun) { + System.exit(0); + } + appMaster.run(); + result = appMaster.finish(); + } catch (Throwable t) { + logger.error("Error running ApplicationMaster", t); + ExitUtil.terminate(1, t); + } + + if (result) { + logger.info("Application Master completed successfully. exiting"); + System.exit(0); + } else { + logger.info("Application Master failed. exiting"); + System.exit(2); + } + } + + public ApplicationMaster() { + Configuration hadoopConf = new Configuration(true); + this.conf = new YarnConfiguration(hadoopConf); + } + + public boolean init(String[] args) throws ParseException, IOException { + Map envs = System.getenv(); + for (Map.Entry env : envs.entrySet()) { + logger.info("env. key: " + env.getKey() + ", value: " + env.getValue()); + } + + if (envs.containsKey(ApplicationConstants.Environment.CONTAINER_ID.name())) { + ContainerId containerId = ConverterUtils + .toContainerId(envs.get(ApplicationConstants.Environment.CONTAINER_ID.name())); + appAttemptId = containerId.getApplicationAttemptId(); + } else { + throw new IllegalArgumentException("Application Attempt Id not set in the environment"); + } + + logger.info("Application master for app, appId={}, clustertimestamp={}, attemptId={}", + appAttemptId.getApplicationId().getId(), + appAttemptId.getApplicationId().getClusterTimestamp(), appAttemptId.getAttemptId()); + + if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN)) { + domainId = envs.get(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN); + } + + return true; + } + + public void run() throws IOException, InterruptedException, YarnException { + logger.info("Starting ApplicationMaster"); + + Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); + DataOutputBuffer dob = new DataOutputBuffer(); + credentials.writeTokenStorageToStream(dob); + + Iterator> iter = credentials.getAllTokens().iterator(); + logger.info("Executing with tokens: "); + while (iter.hasNext()) { + Token token = iter.next(); + logger.info(token.toString()); + if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { + iter.remove(); + } + } + allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + + String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); + + appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); + appSubmitterUgi.addCredentials(credentials); + + AMRMClientAsync.CallbackHandler rmCallbackHandler = new RMCallbackHandler(); + amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, rmCallbackHandler); + amRMClient.init(conf); + amRMClient.start(); + + nmCallbackHandler = new NMCallbackHandler(this); + nmClientAsync = new NMClientAsyncImpl(nmCallbackHandler); + nmClientAsync.init(conf); + nmClientAsync.start(); + + startTimelineClient(conf); + if (timelineClient != null) { + publishApplicationAttemptEvent(timelineClient, appAttemptId.toString(), + DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi); + } + + appMasterHostName = NetUtils.getHostname(); + + RegisterApplicationMasterResponse response = amRMClient + .registerApplicationMaster(appMasterHostName, appMasterRpcPort, appMasterTrackingUrl); + + int maxMem = response.getMaximumResourceCapability().getMemory(); + logger.info("Max mem capability of resources in this cluster {}", maxMem); + + int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); + logger.info("Max vcores capability of resources in this cluster {}", maxVCores); + + if (containerMemory > maxMem) { + containerMemory = maxMem; + logger.info("container memory changed to {} because of limitation of resources", maxMem); + } + + if (containerVCores > maxVCores) { + containerVCores = maxVCores; + logger.info("container vcores changed to {} because of limitation of resources", maxVCores); + } + + List previousAMRunningContainers = response.getContainersFromPreviousAttempts(); + numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); + + int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size(); + + for (int i = 0; i < numTotalContainersToRequest; i++) { + AMRMClient.ContainerRequest containerRequest = setupContainerAskForRM(); + amRMClient.addContainerRequest(containerRequest); + } + numRequestedContainers.set(numTotalContainers); + + } + + public boolean finish() { + while (!done && (numCompletedContainers.get() != numTotalContainers)) { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + } + } + + if (timelineClient != null) { + publishApplicationAttemptEvent(timelineClient, appAttemptId.toString(), + DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi); + } + + for (Thread launchThread : launchThreads) { + try { + launchThread.join(10000); + } catch (InterruptedException e) { + logger.info("Exception thrown in thread join", e); + } + } + + logger.info("Application completed. Stopping running containers"); + nmClientAsync.stop(); + + logger.info("Application completed. Signalling finish to RM"); + + FinalApplicationStatus appStatus; + String appMessage = null; + boolean success = true; + if (numFailedContainers.get() == 0 && numCompletedContainers.get() == numTotalContainers) { + appStatus = FinalApplicationStatus.SUCCEEDED; + } else { + appStatus = FinalApplicationStatus.FAILED; + appMessage = + "Diagnostics. total=" + numTotalContainers + ",completed=" + numCompletedContainers.get() + + ", allocated=" + numAllocatedContainers.get() + ", failed=" + numFailedContainers + .get(); + logger.info(appMessage); + success = false; + } + try { + amRMClient.unregisterApplicationMaster(appStatus, appMessage, null); + } catch (YarnException | IOException e) { + logger.error("Failed to unregister application", e); + } + + amRMClient.stop(); + + if (null != timelineClient) { + timelineClient.stop(); + } + + return success; + } + + private AMRMClient.ContainerRequest setupContainerAskForRM() { + Priority pri = Priority.newInstance(requestPriority); + + Resource capability = Resource.newInstance(containerMemory, containerVCores); + + AMRMClient.ContainerRequest request = + new AMRMClient.ContainerRequest(capability, null, null, pri); + logger.info("Requested container ask: {}", request.toString()); + + return request; + } + + private void startTimelineClient(final Configuration conf) + throws YarnException, IOException, InterruptedException { + try { + appSubmitterUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + // Creating the Timeline Client + timelineClient = TimelineClient.createTimelineClient(); + timelineClient.init(conf); + timelineClient.start(); + } else { + timelineClient = null; + logger.warn("Timeline service is not enabled"); + } + return null; + } + }); + } catch (UndeclaredThrowableException e) { + throw new YarnException(e.getCause()); + } + } + + private static void publishApplicationAttemptEvent(final TimelineClient timelineClient, + String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) { + final TimelineEntity entity = new TimelineEntity(); + entity.setEntityId(appAttemptId); + entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString()); + entity.setDomainId(domainId); + entity.addPrimaryFilter("user", ugi.getShortUserName()); + TimelineEvent event = new TimelineEvent(); + event.setEventType(appEvent.toString()); + event.setTimestamp(System.currentTimeMillis()); + entity.addEvent(event); + try { + timelineClient.putEntities(entity); + } catch (YarnException | IOException e) { + logger.error( + "App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end") + + " event could not be published for " + appAttemptId, e); + } + } + + private static void publishContainerStartEvent(final TimelineClient timelineClient, + Container container, String domainId, UserGroupInformation ugi) { + final TimelineEntity entity = new TimelineEntity(); + entity.setEntityId(container.getId().toString()); + entity.setEntityType(DSEntity.DS_CONTAINER.toString()); + entity.setDomainId(domainId); + entity.addPrimaryFilter("user", ugi.getShortUserName()); + TimelineEvent event = new TimelineEvent(); + event.setTimestamp(System.currentTimeMillis()); + event.setEventType(DSEvent.DS_CONTAINER_START.toString()); + event.addEventInfo("Node", container.getNodeId().toString()); + event.addEventInfo("Resources", container.getResource().toString()); + entity.addEvent(event); + try { + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public TimelinePutResponse run() throws Exception { + return timelineClient.putEntities(entity); + } + }); + } catch (Exception e) { + logger + .error("Container start event could not be published for " + container.getId().toString(), + e instanceof UndeclaredThrowableException ? e.getCause() : e); + } + } + + private static void publishContainerEndEvent(final TimelineClient timelineClient, + ContainerStatus container, String domainId, UserGroupInformation ugi) { + final TimelineEntity entity = new TimelineEntity(); + entity.setEntityId(container.getContainerId().toString()); + entity.setEntityType(DSEntity.DS_CONTAINER.toString()); + entity.setDomainId(domainId); + entity.addPrimaryFilter("user", ugi.getShortUserName()); + TimelineEvent event = new TimelineEvent(); + event.setTimestamp(System.currentTimeMillis()); + event.setEventType(DSEvent.DS_CONTAINER_END.toString()); + event.addEventInfo("State", container.getState().name()); + event.addEventInfo("Exit Status", container.getExitStatus()); + entity.addEvent(event); + try { + timelineClient.putEntities(entity); + } catch (YarnException | IOException e) { + logger.error( + "Container end event could not be published for " + container.getContainerId().toString(), + e); + } + } + + private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler { + @Override + public void onContainersCompleted(List completedContainers) { + logger.info( + "Got response from RM for container ask, completedCnt=" + completedContainers.size()); + for (ContainerStatus containerStatus : completedContainers) { + logger.info(appAttemptId + " got container status for containerID=" + containerStatus + .getContainerId() + ", state=" + containerStatus.getState() + ", exitStatus=" + + containerStatus.getExitStatus() + ", diagnostics=" + containerStatus + .getDiagnostics()); + + // non complete containers should not be here + assert (containerStatus.getState() == ContainerState.COMPLETE); + + // increment counters for completed/failed containers + int exitStatus = containerStatus.getExitStatus(); + if (0 != exitStatus) { + // container failed + if (ContainerExitStatus.ABORTED != exitStatus) { + // shell script failed + // counts as completed + numCompletedContainers.incrementAndGet(); + numFailedContainers.incrementAndGet(); + } else { + // container was killed by framework, possibly preempted + // we should re-try as the container was lost for some reason + numAllocatedContainers.decrementAndGet(); + numRequestedContainers.decrementAndGet(); + // we do not need to release the container as it would be done + // by the RM + } + } else { + // nothing to do + // container completed successfully + numCompletedContainers.incrementAndGet(); + logger.info("Container completed successfully." + ", containerId=" + containerStatus + .getContainerId()); + } + if (timelineClient != null) { + publishContainerEndEvent(timelineClient, containerStatus, domainId, appSubmitterUgi); + } + } + } + + @Override + public void onContainersAllocated(List allocatedContainers) { + logger.info( + "Got response from RM for container ask, allocatedCnt=" + allocatedContainers.size()); + numAllocatedContainers.addAndGet(allocatedContainers.size()); + for (Container allocatedContainer : allocatedContainers) { + logger.info( + "Launching shell command on a new container." + ", containerId=" + allocatedContainer + .getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory" + + allocatedContainer.getResource().getMemory() + ", containerResourceVirtualCores" + + allocatedContainer.getResource().getVirtualCores()); + // + ", containerToken" + // +allocatedContainer.getContainerToken().getIdentifier().toString()); + + LaunchContainerRunnable runnableLaunchContainer = + new LaunchContainerRunnable(allocatedContainer, nmCallbackHandler); + Thread launchThread = new Thread(runnableLaunchContainer); + + // launch and start the container on a separate thread to keep + // the main thread unblocked + // as all containers may not be allocated at one go. + launchThreads.add(launchThread); + launchThread.start(); + } + + } + + @Override + public void onShutdownRequest() { + done = true; + } + + @Override + public void onNodesUpdated(List list) { + + } + + @Override + public float getProgress() { + return (float) numCompletedContainers.get() / numTotalContainers; + } + + @Override + public void onError(Throwable throwable) { + done = true; + amRMClient.stop(); + } + } + + + private class NMCallbackHandler implements NMClientAsync.CallbackHandler { + private ConcurrentMap containers = new ConcurrentHashMap<>(); + private final ApplicationMaster applicationMaster; + + public NMCallbackHandler(ApplicationMaster applicationMaster) { + this.applicationMaster = applicationMaster; + } + + public void addContainer(ContainerId containerId, Container container) { + containers.putIfAbsent(containerId, container); + } + + @Override + public void onContainerStarted(ContainerId containerId, Map map) { + logger.debug("Succeeded to start Container {}", containerId); + Container container = containers.get(containerId); + if (container != null) { + applicationMaster.nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId()); + } + if (applicationMaster.timelineClient != null) { + ApplicationMaster + .publishContainerStartEvent(timelineClient, container, applicationMaster.domainId, + applicationMaster.appSubmitterUgi); + } + } + + @Override + public void onContainerStatusReceived(ContainerId containerId, + ContainerStatus containerStatus) { + logger.debug("Container status: id={}, status={}", containerId, containerStatus); + + } + + @Override + public void onContainerStopped(ContainerId containerId) { + logger.debug("Succeeded to stop Container {}", containerId); + containers.remove(containerId); + } + + @Override + public void onStartContainerError(ContainerId containerId, Throwable throwable) { + logger.error("Failed to start Container {}", containerId); + containers.remove(containerId); + applicationMaster.numCompletedContainers.incrementAndGet(); + applicationMaster.numFailedContainers.incrementAndGet(); + } + + @Override + public void onGetContainerStatusError(ContainerId containerId, Throwable throwable) { + logger.error("Failed to query the status of Container {}", containerId); + } + + @Override + public void onStopContainerError(ContainerId containerId, Throwable throwable) { + logger.error("Failed to stop Container {}", containerId); + containers.remove(containerId); + } + } + + + private class LaunchContainerRunnable implements Runnable { + Container container; + NMCallbackHandler containerListener; + + public LaunchContainerRunnable(Container container, NMCallbackHandler containerListener) { + this.container = container; + this.containerListener = containerListener; + } + + @Override + public void run() { + logger.info("Setting up container launch container for containerid={}", container.getId()); + + Map localResources = new HashMap<>(); + + Vector vargs = new Vector<>(30); + vargs.add("echo \"madeng\""); + vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); + vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); + String command = Joiner.on(" ").join(vargs); + ContainerLaunchContext ctx = ContainerLaunchContext + .newInstance(localResources, System.getenv(), Lists.newArrayList(command), null, + allTokens.duplicate(), null); + containerListener.addContainer(container.getId(), container); + nmClientAsync.startContainerAsync(container, ctx); + } + } + +} diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java new file mode 100644 index 00000000000..9ce218934d1 --- /dev/null +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java @@ -0,0 +1,28 @@ +package org.apache.zeppelin.cluster.yarn; + +/** + * + */ +public class DSConstants { + /** + * Environment key name pointing to the shell script's location + */ + public static final String DISTRIBUTEDSHELLSCRIPTLOCATION = "DISTRIBUTEDSHELLSCRIPTLOCATION"; + + /** + * Environment key name denoting the file timestamp for the shell script. + * Used to validate the local resource. + */ + public static final String DISTRIBUTEDSHELLSCRIPTTIMESTAMP = "DISTRIBUTEDSHELLSCRIPTTIMESTAMP"; + + /** + * Environment key name denoting the file content length for the shell script. + * Used to validate the local resource. + */ + public static final String DISTRIBUTEDSHELLSCRIPTLEN = "DISTRIBUTEDSHELLSCRIPTLEN"; + + /** + * Environment key name denoting the timeline domain ID. + */ + public static final String DISTRIBUTEDSHELLTIMELINEDOMAIN = "DISTRIBUTEDSHELLTIMELINEDOMAIN"; +} diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java new file mode 100644 index 00000000000..a25e76d9539 --- /dev/null +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java @@ -0,0 +1,178 @@ +package org.apache.zeppelin.cluster.yarn; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.Vector; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class SimpleYarnApp { + private static final Logger logger = LoggerFactory.getLogger(SimpleYarnApp.class); + + private String appName = "SIMPLE"; + + private String appMasterJar = "/Users/jl/local/src/g/zeppelin/zeppelin-cluster/yarn/target/" + + "zeppelin-cluster-yarn-0.8.0-SNAPSHOT.jar"; + + private String appMasterJarPath = "zeppelin-cluster.jar"; + + private String appMasterMainClass = "org.apache.zeppelin.cluster.yarn.ApplicationMaster"; + + public static void main(String[] args) throws Exception { + new SimpleYarnApp().run(); + } + + public void run() throws Exception { + boolean keepContainers = false; + String appName = "test"; + Configuration conf = new YarnConfiguration(); + + YarnClient yarnClient = YarnClient.createYarnClient(); + yarnClient.init(conf); + yarnClient.start(); + + YarnClientApplication app = yarnClient.createApplication(); + GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); + + // set the application submission context + ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); + + appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); + appContext.setApplicationName(appName); + + Map localResources = new HashMap<>(); + + FileSystem fs = FileSystem.get(conf); + addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); + + Map env = new HashMap<>(); + + StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$$()) + .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); + for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, + YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { + classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); + classPathEnv.append(c.trim()); + } + + logger.debug("classpath: {}", classPathEnv.toString()); + + env.put("CLASSPATH", classPathEnv.toString()); + + Vector vargs = new Vector<>(30); + + vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); + + vargs.add("-Xmx128m"); + + vargs.add(appMasterMainClass); + + vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ApplicationMaster.stdout"); + vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ApplicationMaster.stderr"); + String command = Joiner.on(" ").join(vargs); + logger.info("command: {}", command); + List commands = Lists.newArrayList(command); + + ContainerLaunchContext amContainer = + ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); + + Resource capability = Resource.newInstance(128, 1); + appContext.setResource(capability); + + if (UserGroupInformation.isSecurityEnabled()) { + Credentials credentials = new Credentials(); + String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); + if (tokenRenewer == null || tokenRenewer.length() == 0) { + throw new IOException("Can't get Master Kerberos prinipal for the RM to use as renewer"); + } + + final Token tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); + if (tokens != null) { + for (Token token : tokens) { + logger.info("Got dt from " + fs.getUri() + "; " + token); + } + } + + DataOutputBuffer dob = new DataOutputBuffer(); + credentials.writeTokenStorageToStream(dob); + ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + amContainer.setTokens(fsTokens); + } + + appContext.setAMContainerSpec(amContainer); + + Priority pri = Priority.newInstance(0); + appContext.setPriority(pri); + + appContext.setQueue("default"); + + appContext.setApplicationType("ZEPPELIN"); + + yarnClient.submitApplication(appContext); + + ApplicationReport report = yarnClient.getApplicationReport(appId); + + yarnClient.stop(); + } + + private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, + String appId, Map localResources, String resources) + throws IOException { + String suffix = appName + "/" + appId + "/" + fileDstPath; + Path dst = new Path(fs.getHomeDirectory(), suffix); + if (fileSrcPath == null) { + FSDataOutputStream ostream = null; + try { + ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710)); + ostream.writeUTF(resources); + } finally { + IOUtils.closeQuietly(ostream); + } + } else { + fs.copyFromLocalFile(new Path(fileSrcPath), dst); + } + FileStatus scFileStatus = fs.getFileStatus(dst); + LocalResource scRsrc = LocalResource + .newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), + scFileStatus.getModificationTime()); + localResources.put(fileDstPath, scRsrc); + } +} diff --git a/zeppelin-cluster/yarn/src/main/resources/log4j.properties b/zeppelin-cluster/yarn/src/main/resources/log4j.properties new file mode 100644 index 00000000000..fb31a1e3417 --- /dev/null +++ b/zeppelin-cluster/yarn/src/main/resources/log4j.properties @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +log4j.rootLogger=DEBUG, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n From cbae84097559d111964a838565521910add449e1 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 24 Apr 2017 06:47:31 +0900 Subject: [PATCH 07/52] Added license header for avoiding rat failed --- .../cluster/yarn/ApplicationMaster.java | 17 +++++++++++++++ .../apache/zeppelin/cluster/yarn/Client.java | 21 +++++++++++++++++++ .../zeppelin/cluster/yarn/DSConstants.java | 17 +++++++++++++++ .../zeppelin/cluster/yarn/SimpleYarnApp.java | 17 +++++++++++++++ 4 files changed, 72 insertions(+) create mode 100644 zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java index 7adebd3aa67..c6d959251f7 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.cluster.yarn; import java.io.IOException; diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java new file mode 100644 index 00000000000..145b70577c8 --- /dev/null +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.cluster.yarn; + +public class Client { +} diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java index 9ce218934d1..190cedc4257 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.cluster.yarn; /** diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java index a25e76d9539..fd383ecf915 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.cluster.yarn; import java.io.IOException; From 6c0b792ab6604840f2c782fc723e1549831fe9a9 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 24 Apr 2017 18:14:51 +0900 Subject: [PATCH 08/52] WIP --- zeppelin-cluster/yarn/pom.xml | 15 ++ .../yarn/ApplicationCallbackHandler.java | 7 + .../apache/zeppelin/cluster/yarn/Client.java | 145 +++++++++++++++++- 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationCallbackHandler.java diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml index 4afdcd57e35..10043af5887 100644 --- a/zeppelin-cluster/yarn/pom.xml +++ b/zeppelin-cluster/yarn/pom.xml @@ -56,10 +56,25 @@ jackson-jaxrs 1.9.13 + + com.jcraft + jsch + 0.1.42 + + + org.apache.commons + commons-compress + 1.4.1 + + + ${project.groupId} + zeppelin-zengine + ${project.version} + org.apache.hadoop hadoop-yarn-client diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationCallbackHandler.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationCallbackHandler.java new file mode 100644 index 00000000000..ff8d844e34b --- /dev/null +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationCallbackHandler.java @@ -0,0 +1,7 @@ +package org.apache.zeppelin.cluster.yarn; + +/** + * + */ +public interface ApplicationCallbackHandler { +} diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 145b70577c8..1c8f29fbcdc 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -17,5 +17,148 @@ package org.apache.zeppelin.cluster.yarn; -public class Client { +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.zeppelin.cluster.ClusterManager; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterSetting; + +/** + * + */ +public class Client implements ClusterManager { + private static final Logger logger = LoggerFactory.getLogger(Client.class); + + private final ZeppelinConfiguration zeppelinConfiguration; + private final ApplicationCallbackHandler applicationHandler; + + private Configuration configuration; + private YarnClient yarnClient; + private boolean initialized; + + /** + * `id` is a unique key to figure out Application + */ + private Map idApplicationIdMap; + + public Client(ZeppelinConfiguration zeppelinConfiguration, + ApplicationCallbackHandler applicationHandler) { + this.zeppelinConfiguration = zeppelinConfiguration; + this.applicationHandler = applicationHandler; + + this.initialized = false; + } + + @Override + public synchronized void start() { + if (!initialized) { // it will help when calling it multiple times from different threads + logger.info("Start to initialize yarn client"); + this.configuration = new Configuration(); + this.yarnClient = YarnClient.createYarnClient(); + this.yarnClient.init(configuration); + this.yarnClient.start(); + + closeAllApplications(); + + this.idApplicationIdMap = new ConcurrentHashMap<>(); + this.initialized = true; + } + } + + @Override + public synchronized void stop() { + if (initialized) { + logger.info("Stop yarn client"); + + closeAllApplications(); + + this.yarnClient.stop(); + this.initialized = false; + } + } + + @Override + public Interpreter createInterpreter(String id, InterpreterSetting interpreterSetting) + throws InterpreterException { + if (!initialized) { + start(); + } + return null; + } + + @Override + public void releaseResource(String id) { + if (!initialized) { + start(); + } + ApplicationId applicationId = idApplicationIdMap.get(id); + try { + ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId); + logApplicationReport(applicationReport); + yarnClient.killApplication(applicationId); + } catch (YarnException | IOException e) { + logger.info("Got error while releasing resource. Resource: {}, applicationId: {}", id, + applicationId); + } + + + } + + private void closeAllApplications() { + if (null != idApplicationIdMap && !idApplicationIdMap.isEmpty()) { + for (ApplicationId applicationId : idApplicationIdMap.values()) { + try { + yarnClient.killApplication(applicationId); + } catch (YarnException | IOException e) { + logger.debug("You might check the status of applicationId: {}", applicationId); + } + } + } + } + + private void logApplicationReport(ApplicationReport applicationReport) { + logger.info("client token", getClientToken(applicationReport)); + logger.info("diagnostics", applicationReport.getDiagnostics()); + logger.info("ApplicationMaster host", applicationReport.getHost()); + logger.info("ApplicationMaster RPC port", String.valueOf(applicationReport.getRpcPort())); + logger.info("queue", applicationReport.getQueue()); + logger.info("start time", String.valueOf(applicationReport.getStartTime())); + logger.info("final status", applicationReport.getFinalApplicationStatus().toString()); + logger.info("tracking URL", applicationReport.getTrackingUrl()); + logger.info("user", applicationReport.getUser()); + } + + private String getClientToken(ApplicationReport applicationReport) { + Token token = applicationReport.getClientToAMToken(); + if (null != token) { + return token.toString(); + } else { + return ""; + } + } + + private class ApplicationMonitor implements Runnable { + + @Override + public void run() { + for (Map.Entry entry : idApplicationIdMap.entrySet()) { + String id = entry.getKey(); + ApplicationId applicationId = entry.getValue(); + + } + } + } } From 06097a0938233c1e1ca48213d50444c5361172d4 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 24 Apr 2017 23:55:08 +0900 Subject: [PATCH 09/52] WIP --- .../yarn/ApplicationCallbackHandler.java | 7 -- .../apache/zeppelin/cluster/yarn/Client.java | 86 ++++++++++++++++--- .../cluster/ApplicationCallbackHandler.java | 10 +++ .../zeppelin/cluster/ClusterManager.java | 49 +++++++++++ 4 files changed, 134 insertions(+), 18 deletions(-) delete mode 100644 zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationCallbackHandler.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationCallbackHandler.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationCallbackHandler.java deleted file mode 100644 index ff8d844e34b..00000000000 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationCallbackHandler.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.apache.zeppelin.cluster.yarn; - -/** - * - */ -public interface ApplicationCallbackHandler { -} diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 1c8f29fbcdc..801cd645fe4 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -18,32 +18,38 @@ package org.apache.zeppelin.cluster.yarn; import java.io.IOException; +import java.util.ArrayList; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.exceptions.YarnException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.zeppelin.cluster.ApplicationCallbackHandler; import org.apache.zeppelin.cluster.ClusterManager; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; /** * */ -public class Client implements ClusterManager { +public class Client extends ClusterManager { private static final Logger logger = LoggerFactory.getLogger(Client.class); - - private final ZeppelinConfiguration zeppelinConfiguration; - private final ApplicationCallbackHandler applicationHandler; + private static final ScheduledExecutorService scheduledExecutorService = + Executors.newScheduledThreadPool(1); private Configuration configuration; private YarnClient yarnClient; @@ -53,16 +59,15 @@ public class Client implements ClusterManager { * `id` is a unique key to figure out Application */ private Map idApplicationIdMap; + private Map appStatusMap; public Client(ZeppelinConfiguration zeppelinConfiguration, ApplicationCallbackHandler applicationHandler) { - this.zeppelinConfiguration = zeppelinConfiguration; - this.applicationHandler = applicationHandler; + super(zeppelinConfiguration, applicationHandler); this.initialized = false; } - @Override public synchronized void start() { if (!initialized) { // it will help when calling it multiple times from different threads logger.info("Start to initialize yarn client"); @@ -74,32 +79,35 @@ public synchronized void start() { closeAllApplications(); this.idApplicationIdMap = new ConcurrentHashMap<>(); + + scheduledExecutorService + .scheduleAtFixedRate(new ApplicationMonitor(), 1, 1, TimeUnit.SECONDS); + this.initialized = true; } } - @Override public synchronized void stop() { if (initialized) { logger.info("Stop yarn client"); closeAllApplications(); + scheduledExecutorService.shutdown(); + this.yarnClient.stop(); this.initialized = false; } } - @Override public Interpreter createInterpreter(String id, InterpreterSetting interpreterSetting) throws InterpreterException { if (!initialized) { start(); } - return null; + return new RemoteInterpreter(); } - @Override public void releaseResource(String id) { if (!initialized) { start(); @@ -154,10 +162,66 @@ private class ApplicationMonitor implements Runnable { @Override public void run() { + ArrayList removedIds = new ArrayList<>(); for (Map.Entry entry : idApplicationIdMap.entrySet()) { String id = entry.getKey(); ApplicationId applicationId = entry.getValue(); + try { + ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId); + YarnApplicationState curState = applicationReport.getYarnApplicationState(); + YarnApplicationState oldState = appStatusMap.get(applicationId); + switch (curState) { + case NEW: + case NEW_SAVING: + case SUBMITTED: + case ACCEPTED: + if (null == oldState) { + logger.info("new application added. Id: {}, applicationId: {}", id, applicationId); + } + appStatusMap.put(applicationId, curState); + break; + case RUNNING: + if (!YarnApplicationState.RUNNING.equals(oldState)) { + logger.info("id {} started with applicationId {}", id, applicationId); + appStatusMap.put(applicationId, curState); + applicationCallbackHandler + .onStarted(applicationReport.getHost(), applicationReport.getRpcPort()); + } + break; + case FINISHED: + if (!YarnApplicationState.FINISHED.equals(oldState)) { + logger.info("id {}, applicationId {} finished with final Status {}", id, + applicationId, applicationReport.getFinalApplicationStatus()); + applicationCallbackHandler.onTerminated(id); + removedIds.add(id); + } + break; + case FAILED: + if (!YarnApplicationState.FAILED.equals(oldState)) { + logger + .info("id {}, applicationId {} failed with final Status {}", id, applicationId, + applicationReport.getFinalApplicationStatus()); + applicationCallbackHandler.onTerminated(id); + removedIds.add(id); + } + break; + case KILLED: + if (!YarnApplicationState.KILLED.equals(oldState)) { + logger + .info("id {}, applicationId {} killed with final Status {}", id, applicationId, + applicationReport.getFinalApplicationStatus()); + applicationCallbackHandler.onTerminated(id); + removedIds.add(id); + } + break; + } + } catch (YarnException | IOException e) { + logger.debug("Error occurs while fetching status of {}", applicationId, e); + } + for (String removedId : removedIds) { + idApplicationIdMap.remove(removedId); + } } } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java new file mode 100644 index 00000000000..82e8debb02b --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java @@ -0,0 +1,10 @@ +package org.apache.zeppelin.cluster; + +/** + * + */ +public interface ApplicationCallbackHandler { + void onStarted(String host, int port); + + void onTerminated(String id); +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java new file mode 100644 index 00000000000..7a504732002 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.cluster; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterSetting; + +public abstract class ClusterManager { + protected final ZeppelinConfiguration zeppelinConfiguration; + protected final ApplicationCallbackHandler applicationCallbackHandler; + + protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration, + ApplicationCallbackHandler applicationCallbackHandler) { + this.zeppelinConfiguration = zeppelinConfiguration; + this.applicationCallbackHandler = applicationCallbackHandler; + } + + /** + * Can throw `RuntimeException` + */ + public abstract void start(); + + /** + * Can throw `RuntimeException` + */ + public abstract void stop(); + + public abstract Interpreter createInterpreter(String id, InterpreterSetting interpreterSetting) + throws InterpreterException; + + public abstract void releaseResource(String id); +} From 4fccd5b87206f0ccef20f0432345fe2d21c7ab46 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 30 Apr 2017 13:34:12 +0900 Subject: [PATCH 10/52] WIP --- pom.xml | 8 +- .../apache/zeppelin/cluster/yarn/Client.java | 165 ++++++++++++------ .../yarn/RemoteInterpreterYarnProcess.java | 64 +++++++ .../zeppelin/cluster/ClusterManager.java | 12 +- 4 files changed, 185 insertions(+), 64 deletions(-) create mode 100644 zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java diff --git a/pom.xml b/pom.xml index 98e28011308..2f370ae095b 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,7 @@ scio zeppelin-web zeppelin-server + zeppelin-cluster/yarn zeppelin-jupyter zeppelin-distribution @@ -1117,13 +1118,6 @@ - - - yarn - - zeppelin-cluster/yarn - - diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 801cd645fe4..3400345e900 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -17,8 +17,12 @@ package org.apache.zeppelin.cluster.yarn; +import com.google.common.base.Joiner; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -26,34 +30,43 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.zeppelin.cluster.ApplicationCallbackHandler; import org.apache.zeppelin.cluster.ClusterManager; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterSetting; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; /** * */ public class Client extends ClusterManager { + private static final Logger logger = LoggerFactory.getLogger(Client.class); private static final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1); private Configuration configuration; private YarnClient yarnClient; - private boolean initialized; + private boolean started; + + private String interpreterLibDir; /** * `id` is a unique key to figure out Application @@ -65,11 +78,11 @@ public Client(ZeppelinConfiguration zeppelinConfiguration, ApplicationCallbackHandler applicationHandler) { super(zeppelinConfiguration, applicationHandler); - this.initialized = false; + this.started = false; } public synchronized void start() { - if (!initialized) { // it will help when calling it multiple times from different threads + if (!started) { // it will help when calling it multiple times from different threads logger.info("Start to initialize yarn client"); this.configuration = new Configuration(); this.yarnClient = YarnClient.createYarnClient(); @@ -80,15 +93,25 @@ public synchronized void start() { this.idApplicationIdMap = new ConcurrentHashMap<>(); - scheduledExecutorService + this.scheduledExecutorService .scheduleAtFixedRate(new ApplicationMonitor(), 1, 1, TimeUnit.SECONDS); - this.initialized = true; + this.interpreterLibDir = Joiner.on(Path.SEPARATOR).join(new String[] { + zeppelinConfiguration.getBasePath(), "zeppelin-interpreter", "target" + }); + + try { + Files.newDirectoryStream(Paths.get(zeppelinConfiguration.getBasePath(), "zeppelin-interpreter", "target"),"*.jar"); + } catch (IOException e) { + e.printStackTrace(); + } + + this.started = true; } } public synchronized void stop() { - if (initialized) { + if (started) { logger.info("Stop yarn client"); closeAllApplications(); @@ -96,20 +119,51 @@ public synchronized void stop() { scheduledExecutorService.shutdown(); this.yarnClient.stop(); - this.initialized = false; + this.started = false; } } - public Interpreter createInterpreter(String id, InterpreterSetting interpreterSetting) + @Override + public RemoteInterpreterProcess createInterpreter(String id, + InterpreterSetting interpreterSetting, int connectTimeout, + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener) throws InterpreterException { - if (!initialized) { + if (!started) { start(); } - return new RemoteInterpreter(); + + try { + YarnClientApplication app = yarnClient.createApplication(); + ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); + + // put this info idApplicationIdMap + ApplicationId applicationId = appContext.getApplicationId(); + + appContext.setKeepContainersAcrossApplicationAttempts(false); + appContext.setApplicationName(interpreterSetting.getName()); + + Map localResources = new HashMap<>(); + + FileSystem fileSystem = FileSystem.get(configuration); + + String interpreterDir = getInterpreterRelativeDir(interpreterSetting.getGroup()); + String interpreterLib = getInterpreterRelativeDir("lib"); + + return new RemoteInterpreterYarnProcess(connectTimeout, listener, appListener, yarnClient, + appContext); + } catch (YarnException | IOException e) { + throw new InterpreterException(e); + } + } + + private String getInterpreterRelativeDir(String dirName) { + return Joiner.on(Path.SEPARATOR).join( + new String[]{zeppelinConfiguration.getInterpreterDir(), dirName}); } public void releaseResource(String id) { - if (!initialized) { + if (!started) { start(); } ApplicationId applicationId = idApplicationIdMap.get(id); @@ -171,49 +225,50 @@ public void run() { YarnApplicationState curState = applicationReport.getYarnApplicationState(); YarnApplicationState oldState = appStatusMap.get(applicationId); switch (curState) { - case NEW: - case NEW_SAVING: - case SUBMITTED: - case ACCEPTED: - if (null == oldState) { - logger.info("new application added. Id: {}, applicationId: {}", id, applicationId); - } - appStatusMap.put(applicationId, curState); - break; - case RUNNING: - if (!YarnApplicationState.RUNNING.equals(oldState)) { - logger.info("id {} started with applicationId {}", id, applicationId); + case NEW: + case NEW_SAVING: + case SUBMITTED: + case ACCEPTED: + if (null == oldState) { + logger.info("new application added. Id: {}, applicationId: {}", id, + applicationId); + } appStatusMap.put(applicationId, curState); - applicationCallbackHandler - .onStarted(applicationReport.getHost(), applicationReport.getRpcPort()); - } - break; - case FINISHED: - if (!YarnApplicationState.FINISHED.equals(oldState)) { - logger.info("id {}, applicationId {} finished with final Status {}", id, - applicationId, applicationReport.getFinalApplicationStatus()); - applicationCallbackHandler.onTerminated(id); - removedIds.add(id); - } - break; - case FAILED: - if (!YarnApplicationState.FAILED.equals(oldState)) { - logger - .info("id {}, applicationId {} failed with final Status {}", id, applicationId, - applicationReport.getFinalApplicationStatus()); - applicationCallbackHandler.onTerminated(id); - removedIds.add(id); - } - break; - case KILLED: - if (!YarnApplicationState.KILLED.equals(oldState)) { - logger - .info("id {}, applicationId {} killed with final Status {}", id, applicationId, - applicationReport.getFinalApplicationStatus()); - applicationCallbackHandler.onTerminated(id); - removedIds.add(id); - } - break; + break; + case RUNNING: + if (!YarnApplicationState.RUNNING.equals(oldState)) { + logger.info("id {} started with applicationId {}", id, applicationId); + appStatusMap.put(applicationId, curState); + applicationCallbackHandler + .onStarted(applicationReport.getHost(), applicationReport.getRpcPort()); + } + break; + case FINISHED: + if (!YarnApplicationState.FINISHED.equals(oldState)) { + logger.info("id {}, applicationId {} finished with final Status {}", id, + applicationId, applicationReport.getFinalApplicationStatus()); + applicationCallbackHandler.onTerminated(id); + removedIds.add(id); + } + break; + case FAILED: + if (!YarnApplicationState.FAILED.equals(oldState)) { + logger + .info("id {}, applicationId {} failed with final Status {}", id, + applicationId, applicationReport.getFinalApplicationStatus()); + applicationCallbackHandler.onTerminated(id); + removedIds.add(id); + } + break; + case KILLED: + if (!YarnApplicationState.KILLED.equals(oldState)) { + logger + .info("id {}, applicationId {} killed with final Status {}", id, + applicationId, applicationReport.getFinalApplicationStatus()); + applicationCallbackHandler.onTerminated(id); + removedIds.add(id); + } + break; } } catch (YarnException | IOException e) { logger.debug("Error occurs while fetching status of {}", applicationId, e); diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java new file mode 100644 index 00000000000..694535bd44c --- /dev/null +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -0,0 +1,64 @@ +package org.apache.zeppelin.cluster.yarn; + +import java.io.IOException; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { + + private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterYarnProcess.class); + + private final YarnClient yarnClient; + private final ApplicationSubmissionContext appContext; + + public RemoteInterpreterYarnProcess(int connectTimeout, + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener, YarnClient yarnClient, + ApplicationSubmissionContext appContext) { + super(connectTimeout, listener, appListener); + this.yarnClient = yarnClient; + this.appContext = appContext; + } + + @Override + public String getHost() { + return null; + } + + @Override + public int getPort() { + return 0; + } + + @Override + public void start(String userName, Boolean isUserImpersonate) { + if (isUserImpersonate) { + + } + try { + yarnClient.submitApplication(appContext); + } catch (YarnException | IOException e) { + throw new InterpreterException(e); + } + } + + @Override + public void stop() { + + } + + @Override + public boolean isRunning() { + return false; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java index 7a504732002..1940dd1006e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -18,10 +18,15 @@ package org.apache.zeppelin.cluster; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +/** + * + */ public abstract class ClusterManager { protected final ZeppelinConfiguration zeppelinConfiguration; protected final ApplicationCallbackHandler applicationCallbackHandler; @@ -42,7 +47,10 @@ protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration, */ public abstract void stop(); - public abstract Interpreter createInterpreter(String id, InterpreterSetting interpreterSetting) + public abstract RemoteInterpreterProcess createInterpreter(String id, + InterpreterSetting interpreterSetting, int connectTimeout, + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener) throws InterpreterException; public abstract void releaseResource(String id); From 6eafe80ed0db94d2ed602e1e9bfefbedbe0218de Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 1 May 2017 17:57:40 +0900 Subject: [PATCH 11/52] remove zeppelin-cluster/yarn --- zeppelin-cluster/yarn/pom.xml | 125 ----------- .../yarn/RemoteInterpreterYarnProcess.java | 64 ------ .../yarn/src/main/resources/log4j.properties | 20 -- .../cluster/yarn/ApplicationMaster.java | 0 .../apache/zeppelin/cluster/yarn/Client.java | 207 ++++++++++-------- .../zeppelin/cluster/yarn/DSConstants.java | 0 .../yarn/RemoteInterpreterYarnProcess.java | 160 ++++++++++++++ .../zeppelin/cluster/yarn/SimpleYarnApp.java | 0 8 files changed, 276 insertions(+), 300 deletions(-) delete mode 100644 zeppelin-cluster/yarn/pom.xml delete mode 100644 zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java delete mode 100644 zeppelin-cluster/yarn/src/main/resources/log4j.properties rename {zeppelin-cluster/yarn => zeppelin-zengine}/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java (100%) rename {zeppelin-cluster/yarn => zeppelin-zengine}/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java (55%) rename {zeppelin-cluster/yarn => zeppelin-zengine}/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java (100%) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java rename {zeppelin-cluster/yarn => zeppelin-zengine}/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java (100%) diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml deleted file mode 100644 index 10043af5887..00000000000 --- a/zeppelin-cluster/yarn/pom.xml +++ /dev/null @@ -1,125 +0,0 @@ - - - - - - 4.0.0 - - - zeppelin - org.apache.zeppelin - 0.8.0-SNAPSHOT - ../.. - - - zeppelin-cluster-yarn - jar - 0.8.0-SNAPSHOT - Zeppelin Cluster: Yarn - - - - - org.codehaus.jackson - jackson-core-asl - 1.9.13 - - - org.codehaus.jackson - jackson-mapper-asl - 1.9.13 - - - org.codehaus.jackson - jackson-xc - 1.9.13 - - - org.codehaus.jackson - jackson-jaxrs - 1.9.13 - - - com.jcraft - jsch - 0.1.42 - - - org.apache.commons - commons-compress - 1.4.1 - - - - - - - ${project.groupId} - zeppelin-zengine - ${project.version} - - - org.apache.hadoop - hadoop-yarn-client - - - - org.apache.hadoop - hadoop-yarn-common - - - - org.apache.hadoop - hadoop-yarn-api - - - - org.apache.hadoop - hadoop-common - - - - - diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java deleted file mode 100644 index 694535bd44c..00000000000 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.apache.zeppelin.cluster.yarn; - -import java.io.IOException; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { - - private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterYarnProcess.class); - - private final YarnClient yarnClient; - private final ApplicationSubmissionContext appContext; - - public RemoteInterpreterYarnProcess(int connectTimeout, - RemoteInterpreterProcessListener listener, - ApplicationEventListener appListener, YarnClient yarnClient, - ApplicationSubmissionContext appContext) { - super(connectTimeout, listener, appListener); - this.yarnClient = yarnClient; - this.appContext = appContext; - } - - @Override - public String getHost() { - return null; - } - - @Override - public int getPort() { - return 0; - } - - @Override - public void start(String userName, Boolean isUserImpersonate) { - if (isUserImpersonate) { - - } - try { - yarnClient.submitApplication(appContext); - } catch (YarnException | IOException e) { - throw new InterpreterException(e); - } - } - - @Override - public void stop() { - - } - - @Override - public boolean isRunning() { - return false; - } -} diff --git a/zeppelin-cluster/yarn/src/main/resources/log4j.properties b/zeppelin-cluster/yarn/src/main/resources/log4j.properties deleted file mode 100644 index fb31a1e3417..00000000000 --- a/zeppelin-cluster/yarn/src/main/resources/log4j.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -log4j.rootLogger=DEBUG, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java similarity index 100% rename from zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java similarity index 55% rename from zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 3400345e900..9266dd696cb 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -17,41 +17,54 @@ package org.apache.zeppelin.cluster.yarn; -import com.google.common.base.Joiner; import java.io.IOException; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.zeppelin.cluster.ApplicationCallbackHandler; import org.apache.zeppelin.cluster.ClusterManager; import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer; /** * @@ -59,14 +72,14 @@ public class Client extends ClusterManager { private static final Logger logger = LoggerFactory.getLogger(Client.class); - private static final ScheduledExecutorService scheduledExecutorService = + public static final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1); private Configuration configuration; private YarnClient yarnClient; private boolean started; - private String interpreterLibDir; + private List interpreterRelatedPaths; /** * `id` is a unique key to figure out Application @@ -74,9 +87,8 @@ public class Client extends ClusterManager { private Map idApplicationIdMap; private Map appStatusMap; - public Client(ZeppelinConfiguration zeppelinConfiguration, - ApplicationCallbackHandler applicationHandler) { - super(zeppelinConfiguration, applicationHandler); + public Client(ZeppelinConfiguration zeppelinConfiguration) { + super(zeppelinConfiguration); this.started = false; } @@ -93,18 +105,10 @@ public synchronized void start() { this.idApplicationIdMap = new ConcurrentHashMap<>(); - this.scheduledExecutorService - .scheduleAtFixedRate(new ApplicationMonitor(), 1, 1, TimeUnit.SECONDS); - - this.interpreterLibDir = Joiner.on(Path.SEPARATOR).join(new String[] { - zeppelinConfiguration.getBasePath(), "zeppelin-interpreter", "target" - }); - - try { - Files.newDirectoryStream(Paths.get(zeppelinConfiguration.getBasePath(), "zeppelin-interpreter", "target"),"*.jar"); - } catch (IOException e) { - e.printStackTrace(); - } + this.interpreterRelatedPaths = Lists.newArrayList( + Paths.get(zeppelinConfiguration.getBasePath(), "zeppelin-interpreter", "target"), + Paths.get(zeppelinConfiguration.getBasePath(), "zeppelin-interpreter", "target", "lib"), + Paths.get(zeppelinConfiguration.getBasePath(), "lib", "interpreter")); this.started = true; } @@ -126,8 +130,7 @@ public synchronized void stop() { @Override public RemoteInterpreterProcess createInterpreter(String id, InterpreterSetting interpreterSetting, int connectTimeout, - RemoteInterpreterProcessListener listener, - ApplicationEventListener appListener) + RemoteInterpreterProcessListener listener, ApplicationEventListener appListener) throws InterpreterException { if (!started) { start(); @@ -143,12 +146,63 @@ public RemoteInterpreterProcess createInterpreter(String id, appContext.setKeepContainersAcrossApplicationAttempts(false); appContext.setApplicationName(interpreterSetting.getName()); + Map env = Maps.newHashMap(); + + ArrayList classpathStrings = Lists + .newArrayList(configuration.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)); + classpathStrings.add(0, "./*"); + classpathStrings.add(0, ApplicationConstants.Environment.CLASSPATH.$$()); + + String classpathEnv = + Joiner.on(ApplicationConstants.CLASS_PATH_SEPARATOR).join(classpathStrings); + + logger.debug("classpath: {}", classpathEnv); + + env.put("CLASSPATH", classpathEnv); + Map localResources = new HashMap<>(); FileSystem fileSystem = FileSystem.get(configuration); - String interpreterDir = getInterpreterRelativeDir(interpreterSetting.getGroup()); - String interpreterLib = getInterpreterRelativeDir("lib"); + java.nio.file.Path interpreterDir = getInterpreterRelativePath(interpreterSetting.getGroup()); + List interpreterPaths = getPathsFromDirPath(interpreterDir); + for (java.nio.file.Path p : interpreterRelatedPaths) { + interpreterPaths.addAll(getPathsFromDirPath(p)); + } + + addLocalResource(fileSystem, String.valueOf(applicationId.getId()), localResources, + interpreterPaths); + + List vargs = Lists.newArrayList(); + + vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); + + vargs.add("-Xmx1024m"); + + vargs.add(RemoteInterpreterServer.class.getName()); + + vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/interpreter.stdout"); + vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/interpreter.stderr"); + + String command = Joiner.on(" ").join(vargs); + logger.debug("command: {}", command); + + List commands = Lists.newArrayList(command); + + ContainerLaunchContext amContainer = + ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); + + Resource capability = Resource.newInstance(1024, 1); + appContext.setResource(capability); + + appContext.setAMContainerSpec(amContainer); + + Priority pri = Priority.newInstance(0); + appContext.setPriority(pri); + + appContext.setQueue("default"); + + appContext.setApplicationType("ZEPPELIN INTERPRETER"); return new RemoteInterpreterYarnProcess(connectTimeout, listener, appListener, yarnClient, appContext); @@ -157,9 +211,8 @@ public RemoteInterpreterProcess createInterpreter(String id, } } - private String getInterpreterRelativeDir(String dirName) { - return Joiner.on(Path.SEPARATOR).join( - new String[]{zeppelinConfiguration.getInterpreterDir(), dirName}); + private java.nio.file.Path getInterpreterRelativePath(String dirName) { + return Paths.get(zeppelinConfiguration.getInterpreterDir(), dirName); } public void releaseResource(String id) { @@ -212,71 +265,43 @@ private String getClientToken(ApplicationReport applicationReport) { } } - private class ApplicationMonitor implements Runnable { + private List getPathsFromDirPath(java.nio.file.Path dirPath) { + if (null == dirPath || Files.notExists(dirPath) || Files.isDirectory(dirPath)) { + return Lists.newArrayList(); + } - @Override - public void run() { - ArrayList removedIds = new ArrayList<>(); - for (Map.Entry entry : idApplicationIdMap.entrySet()) { - String id = entry.getKey(); - ApplicationId applicationId = entry.getValue(); - try { - ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId); - YarnApplicationState curState = applicationReport.getYarnApplicationState(); - YarnApplicationState oldState = appStatusMap.get(applicationId); - switch (curState) { - case NEW: - case NEW_SAVING: - case SUBMITTED: - case ACCEPTED: - if (null == oldState) { - logger.info("new application added. Id: {}, applicationId: {}", id, - applicationId); - } - appStatusMap.put(applicationId, curState); - break; - case RUNNING: - if (!YarnApplicationState.RUNNING.equals(oldState)) { - logger.info("id {} started with applicationId {}", id, applicationId); - appStatusMap.put(applicationId, curState); - applicationCallbackHandler - .onStarted(applicationReport.getHost(), applicationReport.getRpcPort()); - } - break; - case FINISHED: - if (!YarnApplicationState.FINISHED.equals(oldState)) { - logger.info("id {}, applicationId {} finished with final Status {}", id, - applicationId, applicationReport.getFinalApplicationStatus()); - applicationCallbackHandler.onTerminated(id); - removedIds.add(id); - } - break; - case FAILED: - if (!YarnApplicationState.FAILED.equals(oldState)) { - logger - .info("id {}, applicationId {} failed with final Status {}", id, - applicationId, applicationReport.getFinalApplicationStatus()); - applicationCallbackHandler.onTerminated(id); - removedIds.add(id); - } - break; - case KILLED: - if (!YarnApplicationState.KILLED.equals(oldState)) { - logger - .info("id {}, applicationId {} killed with final Status {}", id, - applicationId, applicationReport.getFinalApplicationStatus()); - applicationCallbackHandler.onTerminated(id); - removedIds.add(id); - } - break; - } - } catch (YarnException | IOException e) { - logger.debug("Error occurs while fetching status of {}", applicationId, e); - } + try { + DirectoryStream directoryStream = + Files.newDirectoryStream(dirPath, new DirectoryStream.Filter() { + @Override + public boolean accept(java.nio.file.Path entry) throws IOException { + return entry.endsWith(".jar"); + } + }); + return Lists.newArrayList(directoryStream); + } catch (IOException e) { + logger.error("Cannot read directory: {}", dirPath.toString(), e); + return Lists.newArrayList(); + } + } - for (String removedId : removedIds) { - idApplicationIdMap.remove(removedId); + private void addLocalResource(FileSystem fs, String appId, + Map localResourceMap, List paths) { + for (java.nio.file.Path path : paths) { + String resourcePath = appId + Path.SEPARATOR + path.getFileName().toString(); + Path dst = new Path(fs.getHomeDirectory(), resourcePath); + try { + if (Files.exists(path) && fs.exists(dst)) { + fs.copyFromLocalFile(new Path(path.toUri()), dst); + FileStatus fileStatus = fs.getFileStatus(dst); + LocalResource resource = LocalResource + .newInstance(ConverterUtils.getYarnUrlFromPath(dst), LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION, fileStatus.getLen(), + fileStatus.getModificationTime()); + localResourceMap.put(path.getFileName().toString(), resource); } + } catch (IOException e) { + logger.error("Error while copying resources into hdfs", e); } } } diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java similarity index 100% rename from zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java new file mode 100644 index 00000000000..4418cce68c1 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -0,0 +1,160 @@ +package org.apache.zeppelin.cluster.yarn; + +import java.io.IOException; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventPoller; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; + +/** + * + */ +public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { + + private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterYarnProcess.class); + + private final YarnClient yarnClient; + private final ApplicationSubmissionContext appContext; + private final ApplicationId applicationId; + + private boolean isRunning = false; + private ScheduledFuture monitor; + private YarnApplicationState oldState; + + private String host = null; + private int port = -1; + + public RemoteInterpreterYarnProcess(int connectTimeout, RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener, YarnClient yarnClient, + ApplicationSubmissionContext appContext) { + super(new RemoteInterpreterEventPoller(listener, appListener), connectTimeout); + this.yarnClient = yarnClient; + this.appContext = appContext; + this.applicationId = appContext.getApplicationId(); + } + + @Override + public String getHost() { + return host; + } + + private void setHost(String host) { + this.host = host; + } + + @Override + public int getPort() { + return port; + } + + private void setPort(int port) { + this.port = port; + } + + @Override + public void start(String userName, Boolean isUserImpersonate) { + if (isUserImpersonate) { + + } + try { + yarnClient.submitApplication(appContext); + monitor = Client.scheduledExecutorService + .scheduleAtFixedRate(new ApplicationMonitor(), 1, 1, TimeUnit.SECONDS); + while (!isRunning()) { + Thread.sleep(500); + logger.debug("Waiting until connected"); + } + } catch (YarnException | IOException | InterruptedException e) { + throw new InterpreterException(e); + } + } + + @Override + public void stop() { + isRunning = false; + monitor.cancel(false); + } + + @Override + public boolean isRunning() { + return isRunning; + } + + private void setRunning(boolean running) { + this.isRunning = running; + } + + private class ApplicationMonitor implements Runnable { + + @Override + public void run() { + try { + ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId); + YarnApplicationState curState = applicationReport.getYarnApplicationState(); + switch (curState) { + case NEW: + case NEW_SAVING: + case SUBMITTED: + case ACCEPTED: + if (null == oldState) { + logger.info("new application added. applicationId: {}", applicationId); + } + oldState = curState; + break; + case RUNNING: + if (!YarnApplicationState.RUNNING.equals(oldState)) { + String host = applicationReport.getHost(); + int port = applicationReport.getRpcPort(); + logger + .info("applicationId {} started. Host: {}, port: {}", applicationId, host + , port); + oldState = curState; + setHost(host); + setPort(port); + setRunning(true); + } + break; + case FINISHED: + if (!YarnApplicationState.FINISHED.equals(oldState)) { + logger.info("applicationId {} finished with final Status {}", applicationId, + applicationReport.getFinalApplicationStatus()); + setRunning(false); + //TODO(jl): Handle it!! + } + break; + case FAILED: + if (!YarnApplicationState.FAILED.equals(oldState)) { + logger.info("id {}, applicationId {} failed with final Status {}", applicationId, + applicationReport.getFinalApplicationStatus()); + setRunning(false); + //TODO(jl): Handle it!! + } + break; + case KILLED: + if (!YarnApplicationState.KILLED.equals(oldState)) { + logger.info("applicationId {} killed with final Status {}", applicationId, + applicationReport.getFinalApplicationStatus()); + setRunning(false); + //TODO(jl): Handle it!! + } + break; + } + } catch (YarnException | IOException e) { + logger.debug("Error occurs while fetching status of {}", applicationId, e); + } + } + } +} diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java similarity index 100% rename from zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java From b83715cfe70d5daba70360d2f530f502b3c276c5 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Tue, 2 May 2017 08:56:43 +0900 Subject: [PATCH 12/52] WIP --- pom.xml | 26 ----- .../remote/RemoteInterpreterProcess.java | 2 +- .../zeppelin/server/ZeppelinServer.java | 42 ++++---- zeppelin-zengine/pom.xml | 97 +++++++++++++++++++ .../zeppelin/cluster/ClusterManager.java | 5 +- .../yarn/RemoteInterpreterYarnProcess.java | 10 +- .../interpreter/InterpreterFactory.java | 15 ++- .../helium/HeliumApplicationFactoryTest.java | 2 +- .../interpreter/InterpreterFactoryTest.java | 12 +-- .../notebook/NoteInterpreterLoaderTest.java | 2 +- .../zeppelin/notebook/NotebookTest.java | 4 +- .../notebook/repo/NotebookRepoSyncTest.java | 2 +- .../notebook/repo/VFSNotebookRepoTest.java | 2 +- 13 files changed, 154 insertions(+), 67 deletions(-) diff --git a/pom.xml b/pom.xml index 2f370ae095b..1c173016f55 100644 --- a/pom.xml +++ b/pom.xml @@ -110,8 +110,6 @@ 1.3.1 1.3.2 - 2.7.3 - 4.12 1.10.19 @@ -267,30 +265,6 @@ ${shiro.version} - - org.apache.hadoop - hadoop-yarn-client - ${hadoop.version} - - - - org.apache.hadoop - hadoop-yarn-common - ${hadoop.version} - - - - org.apache.hadoop - hadoop-yarn-api - ${hadoop.version} - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - junit diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 1d48a1e6f6e..248d649aa24 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -48,7 +48,7 @@ public RemoteInterpreterProcess( connectTimeout); } - RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller, + protected RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller, int connectTimeout) { this.interpreterContextRunnerPool = new InterpreterContextRunnerPool(); referenceCount = new AtomicInteger(0); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 745347048a2..d50e8b02f82 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -22,13 +22,31 @@ import java.util.EnumSet; import java.util.HashSet; import java.util.Set; - import javax.servlet.DispatcherType; import javax.ws.rs.core.Application; import org.apache.commons.lang.StringUtils; import org.apache.shiro.web.env.EnvironmentLoaderListener; import org.apache.shiro.web.servlet.ShiroFilter; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.server.handler.ContextHandlerCollection; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.servlet.DefaultServlet; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.eclipse.jetty.webapp.WebAppContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.zeppelin.cluster.ClusterManager; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.DependencyResolver; @@ -57,23 +75,6 @@ import org.apache.zeppelin.socket.NotebookServer; import org.apache.zeppelin.user.Credentials; import org.apache.zeppelin.utils.SecurityUtils; -import org.eclipse.jetty.http.HttpVersion; -import org.eclipse.jetty.server.HttpConfiguration; -import org.eclipse.jetty.server.HttpConnectionFactory; -import org.eclipse.jetty.server.SecureRequestCustomizer; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.SslConnectionFactory; -import org.eclipse.jetty.server.handler.ContextHandlerCollection; -import org.eclipse.jetty.server.session.SessionHandler; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; -import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.eclipse.jetty.webapp.WebAppContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Main class of Zeppelin. @@ -87,6 +88,7 @@ public class ZeppelinServer extends Application { public static Helium helium; private final InterpreterSettingManager interpreterSettingManager; + private final ClusterManager clusterManager; private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; private SearchService noteSearchService; @@ -147,9 +149,11 @@ public ZeppelinServer() throws Exception { this.schedulerFactory = new SchedulerFactory(); this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + this.clusterManager = (ClusterManager) Class.forName("org.apache.zeppelin.cluster.yarn.Client") + .getConstructor(ZeppelinConfiguration.class).newInstance(conf); this.replFactory = new InterpreterFactory(conf, notebookWsServer, notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(), - interpreterSettingManager); + interpreterSettingManager, clusterManager); this.notebookRepo = new NotebookRepoSync(conf); this.noteSearchService = new LuceneSearch(); this.notebookAuthorization = NotebookAuthorization.init(conf); diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 38b1e830068..c7a2af544dc 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -47,6 +47,7 @@ 1.4.01 4.1.1.201511131810-r 1.3 + 2.6.0 0.27 @@ -59,6 +60,102 @@ ${project.version} + + org.apache.hadoop + hadoop-yarn-client + ${hadoop.version} + + + + org.apache.hadoop + hadoop-yarn-common + ${hadoop.version} + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + org.apache.commons + commons-compress + + + org.codehaus.jackson + jackson-mapper-asl + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-core + + + + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop.version} + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + javax.servlet + servlet-api + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + com.jcraft + jsch + + + org.apache.commons + commons-compress + + + org.codehaus.jackson + jackson-mapper-asl + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + + org.slf4j slf4j-api diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java index 1940dd1006e..7d576c46ced 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -29,12 +29,9 @@ */ public abstract class ClusterManager { protected final ZeppelinConfiguration zeppelinConfiguration; - protected final ApplicationCallbackHandler applicationCallbackHandler; - protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration, - ApplicationCallbackHandler applicationCallbackHandler) { + protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration) { this.zeppelinConfiguration = zeppelinConfiguration; - this.applicationCallbackHandler = applicationCallbackHandler; } /** diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index 4418cce68c1..d60f21fc3d8 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -19,6 +19,8 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*; + /** * */ @@ -115,7 +117,7 @@ public void run() { oldState = curState; break; case RUNNING: - if (!YarnApplicationState.RUNNING.equals(oldState)) { + if (!RUNNING.equals(oldState)) { String host = applicationReport.getHost(); int port = applicationReport.getRpcPort(); logger @@ -128,7 +130,7 @@ public void run() { } break; case FINISHED: - if (!YarnApplicationState.FINISHED.equals(oldState)) { + if (!FINISHED.equals(oldState)) { logger.info("applicationId {} finished with final Status {}", applicationId, applicationReport.getFinalApplicationStatus()); setRunning(false); @@ -136,7 +138,7 @@ public void run() { } break; case FAILED: - if (!YarnApplicationState.FAILED.equals(oldState)) { + if (!FAILED.equals(oldState)) { logger.info("id {}, applicationId {} failed with final Status {}", applicationId, applicationReport.getFinalApplicationStatus()); setRunning(false); @@ -144,7 +146,7 @@ public void run() { } break; case KILLED: - if (!YarnApplicationState.KILLED.equals(oldState)) { + if (!KILLED.equals(oldState)) { logger.info("applicationId {} killed with final Status {}", applicationId, applicationReport.getFinalApplicationStatus()); setRunning(false); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index bbc2b3b39ea..a2f77aa1653 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -20,6 +20,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import org.apache.commons.lang.NullArgumentException; +import org.apache.zeppelin.cluster.ClusterManager; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.DependencyResolver; @@ -64,6 +65,8 @@ public class InterpreterFactory implements InterpreterGroupFactory { private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; private final ApplicationEventListener appEventListener; + private final ClusterManager clusterManager; + private boolean shiroEnabled; private Map env = new HashMap<>(); @@ -74,7 +77,8 @@ public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appEventListener, DependencyResolver depResolver, - boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager) + boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager, + ClusterManager clusterManager) throws InterpreterException, IOException, RepositoryException { this.conf = conf; this.angularObjectRegistryListener = angularObjectRegistryListener; @@ -86,6 +90,8 @@ public InterpreterFactory(ZeppelinConfiguration conf, //TODO(jl): Fix it not to use InterpreterGroupFactory interpreterSettingManager.setInterpreterGroupFactory(this); + this.clusterManager = clusterManager; + logger.info("shiroEnabled: {}", shiroEnabled); } @@ -178,6 +184,13 @@ public void createInterpretersForNote(InterpreterSetting interpreterSetting, Str } } logger.info("Interpreter {} {} created", interpreter.getClassName(), interpreter.hashCode()); + + //Testing only + int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); + interpreterGroup.setRemoteInterpreterProcess(clusterManager + .createInterpreter("uniq", interpreterSetting, connectTimeout, + remoteInterpreterProcessListener, appEventListener)); + interpreter.setInterpreterGroup(interpreterGroup); } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 2eef06dfdbf..bb703946c18 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -85,7 +85,7 @@ public void setUp() throws Exception { heliumAppFactory = new HeliumApplicationFactory(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager, null); HashMap env = new HashMap<>(); env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); factory.setEnv(env); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 2e3812c0980..4309ee688d1 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -82,7 +82,7 @@ public void setUp() throws Exception { schedulerFactory = new SchedulerFactory(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null); context = new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null); ArrayList interpreterInfos = new ArrayList<>(); @@ -150,7 +150,7 @@ public void testRemoteRepl() throws Exception { intp1Properties.put("PROPERTY_1", "VALUE_1"); intp1Properties.put("property_2", "value_2"); interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList(), new InterpreterOption(true), intp1Properties); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null); List all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { @@ -187,7 +187,7 @@ public void testRestartInterpreterInScopedMode() throws Exception { intp1Properties.put("PROPERTY_1", "VALUE_1"); intp1Properties.put("property_2", "value_2"); interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList(), new InterpreterOption(true), intp1Properties); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null); List all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { @@ -232,7 +232,7 @@ public void testRestartInterpreterInIsolatedMode() throws Exception { intp1Properties.put("PROPERTY_1", "VALUE_1"); intp1Properties.put("property_2", "value_2"); interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList(), new InterpreterOption(true), intp1Properties); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null); List all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { @@ -341,7 +341,7 @@ public void testInterpreterSettingPropertyClass() throws IOException, Repository @Test public void testInterpreterAliases() throws IOException, RepositoryException { interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null); final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null); final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null); interpreterSettingManager.add("group1", new ArrayList() {{ @@ -366,7 +366,7 @@ public void testInterpreterAliases() throws IOException, RepositoryException { @Test public void testMultiUser() throws IOException, RepositoryException { interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager, null); final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null); interpreterSettingManager.add("group1", new ArrayList(){{ add(info1); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 9c1096dcfc4..e28eb8f11cb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -64,7 +64,7 @@ public void setUp() throws Exception { depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null); ArrayList interpreterInfos = new ArrayList<>(); interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, Maps.newHashMap())); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 495e17b163b..c4f052f3607 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -92,7 +92,7 @@ public void setUp() throws Exception { depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null); ArrayList interpreterInfos = new ArrayList<>(); interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap())); @@ -259,7 +259,7 @@ public void testPersist() throws IOException, SchedulerException, RepositoryExce Notebook notebook2 = new Notebook( conf, notebookRepo, schedulerFactory, - new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager), + new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null), interpreterSettingManager, null, null, null, null); assertEquals(1, notebook2.getAllNotes().size()); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 803912e61a7..b97f92a17b3 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -95,7 +95,7 @@ public void setUp() throws Exception { depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null); search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index c361713e271..a586107ade7 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -91,7 +91,7 @@ public void setUp() throws Exception { this.schedulerFactory = new SchedulerFactory(); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager, null); ArrayList interpreterInfos = new ArrayList<>(); interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap())); From 826e5689e516323ae3c261646dd0e7f40afa53ad Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 4 May 2017 01:10:52 +0900 Subject: [PATCH 13/52] POC in my local --- .../remote/RemoteInterpreterServer.java | 4 +- zeppelin-zengine/pom.xml | 8 ++ .../apache/zeppelin/cluster/yarn/Client.java | 40 ++++-- .../yarn/RemoteInterpreterYarnProcess.java | 30 +++- .../yarn/YarnRemoteInterpreterServer.java | 133 ++++++++++++++++++ .../zeppelin/conf/ZeppelinConfiguration.java | 4 + .../interpreter/InterpreterFactory.java | 10 +- 7 files changed, 204 insertions(+), 25 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 738aed55015..51fa6b68381 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -115,7 +115,7 @@ public class RemoteInterpreterServer private int callbackPort; private String host; private int port; - private TThreadPoolServer server; + protected TThreadPoolServer server; RemoteInterpreterEventClient eventClient = new RemoteInterpreterEventClient(); private DependencyResolver depLoader; @@ -228,7 +228,7 @@ public boolean isRunning() { public static void main(String[] args) - throws TTransportException, InterruptedException, IOException { + throws TTransportException, InterruptedException, Exception { String callbackHost = null; int port = Constants.ZEPPELIN_INTERPRETER_DEFAUlT_PORT; if (args.length > 0) { diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index c7a2af544dc..b33aae0f231 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -103,6 +103,10 @@ com.sun.jersey jersey-core + + javax.servlet + servlet-api + @@ -153,6 +157,10 @@ com.sun.jersey jersey-core + + javax.servlet + servlet-api + diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 9266dd696cb..594c3354689 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -28,7 +28,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import com.google.common.base.Joiner; import com.google.common.collect.Lists; @@ -64,7 +63,6 @@ import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer; /** * @@ -106,9 +104,12 @@ public synchronized void start() { this.idApplicationIdMap = new ConcurrentHashMap<>(); this.interpreterRelatedPaths = Lists.newArrayList( - Paths.get(zeppelinConfiguration.getBasePath(), "zeppelin-interpreter", "target"), - Paths.get(zeppelinConfiguration.getBasePath(), "zeppelin-interpreter", "target", "lib"), - Paths.get(zeppelinConfiguration.getBasePath(), "lib", "interpreter")); + Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target") + .toAbsolutePath(), + Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target", "lib") + .toAbsolutePath(), + Paths.get(zeppelinConfiguration.getHome(), "lib", "interpreter").toAbsolutePath(), + Paths.get(zeppelinConfiguration.getHome(), "zeppelin-zengine", "target")); this.started = true; } @@ -148,10 +149,12 @@ public RemoteInterpreterProcess createInterpreter(String id, Map env = Maps.newHashMap(); - ArrayList classpathStrings = Lists - .newArrayList(configuration.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)); + ArrayList classpathStrings = Lists.newArrayList(configuration + .getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, + YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)); classpathStrings.add(0, "./*"); classpathStrings.add(0, ApplicationConstants.Environment.CLASSPATH.$$()); + classpathStrings.add("${SPARK_HOME}/jars/*"); String classpathEnv = Joiner.on(ApplicationConstants.CLASS_PATH_SEPARATOR).join(classpathStrings); @@ -159,6 +162,7 @@ public RemoteInterpreterProcess createInterpreter(String id, logger.debug("classpath: {}", classpathEnv); env.put("CLASSPATH", classpathEnv); + env.put("SPARK_HOME", "/Users/jl/local/src/spark-2.1.0-bin-hadoop2.7"); Map localResources = new HashMap<>(); @@ -166,6 +170,10 @@ public RemoteInterpreterProcess createInterpreter(String id, java.nio.file.Path interpreterDir = getInterpreterRelativePath(interpreterSetting.getGroup()); List interpreterPaths = getPathsFromDirPath(interpreterDir); + /*if (interpreterSetting.getGroup().equals("spark")) { + interpreterDir = getInterpreterRelativePath("spark/dep"); + interpreterPaths.addAll(getPathsFromDirPath(interpreterDir)); + }*/ for (java.nio.file.Path p : interpreterRelatedPaths) { interpreterPaths.addAll(getPathsFromDirPath(p)); } @@ -179,7 +187,7 @@ public RemoteInterpreterProcess createInterpreter(String id, vargs.add("-Xmx1024m"); - vargs.add(RemoteInterpreterServer.class.getName()); + vargs.add(YarnRemoteInterpreterServer.class.getName()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/interpreter.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/interpreter.stderr"); @@ -266,7 +274,7 @@ private String getClientToken(ApplicationReport applicationReport) { } private List getPathsFromDirPath(java.nio.file.Path dirPath) { - if (null == dirPath || Files.notExists(dirPath) || Files.isDirectory(dirPath)) { + if (null == dirPath || Files.notExists(dirPath) || !Files.isDirectory(dirPath)) { return Lists.newArrayList(); } @@ -275,7 +283,8 @@ private List getPathsFromDirPath(java.nio.file.Path dirPath) Files.newDirectoryStream(dirPath, new DirectoryStream.Filter() { @Override public boolean accept(java.nio.file.Path entry) throws IOException { - return entry.endsWith(".jar"); + String filename = entry.toString(); + return filename.endsWith(".jar") || filename.endsWith(".zip"); } }); return Lists.newArrayList(directoryStream); @@ -291,14 +300,19 @@ private void addLocalResource(FileSystem fs, String appId, String resourcePath = appId + Path.SEPARATOR + path.getFileName().toString(); Path dst = new Path(fs.getHomeDirectory(), resourcePath); try { - if (Files.exists(path) && fs.exists(dst)) { + if (Files.exists(path) && !fs.exists(dst)) { fs.copyFromLocalFile(new Path(path.toUri()), dst); FileStatus fileStatus = fs.getFileStatus(dst); + LocalResourceType localResourceType = LocalResourceType.FILE; + String filename = path.getFileName().toString(); + if (filename.endsWith(".zip")) { + localResourceType = LocalResourceType.ARCHIVE; + } LocalResource resource = LocalResource - .newInstance(ConverterUtils.getYarnUrlFromPath(dst), LocalResourceType.FILE, + .newInstance(ConverterUtils.getYarnUrlFromPath(dst), localResourceType, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime()); - localResourceMap.put(path.getFileName().toString(), resource); + localResourceMap.put(filename, resource); } } catch (IOException e) { logger.error("Error while copying resources into hdfs", e); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index d60f21fc3d8..e8fd88ed513 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -1,6 +1,7 @@ package org.apache.zeppelin.cluster.yarn; import java.io.IOException; +import java.net.InetAddress; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -19,7 +20,10 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*; +import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FAILED; +import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED; +import static org.apache.hadoop.yarn.api.records.YarnApplicationState.KILLED; +import static org.apache.hadoop.yarn.api.records.YarnApplicationState.RUNNING; /** * @@ -30,8 +34,8 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { private final YarnClient yarnClient; private final ApplicationSubmissionContext appContext; - private final ApplicationId applicationId; + private ApplicationId applicationId; private boolean isRunning = false; private ScheduledFuture monitor; private YarnApplicationState oldState; @@ -45,7 +49,6 @@ public RemoteInterpreterYarnProcess(int connectTimeout, RemoteInterpreterProcess super(new RemoteInterpreterEventPoller(listener, appListener), connectTimeout); this.yarnClient = yarnClient; this.appContext = appContext; - this.applicationId = appContext.getApplicationId(); } @Override @@ -72,10 +75,14 @@ public void start(String userName, Boolean isUserImpersonate) { } try { + this.applicationId = appContext.getApplicationId(); yarnClient.submitApplication(appContext); monitor = Client.scheduledExecutorService .scheduleAtFixedRate(new ApplicationMonitor(), 1, 1, TimeUnit.SECONDS); while (!isRunning()) { + if (oldState == FINISHED || oldState == FAILED || oldState == KILLED) { + throw new YarnException("Failed to initialize yarn application: " + applicationId); + } Thread.sleep(500); logger.debug("Waiting until connected"); } @@ -87,6 +94,14 @@ public void start(String userName, Boolean isUserImpersonate) { @Override public void stop() { isRunning = false; + if (null != oldState && oldState != FINISHED && oldState != FAILED && oldState != KILLED) { + try { + yarnClient.killApplication(applicationId); + } catch (YarnException | IOException e) { + logger.debug("error while killing application: {}", applicationId); + } + } + oldState = null; monitor.cancel(false); } @@ -133,7 +148,8 @@ public void run() { if (!FINISHED.equals(oldState)) { logger.info("applicationId {} finished with final Status {}", applicationId, applicationReport.getFinalApplicationStatus()); - setRunning(false); + oldState = FINISHED; + stop(); //TODO(jl): Handle it!! } break; @@ -141,7 +157,8 @@ public void run() { if (!FAILED.equals(oldState)) { logger.info("id {}, applicationId {} failed with final Status {}", applicationId, applicationReport.getFinalApplicationStatus()); - setRunning(false); + oldState = FAILED; + stop(); //TODO(jl): Handle it!! } break; @@ -149,7 +166,8 @@ public void run() { if (!KILLED.equals(oldState)) { logger.info("applicationId {} killed with final Status {}", applicationId, applicationReport.getFinalApplicationStatus()); - setRunning(false); + oldState = KILLED; + stop(); //TODO(jl): Handle it!! } break; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java new file mode 100644 index 00000000000..8d4c0672ce0 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.cluster.yarn; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; + +import static org.apache.hadoop.yarn.api.records.FinalApplicationStatus.SUCCEEDED; + +/** + * + */ +public class YarnRemoteInterpreterServer extends RemoteInterpreterServer { + private static final Logger logger = LoggerFactory.getLogger(YarnRemoteInterpreterServer.class); + + private Configuration configuration; + private AMRMClientAsync amrmClientAsync; + + public YarnRemoteInterpreterServer(int port) throws TTransportException { + super(port); + configuration = new Configuration(); + } + + public static void main(String[] args) throws Exception { + int port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + YarnRemoteInterpreterServer yarnRemoteInterpreterServer = new YarnRemoteInterpreterServer(port); + yarnRemoteInterpreterServer.start(); + + yarnRemoteInterpreterServer.join(); + System.exit(0); + } + + @Override + public void run() { + logger.info("Start AMRMClient"); + amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(1000, new AMRMCallbackHandler()); + amrmClientAsync.init(configuration); + amrmClientAsync.start(); + + try { + String host = InetAddress.getLocalHost().getHostName(); + logger.info("Host: {}, port: {}", host, getPort()); + amrmClientAsync + .registerApplicationMaster(host, getPort(), "http://localhost:8080"); + } catch (YarnException | IOException e) { + logger.error("Error on registering application. Will shutdown"); + try { + shutdown(); + } catch (TException e1) { + logger.error("Never reach at", e1); + } + } + + super.run(); + } + + @Override + public void shutdown() throws TException { + try { + amrmClientAsync.unregisterApplicationMaster(SUCCEEDED, "Shutdown successfully", null); + } catch (IOException | YarnException e) { + logger.error("Error while unregistering application", e); + } + super.shutdown(); + + } + + private class AMRMCallbackHandler implements AMRMClientAsync.CallbackHandler { + @Override + public void onContainersCompleted(List list) { + logger.error("Never called"); + } + + @Override + public void onContainersAllocated(List list) { + logger.error("Never called"); + } + + @Override + public void onShutdownRequest() { + try { + shutdown(); + } catch (TException e) { + logger.error("Error while shutting down", e); + } + } + + @Override + public void onNodesUpdated(List list) { + logger.debug("onNoodesUpdated. {}", list); + } + + @Override + public float getProgress() { + return 0.5f; + } + + @Override + public void onError(Throwable throwable) { + logger.error("Error on Application", throwable); + } + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index ade1fbd2c37..0dafc156f57 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -477,6 +477,10 @@ public String getRelativeDir(String path) { } } + public String getHome() { + return getString(ConfVars.ZEPPELIN_HOME); + } + public boolean isWindowsPath(String path){ return path.matches("^[A-Za-z]:\\\\.*"); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index a2f77aa1653..4980f9894f0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -186,10 +186,12 @@ public void createInterpretersForNote(InterpreterSetting interpreterSetting, Str logger.info("Interpreter {} {} created", interpreter.getClassName(), interpreter.hashCode()); //Testing only - int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); - interpreterGroup.setRemoteInterpreterProcess(clusterManager - .createInterpreter("uniq", interpreterSetting, connectTimeout, - remoteInterpreterProcessListener, appEventListener)); + if (null == interpreterGroup.getRemoteInterpreterProcess()) { + int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); + interpreterGroup.setRemoteInterpreterProcess(clusterManager + .createInterpreter("uniq", interpreterSetting, connectTimeout, + remoteInterpreterProcessListener, appEventListener)); + } interpreter.setInterpreterGroup(interpreterGroup); } From 3d754138986bdfdc9b0d843f57f277a1b2cc0c07 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 8 May 2017 00:49:37 +0900 Subject: [PATCH 14/52] Fixed rat issue --- .../zeppelin/interpreter/Constants.java | 11 +++++-- .../zeppelin/cluster/ClusterManager.java | 5 ++- .../apache/zeppelin/cluster/yarn/Client.java | 10 +++--- .../yarn/RemoteInterpreterYarnProcess.java | 17 ++++++++++ .../zeppelin/conf/ZeppelinConfiguration.java | 3 +- .../interpreter/InterpreterFactory.java | 18 +++++------ .../interpreter/remote/RemoteInterpreter.java | 31 ++++++++++++++++--- 7 files changed, 68 insertions(+), 27 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java index 87748fffabc..5c54a7e1c3f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java @@ -23,10 +23,9 @@ /** * Interpreter related constants - * - * */ public class Constants { + public static final String ZEPPELIN_INTERPRETER_PORT = "zeppelin.interpreter.port"; public static final String ZEPPELIN_INTERPRETER_HOST = "zeppelin.interpreter.host"; @@ -37,6 +36,14 @@ public class Constants { public static final int ZEPPELIN_INTERPRETER_OUTPUT_LIMIT = 1024 * 100; + public static final String ZEPPELIN_CLUSTER_MANAGER_KEY = "zeppelin.cluster_manager"; + + public static final String ZEPPELIN_CLUSTER_MANAGER_YARN = "yarn"; + + public static final String ZEPPELIN_CLUSTER_MANAGER_LOCAL = "local"; + + public static final String ZEPPELIN_YARN_PROPERTY_KEY_PREFIX = "zeppelin.yarn."; + public static final Map TIME_SUFFIXES; static { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java index 7d576c46ced..dde17b504a6 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -44,9 +44,8 @@ protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration) { */ public abstract void stop(); - public abstract RemoteInterpreterProcess createInterpreter(String id, - InterpreterSetting interpreterSetting, int connectTimeout, - RemoteInterpreterProcessListener listener, + public abstract RemoteInterpreterProcess createInterpreter(String id, String name, + String groupName, int connectTimeout, RemoteInterpreterProcessListener listener, ApplicationEventListener appListener) throws InterpreterException; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 594c3354689..bca64106faf 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -129,9 +129,9 @@ public synchronized void stop() { } @Override - public RemoteInterpreterProcess createInterpreter(String id, - InterpreterSetting interpreterSetting, int connectTimeout, - RemoteInterpreterProcessListener listener, ApplicationEventListener appListener) + public RemoteInterpreterProcess createInterpreter(String id, String name, String groupName, + int connectTimeout, RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener) throws InterpreterException { if (!started) { start(); @@ -145,7 +145,7 @@ public RemoteInterpreterProcess createInterpreter(String id, ApplicationId applicationId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(false); - appContext.setApplicationName(interpreterSetting.getName()); + appContext.setApplicationName(name); Map env = Maps.newHashMap(); @@ -168,7 +168,7 @@ public RemoteInterpreterProcess createInterpreter(String id, FileSystem fileSystem = FileSystem.get(configuration); - java.nio.file.Path interpreterDir = getInterpreterRelativePath(interpreterSetting.getGroup()); + java.nio.file.Path interpreterDir = getInterpreterRelativePath(groupName); List interpreterPaths = getPathsFromDirPath(interpreterDir); /*if (interpreterSetting.getGroup().equals("spark")) { interpreterDir = getInterpreterRelativePath("spark/dep"); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index e8fd88ed513..f78f0c769c3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.cluster.yarn; import java.io.IOException; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 0dafc156f57..e3df78a46da 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -668,7 +668,8 @@ public static enum ConfVars { ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true), ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"), ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED("zeppelin.server.default.dir.allowed", false), - ZEPPELIN_SERVER_JETTY_NAME("zeppelin.server.jetty.name", null); + ZEPPELIN_SERVER_JETTY_NAME("zeppelin.server.jetty.name", null), + ZEPPELIN_CLUSTER_DEFAULT("zeppelin.cluster.default", "local"); private String varName; @SuppressWarnings("rawtypes") diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 4980f9894f0..b13ca217f6d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -155,6 +155,8 @@ public void createInterpretersForNote(InterpreterSetting interpreterSetting, Str List interpreterInfos = interpreterSetting.getInterpreterInfos(); String path = interpreterSetting.getPath(); InterpreterRunner runner = interpreterSetting.getInterpreterRunner(); + String name = interpreterSetting.getName(); + String group = interpreterSetting.getGroup(); Interpreter interpreter; for (InterpreterInfo info : interpreterInfos) { if (option.isRemote()) { @@ -185,14 +187,6 @@ public void createInterpretersForNote(InterpreterSetting interpreterSetting, Str } logger.info("Interpreter {} {} created", interpreter.getClassName(), interpreter.hashCode()); - //Testing only - if (null == interpreterGroup.getRemoteInterpreterProcess()) { - int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); - interpreterGroup.setRemoteInterpreterProcess(clusterManager - .createInterpreter("uniq", interpreterSetting, connectTimeout, - remoteInterpreterProcessListener, appEventListener)); - } - interpreter.setInterpreterGroup(interpreterGroup); } } @@ -269,13 +263,14 @@ private Interpreter connectToRemoteRepl(String interpreterSessionKey, String cla } Interpreter createRemoteRepl(String interpreterPath, String interpreterSessionKey, - String className, Properties property, String interpreterSettingId, - String userName, Boolean isUserImpersonate, InterpreterRunner interpreterRunner) { + String className, Properties property, String interpreterSettingId, String userName, + Boolean isUserImpersonate, InterpreterRunner interpreterRunner) { int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId; int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE); String interpreterRunnerPath; String interpreterGroupName = interpreterSettingManager.get(interpreterSettingId).getName(); + String interpreterGroupStr = interpreterSettingManager.get(interpreterSettingId).getGroup(); if (null != interpreterRunner) { interpreterRunnerPath = interpreterRunner.getPath(); Path p = Paths.get(interpreterRunnerPath); @@ -291,7 +286,8 @@ Interpreter createRemoteRepl(String interpreterPath, String interpreterSessionKe new RemoteInterpreter(property, interpreterSessionKey, className, interpreterRunnerPath, interpreterPath, localRepoPath, connectTimeout, maxPoolSize, remoteInterpreterProcessListener, appEventListener, userName, isUserImpersonate, - conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName); + conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName, + clusterManager, interpreterGroupStr); remoteInterpreter.addEnv(env); return new LazyOpenInterpreter(remoteInterpreter); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 9cea693645f..c65daeb69a4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -20,6 +20,7 @@ import java.util.*; import org.apache.thrift.TException; +import org.apache.zeppelin.cluster.ClusterManager; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; @@ -64,6 +65,8 @@ public class RemoteInterpreter extends Interpreter { private Boolean isUserImpersonate; private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT; private String interpreterGroupName; + private ClusterManager clusterManager; + private String group; /** * Remote interpreter and manage interpreter process @@ -72,7 +75,7 @@ public RemoteInterpreter(Properties property, String sessionKey, String classNam String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout, int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appListener, String userName, Boolean isUserImpersonate, - int outputLimit, String interpreterGroupName) { + int outputLimit, String interpreterGroupName, ClusterManager clusterManager, String group) { super(property); this.sessionKey = sessionKey; this.className = className; @@ -89,6 +92,8 @@ public RemoteInterpreter(Properties property, String sessionKey, String classNam this.isUserImpersonate = isUserImpersonate; this.outputLimit = outputLimit; this.interpreterGroupName = interpreterGroupName; + this.clusterManager = clusterManager; + this.group = group; } @@ -175,10 +180,26 @@ public RemoteInterpreterProcess getInterpreterProcess() { host, port); } else { - // create new remote process - remoteProcess = new RemoteInterpreterManagedProcess( - interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, - remoteInterpreterProcessListener, applicationEventListener, interpreterGroupName); + String clusterManagerKey = getProperty(Constants.ZEPPELIN_CLUSTER_MANAGER_KEY); + if (null == clusterManagerKey) { + //TODO(jl): Set defult cluster manager + clusterManagerKey = Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL; + } + switch (clusterManagerKey) { + case Constants.ZEPPELIN_CLUSTER_MANAGER_YARN: + remoteProcess = clusterManager + .createInterpreter("uniq", interpreterGroupName, group, connectTimeout, + remoteInterpreterProcessListener, applicationEventListener); + break; + case Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL: + default: + // Default is local process + // create new remote process + remoteProcess = new RemoteInterpreterManagedProcess( + interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, + remoteInterpreterProcessListener, applicationEventListener, + interpreterGroupName); + } } intpGroup.setRemoteInterpreterProcess(remoteProcess); From 5e4eaf6ff0cbd84d55d19d7f982a4c9e55a0ff47 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 8 May 2017 01:23:33 +0900 Subject: [PATCH 15/52] Added license header --- .../cluster/ApplicationCallbackHandler.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java index 82e8debb02b..c42d9bc6790 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.cluster; /** From 6da48bb13fbe37b4f1aa9f7975a9a67aa7e1b266 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 10 May 2017 09:37:48 +0900 Subject: [PATCH 16/52] First version of yarn cluster manager --- .../zeppelin/interpreter/Constants.java | 20 ++ .../cluster/ApplicationCallbackHandler.java | 27 -- .../zeppelin/cluster/ClusterManager.java | 4 +- .../apache/zeppelin/cluster/yarn/Client.java | 149 +--------- .../yarn/RemoteInterpreterYarnProcess.java | 263 ++++++++++++++++-- .../zeppelin/cluster/yarn/YarnUtils.java | 71 +++++ .../interpreter/remote/RemoteInterpreter.java | 2 +- 7 files changed, 333 insertions(+), 203 deletions(-) delete mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java index 5c54a7e1c3f..f1e9c8a9bf8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java @@ -42,6 +42,26 @@ public class Constants { public static final String ZEPPELIN_CLUSTER_MANAGER_LOCAL = "local"; + public static final String ZEPPELIN_YARN_VCORES_KEY = "zeppelin.yarn.vcores"; + + public static final String ZEPPELIN_YARN_VCORES_DEFAULT = "1"; + + public static final String ZEPPELIN_YARN_MEMORY_KEY = "zeppelin.yarn.memory"; + + public static final String ZEPPELIN_YARN_MEMORY_DEFAULT = "1024"; + + public static final String ZEPPELIN_YARN_QUEUE_KEY = "zeppelin.yarn.queue"; + + public static final String ZEPPELIN_YARN_QUEUE_DEFAULT = "default"; + + public static final String ZEPPELIN_YARN_APPLICATION_TYPE_KEY = "zeppelin.yarn.application.type"; + + public static final String ZEPPELIN_YARN_APPLICATION_TYPE_DEFAULT = "ZEPPELIN INTERPRETER"; + + public static final String ZEPPELIN_YARN_PRIORITY_KEY = "zeppelin.yarn.priority"; + + public static final String ZEPPELIN_YARN_PRIORITY_DEFAULT = "0"; + public static final String ZEPPELIN_YARN_PROPERTY_KEY_PREFIX = "zeppelin.yarn."; public static final Map TIME_SUFFIXES; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java deleted file mode 100644 index c42d9bc6790..00000000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ApplicationCallbackHandler.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.cluster; - -/** - * - */ -public interface ApplicationCallbackHandler { - void onStarted(String host, int port); - - void onTerminated(String id); -} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java index dde17b504a6..53846dce0b2 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -17,6 +17,8 @@ package org.apache.zeppelin.cluster; +import java.util.Map; +import java.util.Properties; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterException; @@ -45,7 +47,7 @@ protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration) { public abstract void stop(); public abstract RemoteInterpreterProcess createInterpreter(String id, String name, - String groupName, int connectTimeout, RemoteInterpreterProcessListener listener, + String groupName, Map env, Properties properties, int connectTimeout, RemoteInterpreterProcessListener listener, ApplicationEventListener appListener) throws InterpreterException; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index bca64106faf..63b5058e849 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -77,13 +78,10 @@ public class Client extends ClusterManager { private YarnClient yarnClient; private boolean started; - private List interpreterRelatedPaths; - /** * `id` is a unique key to figure out Application */ private Map idApplicationIdMap; - private Map appStatusMap; public Client(ZeppelinConfiguration zeppelinConfiguration) { super(zeppelinConfiguration); @@ -103,14 +101,6 @@ public synchronized void start() { this.idApplicationIdMap = new ConcurrentHashMap<>(); - this.interpreterRelatedPaths = Lists.newArrayList( - Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target") - .toAbsolutePath(), - Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target", "lib") - .toAbsolutePath(), - Paths.get(zeppelinConfiguration.getHome(), "lib", "interpreter").toAbsolutePath(), - Paths.get(zeppelinConfiguration.getHome(), "zeppelin-zengine", "target")); - this.started = true; } } @@ -130,97 +120,15 @@ public synchronized void stop() { @Override public RemoteInterpreterProcess createInterpreter(String id, String name, String groupName, - int connectTimeout, RemoteInterpreterProcessListener listener, - ApplicationEventListener appListener) + Map env, Properties properties, int connectTimeout, + RemoteInterpreterProcessListener listener, ApplicationEventListener appListener) throws InterpreterException { if (!started) { start(); } - try { - YarnClientApplication app = yarnClient.createApplication(); - ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); - - // put this info idApplicationIdMap - ApplicationId applicationId = appContext.getApplicationId(); - - appContext.setKeepContainersAcrossApplicationAttempts(false); - appContext.setApplicationName(name); - - Map env = Maps.newHashMap(); - - ArrayList classpathStrings = Lists.newArrayList(configuration - .getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, - YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)); - classpathStrings.add(0, "./*"); - classpathStrings.add(0, ApplicationConstants.Environment.CLASSPATH.$$()); - classpathStrings.add("${SPARK_HOME}/jars/*"); - - String classpathEnv = - Joiner.on(ApplicationConstants.CLASS_PATH_SEPARATOR).join(classpathStrings); - - logger.debug("classpath: {}", classpathEnv); - - env.put("CLASSPATH", classpathEnv); - env.put("SPARK_HOME", "/Users/jl/local/src/spark-2.1.0-bin-hadoop2.7"); - - Map localResources = new HashMap<>(); - - FileSystem fileSystem = FileSystem.get(configuration); - - java.nio.file.Path interpreterDir = getInterpreterRelativePath(groupName); - List interpreterPaths = getPathsFromDirPath(interpreterDir); - /*if (interpreterSetting.getGroup().equals("spark")) { - interpreterDir = getInterpreterRelativePath("spark/dep"); - interpreterPaths.addAll(getPathsFromDirPath(interpreterDir)); - }*/ - for (java.nio.file.Path p : interpreterRelatedPaths) { - interpreterPaths.addAll(getPathsFromDirPath(p)); - } - - addLocalResource(fileSystem, String.valueOf(applicationId.getId()), localResources, - interpreterPaths); - - List vargs = Lists.newArrayList(); - - vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); - - vargs.add("-Xmx1024m"); - - vargs.add(YarnRemoteInterpreterServer.class.getName()); - - vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/interpreter.stdout"); - vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/interpreter.stderr"); - - String command = Joiner.on(" ").join(vargs); - logger.debug("command: {}", command); - - List commands = Lists.newArrayList(command); - - ContainerLaunchContext amContainer = - ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); - - Resource capability = Resource.newInstance(1024, 1); - appContext.setResource(capability); - - appContext.setAMContainerSpec(amContainer); - - Priority pri = Priority.newInstance(0); - appContext.setPriority(pri); - - appContext.setQueue("default"); - - appContext.setApplicationType("ZEPPELIN INTERPRETER"); - - return new RemoteInterpreterYarnProcess(connectTimeout, listener, appListener, yarnClient, - appContext); - } catch (YarnException | IOException e) { - throw new InterpreterException(e); - } - } - - private java.nio.file.Path getInterpreterRelativePath(String dirName) { - return Paths.get(zeppelinConfiguration.getInterpreterDir(), dirName); + return new RemoteInterpreterYarnProcess(connectTimeout, listener, appListener, yarnClient, + zeppelinConfiguration, configuration, name, groupName, env, properties); } public void releaseResource(String id) { @@ -272,51 +180,4 @@ private String getClientToken(ApplicationReport applicationReport) { return ""; } } - - private List getPathsFromDirPath(java.nio.file.Path dirPath) { - if (null == dirPath || Files.notExists(dirPath) || !Files.isDirectory(dirPath)) { - return Lists.newArrayList(); - } - - try { - DirectoryStream directoryStream = - Files.newDirectoryStream(dirPath, new DirectoryStream.Filter() { - @Override - public boolean accept(java.nio.file.Path entry) throws IOException { - String filename = entry.toString(); - return filename.endsWith(".jar") || filename.endsWith(".zip"); - } - }); - return Lists.newArrayList(directoryStream); - } catch (IOException e) { - logger.error("Cannot read directory: {}", dirPath.toString(), e); - return Lists.newArrayList(); - } - } - - private void addLocalResource(FileSystem fs, String appId, - Map localResourceMap, List paths) { - for (java.nio.file.Path path : paths) { - String resourcePath = appId + Path.SEPARATOR + path.getFileName().toString(); - Path dst = new Path(fs.getHomeDirectory(), resourcePath); - try { - if (Files.exists(path) && !fs.exists(dst)) { - fs.copyFromLocalFile(new Path(path.toUri()), dst); - FileStatus fileStatus = fs.getFileStatus(dst); - LocalResourceType localResourceType = LocalResourceType.FILE; - String filename = path.getFileName().toString(); - if (filename.endsWith(".zip")) { - localResourceType = LocalResourceType.ARCHIVE; - } - LocalResource resource = LocalResource - .newInstance(ConverterUtils.getYarnUrlFromPath(dst), localResourceType, - LocalResourceVisibility.APPLICATION, fileStatus.getLen(), - fileStatus.getModificationTime()); - localResourceMap.put(filename, resource); - } - } catch (IOException e) { - logger.error("Error while copying resources into hdfs", e); - } - } - } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index f78f0c769c3..58c6784e880 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -17,17 +17,37 @@ package org.apache.zeppelin.cluster.yarn; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.io.IOException; -import java.net.InetAddress; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +61,18 @@ import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED; import static org.apache.hadoop.yarn.api.records.YarnApplicationState.KILLED; import static org.apache.hadoop.yarn.api.records.YarnApplicationState.RUNNING; +import static org.apache.zeppelin.cluster.yarn.YarnUtils.addLocalResource; +import static org.apache.zeppelin.cluster.yarn.YarnUtils.getPathsFromDirPath; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_APPLICATION_TYPE_DEFAULT; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_APPLICATION_TYPE_KEY; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_PRIORITY_DEFAULT; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_PRIORITY_KEY; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_MEMORY_DEFAULT; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_MEMORY_KEY; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_QUEUE_DEFAULT; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_QUEUE_KEY; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_VCORES_DEFAULT; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_VCORES_KEY; /** * @@ -50,7 +82,15 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterYarnProcess.class); private final YarnClient yarnClient; - private final ApplicationSubmissionContext appContext; + private final ZeppelinConfiguration zeppelinConfiguration; + private final Configuration configuration; + private final String name; + private final String group; + private final Map env; + private final Properties properties; + + private CountDownLatch waitingInitialized; + private List interpreterLibPaths; private ApplicationId applicationId; private boolean isRunning = false; @@ -60,12 +100,25 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { private String host = null; private int port = -1; - public RemoteInterpreterYarnProcess(int connectTimeout, RemoteInterpreterProcessListener listener, + RemoteInterpreterYarnProcess(int connectTimeout, RemoteInterpreterProcessListener listener, ApplicationEventListener appListener, YarnClient yarnClient, - ApplicationSubmissionContext appContext) { + ZeppelinConfiguration zeppelinConfiguration, Configuration configuration, String name, + String group, Map env, Properties properties) { super(new RemoteInterpreterEventPoller(listener, appListener), connectTimeout); this.yarnClient = yarnClient; - this.appContext = appContext; + this.zeppelinConfiguration = zeppelinConfiguration; + this.configuration = configuration; + this.name = name; + this.group = group; + this.env = env; + this.properties = properties; + + this.waitingInitialized = new CountDownLatch(1); + this.interpreterLibPaths = Lists.newArrayList( + Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target"), + Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target", "lib"), + Paths.get(zeppelinConfiguration.getHome(), "lib", "interpreter"), + Paths.get(zeppelinConfiguration.getHome(), "zeppelin-zengine", "target")); } @Override @@ -92,22 +145,184 @@ public void start(String userName, Boolean isUserImpersonate) { } try { + YarnClientApplication app = yarnClient.createApplication(); + ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); + + // put this info idApplicationIdMap + ApplicationId applicationId = appContext.getApplicationId(); + + appContext.setKeepContainersAcrossApplicationAttempts(false); + appContext.setApplicationName(name); + + Map env = Maps.newHashMap(); + + ArrayList classpathStrings = Lists.newArrayList(configuration + .getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, + YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)); + classpathStrings.add(0, "./*"); + classpathStrings.add(0, ApplicationConstants.Environment.CLASSPATH.$$()); + + // For Spark + if (isSparkHomeSet()) { + classpathStrings.add("${SPARK_HOME}/jars/*"); + } + + String classpathEnv = + Joiner.on(ApplicationConstants.CLASS_PATH_SEPARATOR).join(classpathStrings); + + logger.debug("classpath: {}", classpathEnv); + + env.putAll(this.env); + env.put("CLASSPATH", classpathEnv); + + Map localResources = new HashMap<>(); + + FileSystem fileSystem = FileSystem.get(configuration); + + Path interpreterDir = getInterpreterRelativePath(group); + List interpreterPaths = getPathsFromDirPath(interpreterDir); + + // For spark + if (isSparkInterpreter() && !isSparkHomeSet()) { + interpreterDir = getInterpreterRelativePath("spark/dep"); + interpreterPaths.addAll(getPathsFromDirPath(interpreterDir)); + } + + for (Path p : interpreterLibPaths) { + interpreterPaths.addAll(getPathsFromDirPath(p)); + } + + addLocalResource(fileSystem, String.valueOf(applicationId.getId()), localResources, + interpreterPaths); + + List vargs = Lists.newArrayList(); + + vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); + + int memory; + int defaultMemory = Integer.valueOf(ZEPPELIN_YARN_MEMORY_DEFAULT); + + try { + memory = Integer.valueOf(properties.getProperty(ZEPPELIN_YARN_MEMORY_KEY)); + } catch (NumberFormatException e) { + memory = defaultMemory; + } + + // For spark + if (isSparkInterpreter()) { + // Assume that spark.master is "local" without any setting. It, however, doesn't guarantee + // spark works without "spark.master" + String master = properties + .getProperty("master", properties.getProperty("spark.master", "local[*]")); + String deployMode = properties.getProperty("spark.submit.deployMode", "client"); + + if (master.contains("yarn") && (master.contains("client") || deployMode + .contains("client")) && properties.containsKey("spark.yarn.am.memory")) { + memory = convertSparkMemoryFormat(properties.getProperty("spark.yarn.am.memory")); + } else { + memory = convertSparkMemoryFormat(properties.getProperty("spark.driver.memory")); + } + + if (memory < defaultMemory) { + memory = defaultMemory; + } + } + + vargs.add("-Xmx" + memory + "m"); + + vargs.add(YarnRemoteInterpreterServer.class.getName()); + + vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/interpreter.stdout"); + vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/interpreter.stderr"); + + String command = Joiner.on(" ").join(vargs); + logger.debug("command: {}", command); + + List commands = Lists.newArrayList(command); + + ContainerLaunchContext amContainer = + ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); + + int vCores; + int vCoresDefault = Integer.valueOf(ZEPPELIN_YARN_VCORES_DEFAULT); + + try { + vCores = Integer.valueOf(properties.getProperty(ZEPPELIN_YARN_VCORES_KEY)); + } catch (NumberFormatException e) { + vCores = vCoresDefault; + } + + Resource capability = Resource.newInstance(memory, vCores); + appContext.setResource(capability); + + appContext.setAMContainerSpec(amContainer); + + int priority; + int priorityDefault = Integer.valueOf(ZEPPELIN_YARN_PRIORITY_DEFAULT); + + try { + priority = Integer.valueOf(properties.getProperty(ZEPPELIN_YARN_PRIORITY_KEY)); + } catch (NumberFormatException e) { + priority = priorityDefault; + } + + Priority pri = Priority.newInstance(priority); + appContext.setPriority(pri); + + appContext + .setQueue(properties.getProperty(ZEPPELIN_YARN_QUEUE_KEY, ZEPPELIN_YARN_QUEUE_DEFAULT)); + + appContext.setApplicationType(properties + .getProperty(ZEPPELIN_YARN_APPLICATION_TYPE_KEY, ZEPPELIN_YARN_APPLICATION_TYPE_DEFAULT)); + this.applicationId = appContext.getApplicationId(); + yarnClient.submitApplication(appContext); monitor = Client.scheduledExecutorService .scheduleAtFixedRate(new ApplicationMonitor(), 1, 1, TimeUnit.SECONDS); - while (!isRunning()) { - if (oldState == FINISHED || oldState == FAILED || oldState == KILLED) { - throw new YarnException("Failed to initialize yarn application: " + applicationId); - } - Thread.sleep(500); - logger.debug("Waiting until connected"); + + waitingInitialized.await(5, TimeUnit.MINUTES); + if (oldState != RUNNING) { + stop(); + throw new InterpreterException("Failed to initialize yarn application: " + applicationId); } + } catch (YarnException | IOException | InterruptedException e) { + stop(); throw new InterpreterException(e); } } + private boolean isSparkInterpreter() { + return "spark".equals(group); + } + + private boolean isSparkHomeSet() { + return this.env.containsKey("SPARK_HOME"); + } + + private int convertSparkMemoryFormat(String memoryFormat) { + int memory = Integer.valueOf(memoryFormat.substring(0, memoryFormat.length() -1)); + String unit = "" + memoryFormat.charAt(memoryFormat.length() -1); + + switch (unit) { + case "k": + return memory / 1024; + case "m": + return memory; + case "g": + return memory * 1024; + case "t": + return memory * 1024 * 1024; + default: + return 0; + } + } + + private Path getInterpreterRelativePath(String dirName) { + return Paths.get(zeppelinConfiguration.getInterpreterDir(), dirName); + } + @Override public void stop() { isRunning = false; @@ -120,6 +335,7 @@ public void stop() { } oldState = null; monitor.cancel(false); + waitingInitialized = new CountDownLatch(1); } @Override @@ -159,31 +375,18 @@ public void run() { setHost(host); setPort(port); setRunning(true); + waitingInitialized.countDown(); } break; case FINISHED: - if (!FINISHED.equals(oldState)) { - logger.info("applicationId {} finished with final Status {}", applicationId, - applicationReport.getFinalApplicationStatus()); - oldState = FINISHED; - stop(); - //TODO(jl): Handle it!! - } - break; case FAILED: - if (!FAILED.equals(oldState)) { - logger.info("id {}, applicationId {} failed with final Status {}", applicationId, - applicationReport.getFinalApplicationStatus()); - oldState = FAILED; - stop(); - //TODO(jl): Handle it!! - } - break; case KILLED: - if (!KILLED.equals(oldState)) { - logger.info("applicationId {} killed with final Status {}", applicationId, + if (!curState.equals(oldState)) { + logger.info("applicationId {} {} with final Status {}", applicationId, + curState.toString().toLowerCase(), applicationReport.getFinalApplicationStatus()); - oldState = KILLED; + oldState = curState; + waitingInitialized.countDown(); stop(); //TODO(jl): Handle it!! } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java new file mode 100644 index 00000000000..7e8626a823f --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java @@ -0,0 +1,71 @@ +package org.apache.zeppelin.cluster.yarn; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class YarnUtils { + public static final Logger logger = LoggerFactory.getLogger(YarnUtils.class); + + static List getPathsFromDirPath(java.nio.file.Path dirPath) { + if (null == dirPath || Files.notExists(dirPath) || !Files.isDirectory(dirPath)) { + return Lists.newArrayList(); + } + + try { + DirectoryStream directoryStream = + Files.newDirectoryStream(dirPath, new DirectoryStream.Filter() { + @Override + public boolean accept(java.nio.file.Path entry) throws IOException { + String filename = entry.toString(); + return filename.endsWith(".jar") || filename.endsWith(".zip"); + } + }); + return Lists.newArrayList(directoryStream); + } catch (IOException e) { + logger.error("Cannot read directory: {}", dirPath.toString(), e); + return Lists.newArrayList(); + } + } + + static void addLocalResource(FileSystem fs, String appId, + Map localResourceMap, List paths) { + for (java.nio.file.Path path : paths) { + String resourcePath = appId + org.apache.hadoop.fs.Path.SEPARATOR + path.getFileName().toString(); + org.apache.hadoop.fs.Path dst = new org.apache.hadoop.fs.Path(fs.getHomeDirectory(), resourcePath); + try { + if (Files.exists(path) && !fs.exists(dst)) { + fs.copyFromLocalFile(new org.apache.hadoop.fs.Path(path.toUri()), dst); + FileStatus fileStatus = fs.getFileStatus(dst); + LocalResourceType localResourceType = LocalResourceType.FILE; + String filename = path.getFileName().toString(); + if (filename.endsWith(".zip")) { + localResourceType = LocalResourceType.ARCHIVE; + } + LocalResource resource = LocalResource + .newInstance(ConverterUtils.getYarnUrlFromPath(dst), localResourceType, + LocalResourceVisibility.APPLICATION, fileStatus.getLen(), + fileStatus.getModificationTime()); + localResourceMap.put(filename, resource); + } + } catch (IOException e) { + logger.error("Error while copying resources into hdfs", e); + } + } + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index c65daeb69a4..a962bb1b9c4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -188,7 +188,7 @@ public RemoteInterpreterProcess getInterpreterProcess() { switch (clusterManagerKey) { case Constants.ZEPPELIN_CLUSTER_MANAGER_YARN: remoteProcess = clusterManager - .createInterpreter("uniq", interpreterGroupName, group, connectTimeout, + .createInterpreter(sessionKey, interpreterGroupName, group, env, property, connectTimeout, remoteInterpreterProcessListener, applicationEventListener); break; case Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL: From 2c6b77f5660688b7a49e4684503f65b048520ff8 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 10 May 2017 12:01:18 +0900 Subject: [PATCH 17/52] Fixed style Fixed rat issue --- zeppelin-cluster/yarn/pom.xml | 45 +++++++++++++++++++ .../zeppelin/cluster/ClusterManager.java | 4 +- .../yarn/RemoteInterpreterYarnProcess.java | 4 +- .../zeppelin/cluster/yarn/YarnUtils.java | 24 +++++++++- .../interpreter/remote/RemoteInterpreter.java | 4 +- 5 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 zeppelin-cluster/yarn/pom.xml diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml new file mode 100644 index 00000000000..b0ea366db8d --- /dev/null +++ b/zeppelin-cluster/yarn/pom.xml @@ -0,0 +1,45 @@ + + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.8.0-SNAPSHOT + ../.. + + + zeppelin-cluster-yarn + jar + 0.8.0-SNAPSHOT + Zeppelin Cluster: yarn + Zeppelin Yarn Cluster + + + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + + + + diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java index 53846dce0b2..8e83f5a45fa 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -30,6 +30,7 @@ * */ public abstract class ClusterManager { + protected final ZeppelinConfiguration zeppelinConfiguration; protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration) { @@ -47,7 +48,8 @@ protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration) { public abstract void stop(); public abstract RemoteInterpreterProcess createInterpreter(String id, String name, - String groupName, Map env, Properties properties, int connectTimeout, RemoteInterpreterProcessListener listener, + String groupName, Map env, Properties properties, int connectTimeout, + RemoteInterpreterProcessListener listener, ApplicationEventListener appListener) throws InterpreterException; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index 58c6784e880..483a56ba85c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -302,8 +302,8 @@ private boolean isSparkHomeSet() { } private int convertSparkMemoryFormat(String memoryFormat) { - int memory = Integer.valueOf(memoryFormat.substring(0, memoryFormat.length() -1)); - String unit = "" + memoryFormat.charAt(memoryFormat.length() -1); + int memory = Integer.valueOf(memoryFormat.substring(0, memoryFormat.length() - 1)); + String unit = "" + memoryFormat.charAt(memoryFormat.length() - 1); switch (unit) { case "k": diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java index 7e8626a823f..3a1435a9761 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.cluster.yarn; import com.google.common.collect.Lists; @@ -20,6 +37,7 @@ * */ public class YarnUtils { + public static final Logger logger = LoggerFactory.getLogger(YarnUtils.class); static List getPathsFromDirPath(java.nio.file.Path dirPath) { @@ -46,8 +64,10 @@ public boolean accept(java.nio.file.Path entry) throws IOException { static void addLocalResource(FileSystem fs, String appId, Map localResourceMap, List paths) { for (java.nio.file.Path path : paths) { - String resourcePath = appId + org.apache.hadoop.fs.Path.SEPARATOR + path.getFileName().toString(); - org.apache.hadoop.fs.Path dst = new org.apache.hadoop.fs.Path(fs.getHomeDirectory(), resourcePath); + String resourcePath = + appId + org.apache.hadoop.fs.Path.SEPARATOR + path.getFileName().toString(); + org.apache.hadoop.fs.Path dst = new org.apache.hadoop.fs.Path(fs.getHomeDirectory(), + resourcePath); try { if (Files.exists(path) && !fs.exists(dst)) { fs.copyFromLocalFile(new org.apache.hadoop.fs.Path(path.toUri()), dst); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index a962bb1b9c4..30b9aca435d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -188,8 +188,8 @@ public RemoteInterpreterProcess getInterpreterProcess() { switch (clusterManagerKey) { case Constants.ZEPPELIN_CLUSTER_MANAGER_YARN: remoteProcess = clusterManager - .createInterpreter(sessionKey, interpreterGroupName, group, env, property, connectTimeout, - remoteInterpreterProcessListener, applicationEventListener); + .createInterpreter(sessionKey, interpreterGroupName, group, env, property, + connectTimeout, remoteInterpreterProcessListener, applicationEventListener); break; case Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL: default: From 6029bebbf96021704640e4a6a35fbb4e58177d93 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Fri, 12 May 2017 12:25:32 +0900 Subject: [PATCH 18/52] Enabled to read HADOOP_CONF_DIR --- zeppelin-zengine/pom.xml | 56 +++++++++++++++++++ .../yarn/RemoteInterpreterYarnProcess.java | 18 +++++- .../zeppelin/cluster/yarn/YarnUtils.java | 25 ++++++--- 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index b33aae0f231..b6272f958ee 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -164,6 +164,62 @@ + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + javax.servlet + servlet-api + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + com.jcraft + jsch + + + org.apache.commons + commons-compress + + + org.codehaus.jackson + jackson-mapper-asl + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + javax.servlet + servlet-api + + + xerces + xercesImpl + + + io.netty + netty + + + + org.slf4j slf4j-api diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index 483a56ba85c..48a49fcda45 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -117,8 +118,11 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { this.interpreterLibPaths = Lists.newArrayList( Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target"), Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target", "lib"), + Paths.get(zeppelinConfiguration.getHome(), "zeppelin-zengine", "target"), + Paths.get(zeppelinConfiguration.getHome(), "conf", "yarn", "log4j.properties"), Paths.get(zeppelinConfiguration.getHome(), "lib", "interpreter"), - Paths.get(zeppelinConfiguration.getHome(), "zeppelin-zengine", "target")); + Paths.get(zeppelinConfiguration.getHome(), "lib", + "zeppelin-zengine-0.8.0-SNAPSHOT.jar")); } @Override @@ -161,6 +165,8 @@ public void start(String userName, Boolean isUserImpersonate) { YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)); classpathStrings.add(0, "./*"); classpathStrings.add(0, ApplicationConstants.Environment.CLASSPATH.$$()); + classpathStrings.add("./log4j.properties"); + classpathStrings.add(System.getenv("HADOOP_CONF_DIR")); // For Spark if (isSparkHomeSet()) { @@ -192,6 +198,12 @@ public void start(String userName, Boolean isUserImpersonate) { interpreterPaths.addAll(getPathsFromDirPath(p)); } + Map systemEnv = System.getenv(); + if (systemEnv.containsKey("HADOOP_CONF_DIR")) { + interpreterDir = Paths.get(systemEnv.get("HADOOP_CONF_DIR")); + interpreterPaths.addAll(getPathsFromDirPath(interpreterDir)); + } + addLocalResource(fileSystem, String.valueOf(applicationId.getId()), localResources, interpreterPaths); @@ -219,7 +231,7 @@ public void start(String userName, Boolean isUserImpersonate) { if (master.contains("yarn") && (master.contains("client") || deployMode .contains("client")) && properties.containsKey("spark.yarn.am.memory")) { memory = convertSparkMemoryFormat(properties.getProperty("spark.yarn.am.memory")); - } else { + } else if (properties.containsKey("spark.driver.memory")) { memory = convertSparkMemoryFormat(properties.getProperty("spark.driver.memory")); } @@ -230,6 +242,8 @@ public void start(String userName, Boolean isUserImpersonate) { vargs.add("-Xmx" + memory + "m"); + vargs.add("-Dlog4j.configuration=log4j.properties"); + vargs.add(YarnRemoteInterpreterServer.class.getName()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/interpreter.stdout"); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java index 3a1435a9761..9e8d2c24511 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java @@ -22,6 +22,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; import java.util.List; import java.util.Map; import org.apache.hadoop.fs.FileStatus; @@ -40,16 +41,20 @@ public class YarnUtils { public static final Logger logger = LoggerFactory.getLogger(YarnUtils.class); - static List getPathsFromDirPath(java.nio.file.Path dirPath) { - if (null == dirPath || Files.notExists(dirPath) || !Files.isDirectory(dirPath)) { + static List getPathsFromDirPath(Path dirPath) { + if (null == dirPath || Files.notExists(dirPath)) { return Lists.newArrayList(); } + if (Files.isRegularFile(dirPath)) { + return Collections.singletonList(dirPath); + } + try { DirectoryStream directoryStream = - Files.newDirectoryStream(dirPath, new DirectoryStream.Filter() { + Files.newDirectoryStream(dirPath, new DirectoryStream.Filter() { @Override - public boolean accept(java.nio.file.Path entry) throws IOException { + public boolean accept(Path entry) throws IOException { String filename = entry.toString(); return filename.endsWith(".jar") || filename.endsWith(".zip"); } @@ -62,10 +67,11 @@ public boolean accept(java.nio.file.Path entry) throws IOException { } static void addLocalResource(FileSystem fs, String appId, - Map localResourceMap, List paths) { - for (java.nio.file.Path path : paths) { + Map localResourceMap, List paths) { + String resourceDirPath = appId + org.apache.hadoop.fs.Path.SEPARATOR; + for (Path path : paths) { String resourcePath = - appId + org.apache.hadoop.fs.Path.SEPARATOR + path.getFileName().toString(); + resourceDirPath + path.getFileName().toString(); org.apache.hadoop.fs.Path dst = new org.apache.hadoop.fs.Path(fs.getHomeDirectory(), resourcePath); try { @@ -87,5 +93,10 @@ static void addLocalResource(FileSystem fs, String appId, logger.error("Error while copying resources into hdfs", e); } } + try { + fs.deleteOnExit(new org.apache.hadoop.fs.Path(resourceDirPath)); + } catch (IOException e) { + logger.error("Error while removing {}", resourceDirPath, e); + } } } From 78d3fc5031279e64697b085a68f47655339bdceb Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 14 May 2017 05:09:09 +0900 Subject: [PATCH 19/52] Support remote yarn cluster with spark's yarn-client mode --- .../yarn/RemoteInterpreterYarnProcess.java | 57 +++++++++++++++++-- .../zeppelin/cluster/yarn/YarnUtils.java | 8 +-- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index 48a49fcda45..ad10eb9c6c9 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -20,6 +20,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -33,6 +34,8 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -168,11 +171,6 @@ public void start(String userName, Boolean isUserImpersonate) { classpathStrings.add("./log4j.properties"); classpathStrings.add(System.getenv("HADOOP_CONF_DIR")); - // For Spark - if (isSparkHomeSet()) { - classpathStrings.add("${SPARK_HOME}/jars/*"); - } - String classpathEnv = Joiner.on(ApplicationConstants.CLASS_PATH_SEPARATOR).join(classpathStrings); @@ -181,6 +179,10 @@ public void start(String userName, Boolean isUserImpersonate) { env.putAll(this.env); env.put("CLASSPATH", classpathEnv); + if (isHadoopConfSet()) { + env.put("HADOOP_CONF_DIR", System.getenv("HADOOP_CONF_DIR")); + } + Map localResources = new HashMap<>(); FileSystem fileSystem = FileSystem.get(configuration); @@ -194,6 +196,47 @@ public void start(String userName, Boolean isUserImpersonate) { interpreterPaths.addAll(getPathsFromDirPath(interpreterDir)); } + // For spark + if (isSparkHomeSet()) { + // For pyspark + interpreterDir = Paths.get(this.env.get("SPARK_HOME"), "python", "lib"); + List pythonLibPath = getPathsFromDirPath(interpreterDir); + interpreterPaths.addAll(pythonLibPath); + + // Set PYSPARK_ARCHIVES_PATH + List pythonLibPaths = new ArrayList<>(); + for (Path p : pythonLibPath) { + String pathFilenameString = p.getFileName().toString(); + if (pathFilenameString.endsWith(".zip")) { + pythonLibPaths.add(pathFilenameString); + } + } + env.put("PYSPARK_ARCHIVES_PATH", Joiner.on(",").join(pythonLibPaths)); + + Path jarsArchive = Files.createTempFile("spark_jars", ".zip"); + try (ZipOutputStream jarsStream = new ZipOutputStream( + new FileOutputStream(jarsArchive.toFile()))) { + jarsStream.setLevel(0); + + interpreterDir = Paths.get(this.env.get("SPARK_HOME"), "jars"); + List jarPaths = getPathsFromDirPath(interpreterDir); + for (Path p : jarPaths) { + jarsStream.putNextEntry(new ZipEntry(p.getFileName().toString())); + Files.copy(p, jarsStream); + jarsStream.closeEntry(); + } + + interpreterPaths.addAll(jarPaths); + } + + String dstPath = "hdfs:///.zeppelin/spark_jars.zip"; + fileSystem.copyFromLocalFile(new org.apache.hadoop.fs.Path(jarsArchive.toUri()), + new org.apache.hadoop.fs.Path(dstPath)); + Files.deleteIfExists(jarsArchive); + properties.setProperty("spark.yarn.archive", dstPath); + } + + for (Path p : interpreterLibPaths) { interpreterPaths.addAll(getPathsFromDirPath(p)); } @@ -315,6 +358,10 @@ private boolean isSparkHomeSet() { return this.env.containsKey("SPARK_HOME"); } + private boolean isHadoopConfSet() { + return this.env.containsKey("HADOOP_CONF_DIR"); + } + private int convertSparkMemoryFormat(String memoryFormat) { int memory = Integer.valueOf(memoryFormat.substring(0, memoryFormat.length() - 1)); String unit = "" + memoryFormat.charAt(memoryFormat.length() - 1); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java index 9e8d2c24511..cfa999528e4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java @@ -55,8 +55,7 @@ static List getPathsFromDirPath(Path dirPath) { Files.newDirectoryStream(dirPath, new DirectoryStream.Filter() { @Override public boolean accept(Path entry) throws IOException { - String filename = entry.toString(); - return filename.endsWith(".jar") || filename.endsWith(".zip"); + return Files.isRegularFile(entry); } }); return Lists.newArrayList(directoryStream); @@ -80,9 +79,6 @@ static void addLocalResource(FileSystem fs, String appId, FileStatus fileStatus = fs.getFileStatus(dst); LocalResourceType localResourceType = LocalResourceType.FILE; String filename = path.getFileName().toString(); - if (filename.endsWith(".zip")) { - localResourceType = LocalResourceType.ARCHIVE; - } LocalResource resource = LocalResource .newInstance(ConverterUtils.getYarnUrlFromPath(dst), localResourceType, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), @@ -93,10 +89,12 @@ static void addLocalResource(FileSystem fs, String appId, logger.error("Error while copying resources into hdfs", e); } } +/* try { fs.deleteOnExit(new org.apache.hadoop.fs.Path(resourceDirPath)); } catch (IOException e) { logger.error("Error while removing {}", resourceDirPath, e); } +*/ } } From 0180602fdbf54d2a8105da6d65b6bab1b53e0f9a Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Fri, 19 May 2017 13:27:49 -0400 Subject: [PATCH 20/52] End of PoC --- conf/yarn/log4j.properties | 22 ++++ python/pom.xml | 2 + .../zeppelin/python/PythonInterpreter.java | 2 + .../zeppelin/spark/PySparkInterpreter.java | 4 +- .../src/assemble/distribution.xml | 4 + .../interpreter/remote/ClientFactory.java | 4 + .../remote/RemoteInterpreterEventPoller.java | 3 +- .../remote/RemoteInterpreterProcess.java | 7 ++ zeppelin-server/pom.xml | 2 +- .../src/main/resources/log4j.properties | 1 + zeppelin-zengine/pom.xml | 17 ++- .../yarn/RemoteInterpreterYarnProcess.java | 109 +++++++++++++----- .../zeppelin/cluster/yarn/YarnUtils.java | 6 +- 13 files changed, 148 insertions(+), 35 deletions(-) create mode 100644 conf/yarn/log4j.properties diff --git a/conf/yarn/log4j.properties b/conf/yarn/log4j.properties new file mode 100644 index 00000000000..8daee59d60d --- /dev/null +++ b/conf/yarn/log4j.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger = INFO, stdout + +log4j.appender.stdout = org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout = org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n diff --git a/python/pom.xml b/python/pom.xml index e520b4beffd..b06c05fb750 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -125,6 +125,8 @@ + diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index 0bfcae0d3e6..affd51af2a1 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -202,6 +202,8 @@ private void createGatewayServerAndStartScript() throws UnknownHostException { py4jLibPath + File.pathSeparator + pythonLibPath); } + logger.info("PYTHONPATH: {}", env.get("PYTHONPATH")); + logger.info("cmd = {}", cmd.toString()); executor.execute(cmd, env, this); pythonscriptRunning = true; diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index 28910b2546b..66eb7c8c329 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -175,7 +175,7 @@ private Map setupPySparkEnv() throws IOException{ if (!env.containsKey("PYTHONPATH")) { SparkConf conf = getSparkConf(); env.put("PYTHONPATH", conf.get("spark.submit.pyFiles").replaceAll(",", ":") + - ":../interpreter/lib/python"); + ":../interpreter/lib/python:python"); } // get additional class paths when using SPARK_SUBMIT and not using YARN-CLIENT @@ -186,6 +186,8 @@ private Map setupPySparkEnv() throws IOException{ String sparkSubmitJars = getSparkConf().get("spark.jars").replace(",", ":"); if (!"".equals(sparkSubmitJars)) { + + env.put("PYTHONPATH", env.get("PYTHONPATH") + sparkSubmitJars); } } diff --git a/zeppelin-distribution/src/assemble/distribution.xml b/zeppelin-distribution/src/assemble/distribution.xml index 5c369e256a3..00ca2eb818c 100644 --- a/zeppelin-distribution/src/assemble/distribution.xml +++ b/zeppelin-distribution/src/assemble/distribution.xml @@ -107,5 +107,9 @@ /lib/node_modules/zeppelin-spell ../zeppelin-web/src/app/spell + + /lib/zengine + ../zeppelin-zengine/target/lib + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java index b2cb78f810e..c0fc6d79a49 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java @@ -30,11 +30,14 @@ import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * */ public class ClientFactory extends BasePooledObjectFactory{ + private final Logger logger = LoggerFactory.getLogger(ClientFactory.class); private String host; private int port; Map clientSocketMap = new HashMap<>(); @@ -46,6 +49,7 @@ public ClientFactory(String host, int port) { @Override public Client create() throws Exception { + logger.info("Create. host: {}, port: {}", host, port); TSocket transport = new TSocket(host, port); try { transport.open(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 6927b3b6687..a8ee26e4810 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -100,16 +100,17 @@ public void run() { continue; } + boolean broken = false; try { client = interpreterProcess.getClient(); } catch (Exception e1) { + interpreterProcess.releaseBrokenClient(client); logger.error("Can't get RemoteInterpreterEvent", e1); waitQuietly(); continue; } RemoteInterpreterEvent event = null; - boolean broken = false; try { event = client.getEvent(); } catch (TException e) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 248d649aa24..2e8188a2fbd 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -113,6 +113,13 @@ public void releaseBrokenClient(Client client) { clientPool.invalidateObject(client); } catch (Exception e) { logger.warn("exception occurred during releasing thrift client", e); + } finally { + clientPool.clear(); + clientPool.close(); + + logger.info("releaseBrokenClient. Host: {}, port: {}", getHost(), getPort()); + clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort())); + clientPool.setTestOnBorrow(true); } } diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 295f6eb3868..1df9dac6126 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -36,7 +36,7 @@ 4.3.6 2.22.2 - 2.6.0 + 2.7.3 2.2.1 1.13 2.0.1 diff --git a/zeppelin-server/src/main/resources/log4j.properties b/zeppelin-server/src/main/resources/log4j.properties index 2f644074c89..f5f3a0ed50e 100644 --- a/zeppelin-server/src/main/resources/log4j.properties +++ b/zeppelin-server/src/main/resources/log4j.properties @@ -23,3 +23,4 @@ log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n log4j.additivity.org.apache.zeppelin.interpreter = false log4j.logger.org.apache.zeppelin.interpreter = DEBUG, stdout +log4j.logger.org.apache.zeppelin.cluster = DEBUG, stdout diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index b6272f958ee..7467bc33d0d 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -47,12 +47,27 @@ 1.4.01 4.1.1.201511131810-r 1.3 - 2.6.0 + 2.7.3 0.27 + + + + com.fasterxml.jackson.core + jackson-core + 2.6.5 + + + com.fasterxml.jackson.core + jackson-databind + 2.6.5 + + + + ${project.groupId} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index ad10eb9c6c9..ebbd5a5a96c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -20,6 +20,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; @@ -39,6 +40,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -122,10 +124,12 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target"), Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target", "lib"), Paths.get(zeppelinConfiguration.getHome(), "zeppelin-zengine", "target"), + Paths.get(zeppelinConfiguration.getHome(), "zeppelin-zengine", "target", "lib"), Paths.get(zeppelinConfiguration.getHome(), "conf", "yarn", "log4j.properties"), Paths.get(zeppelinConfiguration.getHome(), "lib", "interpreter"), Paths.get(zeppelinConfiguration.getHome(), "lib", - "zeppelin-zengine-0.8.0-SNAPSHOT.jar")); + "zeppelin-zengine-0.8.0-SNAPSHOT.jar"), + Paths.get(zeppelinConfiguration.getHome(), "lib", "zengine")); } @Override @@ -190,16 +194,10 @@ public void start(String userName, Boolean isUserImpersonate) { Path interpreterDir = getInterpreterRelativePath(group); List interpreterPaths = getPathsFromDirPath(interpreterDir); - // For spark - if (isSparkInterpreter() && !isSparkHomeSet()) { - interpreterDir = getInterpreterRelativePath("spark/dep"); - interpreterPaths.addAll(getPathsFromDirPath(interpreterDir)); - } - - // For spark - if (isSparkHomeSet()) { + if (isSparkInterpreter()) { // For pyspark - interpreterDir = Paths.get(this.env.get("SPARK_HOME"), "python", "lib"); + interpreterDir = isSparkHomeSet() ? Paths.get(this.env.get("SPARK_HOME"), "python", "lib") + : getInterpreterRelativePath(group, "pyspark"); List pythonLibPath = getPathsFromDirPath(interpreterDir); interpreterPaths.addAll(pythonLibPath); @@ -213,29 +211,72 @@ public void start(String userName, Boolean isUserImpersonate) { } env.put("PYSPARK_ARCHIVES_PATH", Joiner.on(",").join(pythonLibPaths)); - Path jarsArchive = Files.createTempFile("spark_jars", ".zip"); - try (ZipOutputStream jarsStream = new ZipOutputStream( - new FileOutputStream(jarsArchive.toFile()))) { - jarsStream.setLevel(0); - - interpreterDir = Paths.get(this.env.get("SPARK_HOME"), "jars"); - List jarPaths = getPathsFromDirPath(interpreterDir); - for (Path p : jarPaths) { - jarsStream.putNextEntry(new ZipEntry(p.getFileName().toString())); - Files.copy(p, jarsStream); - jarsStream.closeEntry(); + interpreterDir = isSparkHomeSet() ? Paths.get(this.env.get("SPARK_HOME"), "jars") + : getInterpreterRelativePath(group, "dep"); + List jarPaths = getPathsFromDirPath(interpreterDir); + interpreterPaths.addAll(jarPaths); + + String dstPath = "hdfs:///user/" + userName + "/.zeppelin/spark_" + name + "_jars.zip"; + if (!fileSystem.exists(new org.apache.hadoop.fs.Path(dstPath))) { + Path jarsArchive = Files.createTempFile("spark_jars", ".zip"); + try (ZipOutputStream jarsStream = new ZipOutputStream( + new FileOutputStream(jarsArchive.toFile()))) { + jarsStream.setLevel(0); + + for (Path p : jarPaths) { + jarsStream.putNextEntry(new ZipEntry(p.getFileName().toString())); + Files.copy(p, jarsStream); + jarsStream.closeEntry(); + } } - interpreterPaths.addAll(jarPaths); + fileSystem.copyFromLocalFile(new org.apache.hadoop.fs.Path(jarsArchive.toUri()), + new org.apache.hadoop.fs.Path(dstPath)); + Files.deleteIfExists(jarsArchive); } - - String dstPath = "hdfs:///.zeppelin/spark_jars.zip"; - fileSystem.copyFromLocalFile(new org.apache.hadoop.fs.Path(jarsArchive.toUri()), - new org.apache.hadoop.fs.Path(dstPath)); - Files.deleteIfExists(jarsArchive); properties.setProperty("spark.yarn.archive", dstPath); + } else if (isPythonInterpreter()) { + interpreterDir = getInterpreterRelativePath(group); + List pythonLibPath = getPathsFromDirPath(interpreterDir); + interpreterPaths.addAll(pythonLibPath); + + // set PYTHONPATH + List pythonLibPaths = new ArrayList<>(); + for (Path p : pythonLibPath) { + String pathFilenameString = p.getFileName().toString(); + if (pathFilenameString.endsWith(".zip")) { + pythonLibPaths.add(Environment.PWD.$$() + File.separator + pathFilenameString); + } + } + env.put("PYTHONPATH", Joiner.on(File.pathSeparator).join(pythonLibPaths)); + } + + // For pyspark and python + List zeppelinPythonLibPaths = getPathsFromDirPath( + getInterpreterRelativePath("lib", "python")); + interpreterPaths.addAll(zeppelinPythonLibPaths); + List distributedZeppelinPythonLibPaths = Lists.newArrayList(); + for (Path p : zeppelinPythonLibPaths) { + if (p.getFileName().toString().endsWith(".py")) { + distributedZeppelinPythonLibPaths.add( + "python" + File.separator + p.getFileName().toString()); + } } + if (isSparkInterpreter()) { + String pyFiles = + (env.containsKey("PYSPARK_ARCHIVES_PATH") ? env.get("PYSPARK_ARCHIVES_PATH") + "," : "") + + Joiner.on(",").join(distributedZeppelinPythonLibPaths); + properties.setProperty("spark.submit.pyFiles", pyFiles); + } else if (isPythonInterpreter()) { + List pwdDistributedZeppelinPythonLibPaths = Lists.newArrayList(); + for (String p : distributedZeppelinPythonLibPaths) { + pwdDistributedZeppelinPythonLibPaths.add(Environment.PWD.$$() + File.separator + p); + } + String path = env.get("PYTHONPATH") + File.pathSeparator + Joiner.on(File.pathSeparator) + .join(pwdDistributedZeppelinPythonLibPaths); + env.put("PYTHONPATH", "./" + File.pathSeparator + "./python" + File.pathSeparator + path); + } for (Path p : interpreterLibPaths) { interpreterPaths.addAll(getPathsFromDirPath(p)); @@ -354,6 +395,10 @@ private boolean isSparkInterpreter() { return "spark".equals(group); } + private boolean isPythonInterpreter() { + return "python".equals(group); + } + private boolean isSparkHomeSet() { return this.env.containsKey("SPARK_HOME"); } @@ -380,12 +425,13 @@ private int convertSparkMemoryFormat(String memoryFormat) { } } - private Path getInterpreterRelativePath(String dirName) { - return Paths.get(zeppelinConfiguration.getInterpreterDir(), dirName); + private Path getInterpreterRelativePath(String... dirNames) { + return Paths.get(zeppelinConfiguration.getInterpreterDir(), dirNames); } @Override public void stop() { + logger.info("called stop"); isRunning = false; if (null != oldState && oldState != FINISHED && oldState != FAILED && oldState != KILLED) { try { @@ -420,8 +466,11 @@ public void run() { case NEW_SAVING: case SUBMITTED: case ACCEPTED: - if (null == oldState) { + if (null == oldState || !oldState.equals(curState)) { logger.info("new application added. applicationId: {}", applicationId); + setRunning(false); + setHost("N/A"); + setPort(-1); } oldState = curState; break; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java index cfa999528e4..348bd9ab693 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.cluster.yarn; import com.google.common.collect.Lists; +import java.io.File; import java.io.IOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -79,9 +80,12 @@ static void addLocalResource(FileSystem fs, String appId, FileStatus fileStatus = fs.getFileStatus(dst); LocalResourceType localResourceType = LocalResourceType.FILE; String filename = path.getFileName().toString(); + if (filename.endsWith(".py")) { + filename = "python" + File.separator + filename; + } LocalResource resource = LocalResource .newInstance(ConverterUtils.getYarnUrlFromPath(dst), localResourceType, - LocalResourceVisibility.APPLICATION, fileStatus.getLen(), + LocalResourceVisibility.PUBLIC, fileStatus.getLen(), fileStatus.getModificationTime()); localResourceMap.put(filename, resource); } From 63792c2c5414c7398eb7d851f03273c9fe924c0f Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Tue, 6 Jun 2017 00:39:27 +0900 Subject: [PATCH 21/52] Fix style --- .../yarn/RemoteInterpreterYarnProcess.java | 96 +++++++++---------- .../interpreter/remote/RemoteInterpreter.java | 27 +++--- 2 files changed, 62 insertions(+), 61 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index ebbd5a5a96c..a74e1e2a73e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -412,16 +412,16 @@ private int convertSparkMemoryFormat(String memoryFormat) { String unit = "" + memoryFormat.charAt(memoryFormat.length() - 1); switch (unit) { - case "k": - return memory / 1024; - case "m": - return memory; - case "g": - return memory * 1024; - case "t": - return memory * 1024 * 1024; - default: - return 0; + case "k": + return memory / 1024; + case "m": + return memory; + case "g": + return memory * 1024; + case "t": + return memory * 1024 * 1024; + default: + return 0; } } @@ -462,45 +462,45 @@ public void run() { ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId); YarnApplicationState curState = applicationReport.getYarnApplicationState(); switch (curState) { - case NEW: - case NEW_SAVING: - case SUBMITTED: - case ACCEPTED: - if (null == oldState || !oldState.equals(curState)) { - logger.info("new application added. applicationId: {}", applicationId); - setRunning(false); - setHost("N/A"); - setPort(-1); - } + case NEW: + case NEW_SAVING: + case SUBMITTED: + case ACCEPTED: + if (null == oldState || !oldState.equals(curState)) { + logger.info("new application added. applicationId: {}", applicationId); + setRunning(false); + setHost("N/A"); + setPort(-1); + } + oldState = curState; + break; + case RUNNING: + if (!RUNNING.equals(oldState)) { + String host = applicationReport.getHost(); + int port = applicationReport.getRpcPort(); + logger + .info("applicationId {} started. Host: {}, port: {}", applicationId, host + , port); + oldState = curState; + setHost(host); + setPort(port); + setRunning(true); + waitingInitialized.countDown(); + } + break; + case FINISHED: + case FAILED: + case KILLED: + if (!curState.equals(oldState)) { + logger.info("applicationId {} {} with final Status {}", applicationId, + curState.toString().toLowerCase(), + applicationReport.getFinalApplicationStatus()); oldState = curState; - break; - case RUNNING: - if (!RUNNING.equals(oldState)) { - String host = applicationReport.getHost(); - int port = applicationReport.getRpcPort(); - logger - .info("applicationId {} started. Host: {}, port: {}", applicationId, host - , port); - oldState = curState; - setHost(host); - setPort(port); - setRunning(true); - waitingInitialized.countDown(); - } - break; - case FINISHED: - case FAILED: - case KILLED: - if (!curState.equals(oldState)) { - logger.info("applicationId {} {} with final Status {}", applicationId, - curState.toString().toLowerCase(), - applicationReport.getFinalApplicationStatus()); - oldState = curState; - waitingInitialized.countDown(); - stop(); - //TODO(jl): Handle it!! - } - break; + waitingInitialized.countDown(); + stop(); + //TODO(jl): Handle it!! + } + break; } } catch (YarnException | IOException e) { logger.debug("Error occurs while fetching status of {}", applicationId, e); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 30b9aca435d..2fbca35bf2c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -44,6 +44,7 @@ * Proxy for Interpreter instance that runs on separate process */ public class RemoteInterpreter extends Interpreter { + private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class); private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; @@ -186,19 +187,19 @@ public RemoteInterpreterProcess getInterpreterProcess() { clusterManagerKey = Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL; } switch (clusterManagerKey) { - case Constants.ZEPPELIN_CLUSTER_MANAGER_YARN: - remoteProcess = clusterManager - .createInterpreter(sessionKey, interpreterGroupName, group, env, property, - connectTimeout, remoteInterpreterProcessListener, applicationEventListener); - break; - case Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL: - default: - // Default is local process - // create new remote process - remoteProcess = new RemoteInterpreterManagedProcess( - interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, - remoteInterpreterProcessListener, applicationEventListener, - interpreterGroupName); + case Constants.ZEPPELIN_CLUSTER_MANAGER_YARN: + remoteProcess = clusterManager + .createInterpreter(sessionKey, interpreterGroupName, group, env, property, + connectTimeout, remoteInterpreterProcessListener, applicationEventListener); + break; + case Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL: + default: + // Default is local process + // create new remote process + remoteProcess = new RemoteInterpreterManagedProcess( + interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, + remoteInterpreterProcessListener, applicationEventListener, + interpreterGroupName); } } From e326dd44c97fc84a486fc9c0b02a49ac76008897 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 11 Jun 2017 03:28:40 +0900 Subject: [PATCH 22/52] Removed unused files --- .../cluster/yarn/ApplicationMaster.java | 611 ------------------ .../zeppelin/cluster/yarn/DSConstants.java | 45 -- .../zeppelin/cluster/yarn/SimpleYarnApp.java | 195 ------ 3 files changed, 851 deletions(-) delete mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java delete mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java delete mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java deleted file mode 100644 index c6d959251f7..00000000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/ApplicationMaster.java +++ /dev/null @@ -1,611 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.cluster.yarn; - -import java.io.IOException; -import java.lang.reflect.UndeclaredThrowableException; -import java.nio.ByteBuffer; -import java.security.PrivilegedExceptionAction; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Vector; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerExitStatus; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.TimelineClient; -import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; -import org.apache.hadoop.yarn.client.api.async.NMClientAsync; -import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -public class ApplicationMaster { - private static final Logger logger = LoggerFactory.getLogger(ApplicationMaster.class); - - - /** - * - */ - public static enum DSEvent { - DS_APP_ATTEMPT_START, DS_APP_ATTEMPT_END, DS_CONTAINER_START, DS_CONTAINER_END - } - - - /** - * - */ - public static enum DSEntity { - DS_APP_ATTEMPT, DS_CONTAINER - } - - - private final Configuration conf; - - private ByteBuffer allTokens; - - private UserGroupInformation appSubmitterUgi; - - private ApplicationAttemptId appAttemptId; - - private AMRMClientAsync amRMClient; - - private NMClientAsync nmClientAsync; - - private String domainId; - - TimelineClient timelineClient; - - private NMCallbackHandler nmCallbackHandler; - - private String appMasterHostName = ""; - - private int appMasterRpcPort = -1; - - private String appMasterTrackingUrl = ""; - - private int containerMemory = 128; - - private int containerVCores = 1; - - private int numTotalContainers = 1; - - private int requestPriority = 0; - - private boolean done = false; - - private AtomicInteger numRequestedContainers = new AtomicInteger(); - - private AtomicInteger numAllocatedContainers = new AtomicInteger(); - - private AtomicInteger numCompletedContainers = new AtomicInteger(); - - private AtomicInteger numFailedContainers = new AtomicInteger(); - - private List launchThreads = new ArrayList<>(); - - public static void main(String[] args) { - boolean result = false; - try { - ApplicationMaster appMaster = new ApplicationMaster(); - boolean doRun = appMaster.init(args); - if (!doRun) { - System.exit(0); - } - appMaster.run(); - result = appMaster.finish(); - } catch (Throwable t) { - logger.error("Error running ApplicationMaster", t); - ExitUtil.terminate(1, t); - } - - if (result) { - logger.info("Application Master completed successfully. exiting"); - System.exit(0); - } else { - logger.info("Application Master failed. exiting"); - System.exit(2); - } - } - - public ApplicationMaster() { - Configuration hadoopConf = new Configuration(true); - this.conf = new YarnConfiguration(hadoopConf); - } - - public boolean init(String[] args) throws ParseException, IOException { - Map envs = System.getenv(); - for (Map.Entry env : envs.entrySet()) { - logger.info("env. key: " + env.getKey() + ", value: " + env.getValue()); - } - - if (envs.containsKey(ApplicationConstants.Environment.CONTAINER_ID.name())) { - ContainerId containerId = ConverterUtils - .toContainerId(envs.get(ApplicationConstants.Environment.CONTAINER_ID.name())); - appAttemptId = containerId.getApplicationAttemptId(); - } else { - throw new IllegalArgumentException("Application Attempt Id not set in the environment"); - } - - logger.info("Application master for app, appId={}, clustertimestamp={}, attemptId={}", - appAttemptId.getApplicationId().getId(), - appAttemptId.getApplicationId().getClusterTimestamp(), appAttemptId.getAttemptId()); - - if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN)) { - domainId = envs.get(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN); - } - - return true; - } - - public void run() throws IOException, InterruptedException, YarnException { - logger.info("Starting ApplicationMaster"); - - Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); - DataOutputBuffer dob = new DataOutputBuffer(); - credentials.writeTokenStorageToStream(dob); - - Iterator> iter = credentials.getAllTokens().iterator(); - logger.info("Executing with tokens: "); - while (iter.hasNext()) { - Token token = iter.next(); - logger.info(token.toString()); - if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { - iter.remove(); - } - } - allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - - String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); - - appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); - appSubmitterUgi.addCredentials(credentials); - - AMRMClientAsync.CallbackHandler rmCallbackHandler = new RMCallbackHandler(); - amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, rmCallbackHandler); - amRMClient.init(conf); - amRMClient.start(); - - nmCallbackHandler = new NMCallbackHandler(this); - nmClientAsync = new NMClientAsyncImpl(nmCallbackHandler); - nmClientAsync.init(conf); - nmClientAsync.start(); - - startTimelineClient(conf); - if (timelineClient != null) { - publishApplicationAttemptEvent(timelineClient, appAttemptId.toString(), - DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi); - } - - appMasterHostName = NetUtils.getHostname(); - - RegisterApplicationMasterResponse response = amRMClient - .registerApplicationMaster(appMasterHostName, appMasterRpcPort, appMasterTrackingUrl); - - int maxMem = response.getMaximumResourceCapability().getMemory(); - logger.info("Max mem capability of resources in this cluster {}", maxMem); - - int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); - logger.info("Max vcores capability of resources in this cluster {}", maxVCores); - - if (containerMemory > maxMem) { - containerMemory = maxMem; - logger.info("container memory changed to {} because of limitation of resources", maxMem); - } - - if (containerVCores > maxVCores) { - containerVCores = maxVCores; - logger.info("container vcores changed to {} because of limitation of resources", maxVCores); - } - - List previousAMRunningContainers = response.getContainersFromPreviousAttempts(); - numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); - - int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size(); - - for (int i = 0; i < numTotalContainersToRequest; i++) { - AMRMClient.ContainerRequest containerRequest = setupContainerAskForRM(); - amRMClient.addContainerRequest(containerRequest); - } - numRequestedContainers.set(numTotalContainers); - - } - - public boolean finish() { - while (!done && (numCompletedContainers.get() != numTotalContainers)) { - try { - Thread.sleep(200); - } catch (InterruptedException e) { - } - } - - if (timelineClient != null) { - publishApplicationAttemptEvent(timelineClient, appAttemptId.toString(), - DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi); - } - - for (Thread launchThread : launchThreads) { - try { - launchThread.join(10000); - } catch (InterruptedException e) { - logger.info("Exception thrown in thread join", e); - } - } - - logger.info("Application completed. Stopping running containers"); - nmClientAsync.stop(); - - logger.info("Application completed. Signalling finish to RM"); - - FinalApplicationStatus appStatus; - String appMessage = null; - boolean success = true; - if (numFailedContainers.get() == 0 && numCompletedContainers.get() == numTotalContainers) { - appStatus = FinalApplicationStatus.SUCCEEDED; - } else { - appStatus = FinalApplicationStatus.FAILED; - appMessage = - "Diagnostics. total=" + numTotalContainers + ",completed=" + numCompletedContainers.get() - + ", allocated=" + numAllocatedContainers.get() + ", failed=" + numFailedContainers - .get(); - logger.info(appMessage); - success = false; - } - try { - amRMClient.unregisterApplicationMaster(appStatus, appMessage, null); - } catch (YarnException | IOException e) { - logger.error("Failed to unregister application", e); - } - - amRMClient.stop(); - - if (null != timelineClient) { - timelineClient.stop(); - } - - return success; - } - - private AMRMClient.ContainerRequest setupContainerAskForRM() { - Priority pri = Priority.newInstance(requestPriority); - - Resource capability = Resource.newInstance(containerMemory, containerVCores); - - AMRMClient.ContainerRequest request = - new AMRMClient.ContainerRequest(capability, null, null, pri); - logger.info("Requested container ask: {}", request.toString()); - - return request; - } - - private void startTimelineClient(final Configuration conf) - throws YarnException, IOException, InterruptedException { - try { - appSubmitterUgi.doAs(new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { - // Creating the Timeline Client - timelineClient = TimelineClient.createTimelineClient(); - timelineClient.init(conf); - timelineClient.start(); - } else { - timelineClient = null; - logger.warn("Timeline service is not enabled"); - } - return null; - } - }); - } catch (UndeclaredThrowableException e) { - throw new YarnException(e.getCause()); - } - } - - private static void publishApplicationAttemptEvent(final TimelineClient timelineClient, - String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) { - final TimelineEntity entity = new TimelineEntity(); - entity.setEntityId(appAttemptId); - entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString()); - entity.setDomainId(domainId); - entity.addPrimaryFilter("user", ugi.getShortUserName()); - TimelineEvent event = new TimelineEvent(); - event.setEventType(appEvent.toString()); - event.setTimestamp(System.currentTimeMillis()); - entity.addEvent(event); - try { - timelineClient.putEntities(entity); - } catch (YarnException | IOException e) { - logger.error( - "App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end") - + " event could not be published for " + appAttemptId, e); - } - } - - private static void publishContainerStartEvent(final TimelineClient timelineClient, - Container container, String domainId, UserGroupInformation ugi) { - final TimelineEntity entity = new TimelineEntity(); - entity.setEntityId(container.getId().toString()); - entity.setEntityType(DSEntity.DS_CONTAINER.toString()); - entity.setDomainId(domainId); - entity.addPrimaryFilter("user", ugi.getShortUserName()); - TimelineEvent event = new TimelineEvent(); - event.setTimestamp(System.currentTimeMillis()); - event.setEventType(DSEvent.DS_CONTAINER_START.toString()); - event.addEventInfo("Node", container.getNodeId().toString()); - event.addEventInfo("Resources", container.getResource().toString()); - entity.addEvent(event); - try { - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public TimelinePutResponse run() throws Exception { - return timelineClient.putEntities(entity); - } - }); - } catch (Exception e) { - logger - .error("Container start event could not be published for " + container.getId().toString(), - e instanceof UndeclaredThrowableException ? e.getCause() : e); - } - } - - private static void publishContainerEndEvent(final TimelineClient timelineClient, - ContainerStatus container, String domainId, UserGroupInformation ugi) { - final TimelineEntity entity = new TimelineEntity(); - entity.setEntityId(container.getContainerId().toString()); - entity.setEntityType(DSEntity.DS_CONTAINER.toString()); - entity.setDomainId(domainId); - entity.addPrimaryFilter("user", ugi.getShortUserName()); - TimelineEvent event = new TimelineEvent(); - event.setTimestamp(System.currentTimeMillis()); - event.setEventType(DSEvent.DS_CONTAINER_END.toString()); - event.addEventInfo("State", container.getState().name()); - event.addEventInfo("Exit Status", container.getExitStatus()); - entity.addEvent(event); - try { - timelineClient.putEntities(entity); - } catch (YarnException | IOException e) { - logger.error( - "Container end event could not be published for " + container.getContainerId().toString(), - e); - } - } - - private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler { - @Override - public void onContainersCompleted(List completedContainers) { - logger.info( - "Got response from RM for container ask, completedCnt=" + completedContainers.size()); - for (ContainerStatus containerStatus : completedContainers) { - logger.info(appAttemptId + " got container status for containerID=" + containerStatus - .getContainerId() + ", state=" + containerStatus.getState() + ", exitStatus=" - + containerStatus.getExitStatus() + ", diagnostics=" + containerStatus - .getDiagnostics()); - - // non complete containers should not be here - assert (containerStatus.getState() == ContainerState.COMPLETE); - - // increment counters for completed/failed containers - int exitStatus = containerStatus.getExitStatus(); - if (0 != exitStatus) { - // container failed - if (ContainerExitStatus.ABORTED != exitStatus) { - // shell script failed - // counts as completed - numCompletedContainers.incrementAndGet(); - numFailedContainers.incrementAndGet(); - } else { - // container was killed by framework, possibly preempted - // we should re-try as the container was lost for some reason - numAllocatedContainers.decrementAndGet(); - numRequestedContainers.decrementAndGet(); - // we do not need to release the container as it would be done - // by the RM - } - } else { - // nothing to do - // container completed successfully - numCompletedContainers.incrementAndGet(); - logger.info("Container completed successfully." + ", containerId=" + containerStatus - .getContainerId()); - } - if (timelineClient != null) { - publishContainerEndEvent(timelineClient, containerStatus, domainId, appSubmitterUgi); - } - } - } - - @Override - public void onContainersAllocated(List allocatedContainers) { - logger.info( - "Got response from RM for container ask, allocatedCnt=" + allocatedContainers.size()); - numAllocatedContainers.addAndGet(allocatedContainers.size()); - for (Container allocatedContainer : allocatedContainers) { - logger.info( - "Launching shell command on a new container." + ", containerId=" + allocatedContainer - .getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" - + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" - + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory" - + allocatedContainer.getResource().getMemory() + ", containerResourceVirtualCores" - + allocatedContainer.getResource().getVirtualCores()); - // + ", containerToken" - // +allocatedContainer.getContainerToken().getIdentifier().toString()); - - LaunchContainerRunnable runnableLaunchContainer = - new LaunchContainerRunnable(allocatedContainer, nmCallbackHandler); - Thread launchThread = new Thread(runnableLaunchContainer); - - // launch and start the container on a separate thread to keep - // the main thread unblocked - // as all containers may not be allocated at one go. - launchThreads.add(launchThread); - launchThread.start(); - } - - } - - @Override - public void onShutdownRequest() { - done = true; - } - - @Override - public void onNodesUpdated(List list) { - - } - - @Override - public float getProgress() { - return (float) numCompletedContainers.get() / numTotalContainers; - } - - @Override - public void onError(Throwable throwable) { - done = true; - amRMClient.stop(); - } - } - - - private class NMCallbackHandler implements NMClientAsync.CallbackHandler { - private ConcurrentMap containers = new ConcurrentHashMap<>(); - private final ApplicationMaster applicationMaster; - - public NMCallbackHandler(ApplicationMaster applicationMaster) { - this.applicationMaster = applicationMaster; - } - - public void addContainer(ContainerId containerId, Container container) { - containers.putIfAbsent(containerId, container); - } - - @Override - public void onContainerStarted(ContainerId containerId, Map map) { - logger.debug("Succeeded to start Container {}", containerId); - Container container = containers.get(containerId); - if (container != null) { - applicationMaster.nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId()); - } - if (applicationMaster.timelineClient != null) { - ApplicationMaster - .publishContainerStartEvent(timelineClient, container, applicationMaster.domainId, - applicationMaster.appSubmitterUgi); - } - } - - @Override - public void onContainerStatusReceived(ContainerId containerId, - ContainerStatus containerStatus) { - logger.debug("Container status: id={}, status={}", containerId, containerStatus); - - } - - @Override - public void onContainerStopped(ContainerId containerId) { - logger.debug("Succeeded to stop Container {}", containerId); - containers.remove(containerId); - } - - @Override - public void onStartContainerError(ContainerId containerId, Throwable throwable) { - logger.error("Failed to start Container {}", containerId); - containers.remove(containerId); - applicationMaster.numCompletedContainers.incrementAndGet(); - applicationMaster.numFailedContainers.incrementAndGet(); - } - - @Override - public void onGetContainerStatusError(ContainerId containerId, Throwable throwable) { - logger.error("Failed to query the status of Container {}", containerId); - } - - @Override - public void onStopContainerError(ContainerId containerId, Throwable throwable) { - logger.error("Failed to stop Container {}", containerId); - containers.remove(containerId); - } - } - - - private class LaunchContainerRunnable implements Runnable { - Container container; - NMCallbackHandler containerListener; - - public LaunchContainerRunnable(Container container, NMCallbackHandler containerListener) { - this.container = container; - this.containerListener = containerListener; - } - - @Override - public void run() { - logger.info("Setting up container launch container for containerid={}", container.getId()); - - Map localResources = new HashMap<>(); - - Vector vargs = new Vector<>(30); - vargs.add("echo \"madeng\""); - vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); - vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); - String command = Joiner.on(" ").join(vargs); - ContainerLaunchContext ctx = ContainerLaunchContext - .newInstance(localResources, System.getenv(), Lists.newArrayList(command), null, - allTokens.duplicate(), null); - containerListener.addContainer(container.getId(), container); - nmClientAsync.startContainerAsync(container, ctx); - } - } - -} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java deleted file mode 100644 index 190cedc4257..00000000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/DSConstants.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.cluster.yarn; - -/** - * - */ -public class DSConstants { - /** - * Environment key name pointing to the shell script's location - */ - public static final String DISTRIBUTEDSHELLSCRIPTLOCATION = "DISTRIBUTEDSHELLSCRIPTLOCATION"; - - /** - * Environment key name denoting the file timestamp for the shell script. - * Used to validate the local resource. - */ - public static final String DISTRIBUTEDSHELLSCRIPTTIMESTAMP = "DISTRIBUTEDSHELLSCRIPTTIMESTAMP"; - - /** - * Environment key name denoting the file content length for the shell script. - * Used to validate the local resource. - */ - public static final String DISTRIBUTEDSHELLSCRIPTLEN = "DISTRIBUTEDSHELLSCRIPTLEN"; - - /** - * Environment key name denoting the timeline domain ID. - */ - public static final String DISTRIBUTEDSHELLTIMELINEDOMAIN = "DISTRIBUTEDSHELLTIMELINEDOMAIN"; -} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java deleted file mode 100644 index fd383ecf915..00000000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/SimpleYarnApp.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.cluster.yarn; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Scanner; -import java.util.Vector; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -public class SimpleYarnApp { - private static final Logger logger = LoggerFactory.getLogger(SimpleYarnApp.class); - - private String appName = "SIMPLE"; - - private String appMasterJar = "/Users/jl/local/src/g/zeppelin/zeppelin-cluster/yarn/target/" - + "zeppelin-cluster-yarn-0.8.0-SNAPSHOT.jar"; - - private String appMasterJarPath = "zeppelin-cluster.jar"; - - private String appMasterMainClass = "org.apache.zeppelin.cluster.yarn.ApplicationMaster"; - - public static void main(String[] args) throws Exception { - new SimpleYarnApp().run(); - } - - public void run() throws Exception { - boolean keepContainers = false; - String appName = "test"; - Configuration conf = new YarnConfiguration(); - - YarnClient yarnClient = YarnClient.createYarnClient(); - yarnClient.init(conf); - yarnClient.start(); - - YarnClientApplication app = yarnClient.createApplication(); - GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); - - // set the application submission context - ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); - ApplicationId appId = appContext.getApplicationId(); - - appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); - appContext.setApplicationName(appName); - - Map localResources = new HashMap<>(); - - FileSystem fs = FileSystem.get(conf); - addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); - - Map env = new HashMap<>(); - - StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$$()) - .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); - for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, - YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { - classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); - classPathEnv.append(c.trim()); - } - - logger.debug("classpath: {}", classPathEnv.toString()); - - env.put("CLASSPATH", classPathEnv.toString()); - - Vector vargs = new Vector<>(30); - - vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); - - vargs.add("-Xmx128m"); - - vargs.add(appMasterMainClass); - - vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ApplicationMaster.stdout"); - vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ApplicationMaster.stderr"); - String command = Joiner.on(" ").join(vargs); - logger.info("command: {}", command); - List commands = Lists.newArrayList(command); - - ContainerLaunchContext amContainer = - ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); - - Resource capability = Resource.newInstance(128, 1); - appContext.setResource(capability); - - if (UserGroupInformation.isSecurityEnabled()) { - Credentials credentials = new Credentials(); - String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); - if (tokenRenewer == null || tokenRenewer.length() == 0) { - throw new IOException("Can't get Master Kerberos prinipal for the RM to use as renewer"); - } - - final Token tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); - if (tokens != null) { - for (Token token : tokens) { - logger.info("Got dt from " + fs.getUri() + "; " + token); - } - } - - DataOutputBuffer dob = new DataOutputBuffer(); - credentials.writeTokenStorageToStream(dob); - ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - amContainer.setTokens(fsTokens); - } - - appContext.setAMContainerSpec(amContainer); - - Priority pri = Priority.newInstance(0); - appContext.setPriority(pri); - - appContext.setQueue("default"); - - appContext.setApplicationType("ZEPPELIN"); - - yarnClient.submitApplication(appContext); - - ApplicationReport report = yarnClient.getApplicationReport(appId); - - yarnClient.stop(); - } - - private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, - String appId, Map localResources, String resources) - throws IOException { - String suffix = appName + "/" + appId + "/" + fileDstPath; - Path dst = new Path(fs.getHomeDirectory(), suffix); - if (fileSrcPath == null) { - FSDataOutputStream ostream = null; - try { - ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710)); - ostream.writeUTF(resources); - } finally { - IOUtils.closeQuietly(ostream); - } - } else { - fs.copyFromLocalFile(new Path(fileSrcPath), dst); - } - FileStatus scFileStatus = fs.getFileStatus(dst); - LocalResource scRsrc = LocalResource - .newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, - LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), - scFileStatus.getModificationTime()); - localResources.put(fileDstPath, scRsrc); - } -} From 6353266da294d444babbd1a37ed1474b7a26cc7d Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 11 Jun 2017 23:37:42 +0900 Subject: [PATCH 23/52] Refactored the structure of cluster manage in order to extend it --- pom.xml | 36 +++- zeppelin-cluster/common/pom.xml | 45 +++++ .../zeppelin/cluster/ClusterManager.java | 13 +- zeppelin-cluster/yarn/pom.xml | 158 ++++++++++++++++- .../apache/zeppelin/cluster/yarn/Client.java | 36 +--- .../yarn/RemoteInterpreterYarnProcess.java | 37 ++-- .../yarn/YarnRemoteInterpreterServer.java | 0 .../zeppelin/cluster/yarn/YarnUtils.java | 0 zeppelin-zengine/pom.xml | 160 +----------------- .../interpreter/InterpreterFactory.java | 2 +- .../interpreter/remote/RemoteInterpreter.java | 7 +- 11 files changed, 274 insertions(+), 220 deletions(-) create mode 100644 zeppelin-cluster/common/pom.xml rename {zeppelin-zengine => zeppelin-cluster/common}/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java (80%) rename {zeppelin-zengine => zeppelin-cluster/yarn}/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java (78%) rename {zeppelin-zengine => zeppelin-cluster/yarn}/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java (94%) rename {zeppelin-zengine => zeppelin-cluster/yarn}/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java (100%) rename {zeppelin-zengine => zeppelin-cluster/yarn}/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java (100%) diff --git a/pom.xml b/pom.xml index 1c173016f55..7f1a480ec6c 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,8 @@ zeppelin-interpreter + zeppelin-cluster/common + zeppelin-cluster/yarn zeppelin-zengine zeppelin-display spark-dependencies @@ -98,6 +100,7 @@ 0.2.1 15.0 9.2.15.v20160210 + 2.7.3 4.4.1 4.5.1 4.0.2 @@ -265,6 +268,38 @@ ${shiro.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop.version} + + + + org.apache.hadoop + hadoop-yarn-common + ${hadoop.version} + + + + org.apache.hadoop + hadoop-yarn-client + ${hadoop.version} + + + junit @@ -321,7 +356,6 @@ ${powermock.version} test - diff --git a/zeppelin-cluster/common/pom.xml b/zeppelin-cluster/common/pom.xml new file mode 100644 index 00000000000..b025b113648 --- /dev/null +++ b/zeppelin-cluster/common/pom.xml @@ -0,0 +1,45 @@ + + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.8.0-SNAPSHOT + ../.. + + + zeppelin-cluster-common + jar + 0.8.0-SNAPSHOT + Zeppelin Cluster: Common + Zeppelin Cluster + + + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + + + + diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java similarity index 80% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java rename to zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java index 8e83f5a45fa..a4e80d5f45a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -19,10 +19,9 @@ import java.util.Map; import java.util.Properties; -import org.apache.zeppelin.conf.ZeppelinConfiguration; + import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; @@ -31,12 +30,6 @@ */ public abstract class ClusterManager { - protected final ZeppelinConfiguration zeppelinConfiguration; - - protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration) { - this.zeppelinConfiguration = zeppelinConfiguration; - } - /** * Can throw `RuntimeException` */ @@ -49,8 +42,8 @@ protected ClusterManager(ZeppelinConfiguration zeppelinConfiguration) { public abstract RemoteInterpreterProcess createInterpreter(String id, String name, String groupName, Map env, Properties properties, int connectTimeout, - RemoteInterpreterProcessListener listener, - ApplicationEventListener appListener) + RemoteInterpreterProcessListener listener, ApplicationEventListener appListener, + String homeDir, String interpreterDir) throws InterpreterException; public abstract void releaseResource(String id); diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml index b0ea366db8d..ff5f145e7b2 100644 --- a/zeppelin-cluster/yarn/pom.xml +++ b/zeppelin-cluster/yarn/pom.xml @@ -37,9 +37,165 @@ org.apache.zeppelin - zeppelin-interpreter + zeppelin-cluster-common ${project.version} + + + org.apache.hadoop + hadoop-yarn-client + + + + org.apache.hadoop + hadoop-yarn-common + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + org.apache.commons + commons-compress + + + org.codehaus.jackson + jackson-mapper-asl + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-core + + + javax.servlet + servlet-api + + + + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop.version} + + + + org.apache.hadoop + hadoop-common + + + javax.servlet + servlet-api + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + com.jcraft + jsch + + + org.apache.commons + commons-compress + + + org.codehaus.jackson + jackson-mapper-asl + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + javax.servlet + servlet-api + + + + + + org.apache.hadoop + hadoop-hdfs + + + javax.servlet + servlet-api + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + com.jcraft + jsch + + + org.apache.commons + commons-compress + + + org.codehaus.jackson + jackson-mapper-asl + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + javax.servlet + servlet-api + + + xerces + xercesImpl + + + io.netty + netty + + + diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java similarity index 78% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java rename to zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 63b5058e849..00f18a793e5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -18,50 +18,23 @@ package org.apache.zeppelin.cluster.yarn; import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.zeppelin.cluster.ClusterManager; -import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; @@ -83,9 +56,7 @@ public class Client extends ClusterManager { */ private Map idApplicationIdMap; - public Client(ZeppelinConfiguration zeppelinConfiguration) { - super(zeppelinConfiguration); - + public Client() { this.started = false; } @@ -121,14 +92,15 @@ public synchronized void stop() { @Override public RemoteInterpreterProcess createInterpreter(String id, String name, String groupName, Map env, Properties properties, int connectTimeout, - RemoteInterpreterProcessListener listener, ApplicationEventListener appListener) + RemoteInterpreterProcessListener listener, ApplicationEventListener appListener, + String homeDir, String interpreterDir) throws InterpreterException { if (!started) { start(); } return new RemoteInterpreterYarnProcess(connectTimeout, listener, appListener, yarnClient, - zeppelinConfiguration, configuration, name, groupName, env, properties); + homeDir, interpreterDir, configuration, name, groupName, env, properties); } public void releaseResource(String id) { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java similarity index 94% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java rename to zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index a74e1e2a73e..f565727855e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -34,7 +34,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; - import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.apache.hadoop.conf.Configuration; @@ -53,7 +52,6 @@ import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,10 +69,10 @@ import static org.apache.zeppelin.cluster.yarn.YarnUtils.getPathsFromDirPath; import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_APPLICATION_TYPE_DEFAULT; import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_APPLICATION_TYPE_KEY; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_PRIORITY_DEFAULT; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_PRIORITY_KEY; import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_MEMORY_DEFAULT; import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_MEMORY_KEY; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_PRIORITY_DEFAULT; +import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_PRIORITY_KEY; import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_QUEUE_DEFAULT; import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_QUEUE_KEY; import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_VCORES_DEFAULT; @@ -88,7 +86,8 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterYarnProcess.class); private final YarnClient yarnClient; - private final ZeppelinConfiguration zeppelinConfiguration; + private final String homeDir; + private final String interpreterDir; private final Configuration configuration; private final String name; private final String group; @@ -107,12 +106,13 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { private int port = -1; RemoteInterpreterYarnProcess(int connectTimeout, RemoteInterpreterProcessListener listener, - ApplicationEventListener appListener, YarnClient yarnClient, - ZeppelinConfiguration zeppelinConfiguration, Configuration configuration, String name, - String group, Map env, Properties properties) { + ApplicationEventListener appListener, YarnClient yarnClient, String homeDir, + String interpreterDir, Configuration configuration, String name, String group, + Map env, Properties properties) { super(new RemoteInterpreterEventPoller(listener, appListener), connectTimeout); this.yarnClient = yarnClient; - this.zeppelinConfiguration = zeppelinConfiguration; + this.homeDir = homeDir; + this.interpreterDir = interpreterDir; this.configuration = configuration; this.name = name; this.group = group; @@ -121,15 +121,14 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { this.waitingInitialized = new CountDownLatch(1); this.interpreterLibPaths = Lists.newArrayList( - Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target"), - Paths.get(zeppelinConfiguration.getHome(), "zeppelin-interpreter", "target", "lib"), - Paths.get(zeppelinConfiguration.getHome(), "zeppelin-zengine", "target"), - Paths.get(zeppelinConfiguration.getHome(), "zeppelin-zengine", "target", "lib"), - Paths.get(zeppelinConfiguration.getHome(), "conf", "yarn", "log4j.properties"), - Paths.get(zeppelinConfiguration.getHome(), "lib", "interpreter"), - Paths.get(zeppelinConfiguration.getHome(), "lib", - "zeppelin-zengine-0.8.0-SNAPSHOT.jar"), - Paths.get(zeppelinConfiguration.getHome(), "lib", "zengine")); + Paths.get(homeDir, "zeppelin-interpreter", "target"), + Paths.get(homeDir, "zeppelin-interpreter", "target", "lib"), + Paths.get(homeDir, "zeppelin-zengine", "target"), + Paths.get(homeDir, "zeppelin-zengine", "target", "lib"), + Paths.get(homeDir, "conf", "yarn", "log4j.properties"), + Paths.get(homeDir, "lib", "interpreter"), + Paths.get(homeDir, "lib", "zeppelin-zengine-0.8.0-SNAPSHOT.jar"), + Paths.get(homeDir, "lib", "zengine")); } @Override @@ -426,7 +425,7 @@ private int convertSparkMemoryFormat(String memoryFormat) { } private Path getInterpreterRelativePath(String... dirNames) { - return Paths.get(zeppelinConfiguration.getInterpreterDir(), dirNames); + return Paths.get(this.interpreterDir, dirNames); } @Override diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java similarity index 100% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java rename to zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java similarity index 100% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java rename to zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnUtils.java diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 7467bc33d0d..ee9603f7667 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -76,163 +76,15 @@ - org.apache.hadoop - hadoop-yarn-client - ${hadoop.version} - - - - org.apache.hadoop - hadoop-yarn-common - ${hadoop.version} - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-jaxrs - - - org.codehaus.jackson - jackson-xc - - - org.apache.commons - commons-compress - - - org.codehaus.jackson - jackson-mapper-asl - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-core - - - javax.servlet - servlet-api - - - - - - org.apache.hadoop - hadoop-yarn-api - ${hadoop.version} - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - javax.servlet - servlet-api - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-jaxrs - - - org.codehaus.jackson - jackson-xc - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - org.codehaus.jackson - jackson-mapper-asl - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - javax.servlet - servlet-api - - + org.apache.zeppelin + zeppelin-cluster-common + ${project.version} - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - - - javax.servlet - servlet-api - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-jaxrs - - - org.codehaus.jackson - jackson-xc - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - org.codehaus.jackson - jackson-mapper-asl - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - javax.servlet - servlet-api - - - xerces - xercesImpl - - - io.netty - netty - - + org.apache.zeppelin + zeppelin-cluster-yarn + ${project.version} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index b13ca217f6d..7a16a05a39c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -283,7 +283,7 @@ Interpreter createRemoteRepl(String interpreterPath, String interpreterSessionKe } RemoteInterpreter remoteInterpreter = - new RemoteInterpreter(property, interpreterSessionKey, className, + new RemoteInterpreter(property, interpreterSessionKey, className, conf.getHome(), interpreterRunnerPath, interpreterPath, localRepoPath, connectTimeout, maxPoolSize, remoteInterpreterProcessListener, appEventListener, userName, isUserImpersonate, conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName, diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 2fbca35bf2c..c672fee88c3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -50,6 +50,7 @@ public class RemoteInterpreter extends Interpreter { private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; private final ApplicationEventListener applicationEventListener; private Gson gson = new Gson(); + private String homeDir; private String interpreterRunner; private String interpreterPath; private String localRepoPath; @@ -72,7 +73,7 @@ public class RemoteInterpreter extends Interpreter { /** * Remote interpreter and manage interpreter process */ - public RemoteInterpreter(Properties property, String sessionKey, String className, + public RemoteInterpreter(Properties property, String sessionKey, String className, String homeDir, String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout, int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appListener, String userName, Boolean isUserImpersonate, @@ -81,6 +82,7 @@ public RemoteInterpreter(Properties property, String sessionKey, String classNam this.sessionKey = sessionKey; this.className = className; initialized = false; + this.homeDir = homeDir; this.interpreterRunner = interpreterRunner; this.interpreterPath = interpreterPath; this.localRepoPath = localRepoPath; @@ -190,7 +192,8 @@ public RemoteInterpreterProcess getInterpreterProcess() { case Constants.ZEPPELIN_CLUSTER_MANAGER_YARN: remoteProcess = clusterManager .createInterpreter(sessionKey, interpreterGroupName, group, env, property, - connectTimeout, remoteInterpreterProcessListener, applicationEventListener); + connectTimeout, remoteInterpreterProcessListener, applicationEventListener, + homeDir, interpreterPath); break; case Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL: default: From b0de6acba33b6d9c0d0dfa272caee0dffd9a3730 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 11 Jun 2017 23:47:45 +0900 Subject: [PATCH 24/52] Cleaned up the code --- .../zeppelin/spark/PySparkInterpreter.java | 2 -- .../src/assemble/distribution.xml | 4 ---- zeppelin-server/pom.xml | 2 -- zeppelin-zengine/pom.xml | 16 ---------------- .../zeppelin/interpreter/InterpreterFactory.java | 2 +- 5 files changed, 1 insertion(+), 25 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index 66eb7c8c329..d312ac1bd92 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -186,8 +186,6 @@ private Map setupPySparkEnv() throws IOException{ String sparkSubmitJars = getSparkConf().get("spark.jars").replace(",", ":"); if (!"".equals(sparkSubmitJars)) { - - env.put("PYTHONPATH", env.get("PYTHONPATH") + sparkSubmitJars); } } diff --git a/zeppelin-distribution/src/assemble/distribution.xml b/zeppelin-distribution/src/assemble/distribution.xml index 00ca2eb818c..5c369e256a3 100644 --- a/zeppelin-distribution/src/assemble/distribution.xml +++ b/zeppelin-distribution/src/assemble/distribution.xml @@ -107,9 +107,5 @@ /lib/node_modules/zeppelin-spell ../zeppelin-web/src/app/spell - - /lib/zengine - ../zeppelin-zengine/target/lib - diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 1df9dac6126..b9d8d8bdcf6 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -36,7 +36,6 @@ 4.3.6 2.22.2 - 2.7.3 2.2.1 1.13 2.0.1 @@ -235,7 +234,6 @@ org.apache.hadoop hadoop-common - ${hadoop-common.version} com.sun.jersey diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index ee9603f7667..2eec26d3c58 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -47,27 +47,11 @@ 1.4.01 4.1.1.201511131810-r 1.3 - 2.7.3 0.27 - - - - com.fasterxml.jackson.core - jackson-core - 2.6.5 - - - com.fasterxml.jackson.core - jackson-databind - 2.6.5 - - - - ${project.groupId} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 7a16a05a39c..536d21245de 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -20,7 +20,6 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import org.apache.commons.lang.NullArgumentException; -import org.apache.zeppelin.cluster.ClusterManager; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.DependencyResolver; @@ -53,6 +52,7 @@ * Manage interpreters. */ public class InterpreterFactory implements InterpreterGroupFactory { + private static final Logger logger = LoggerFactory.getLogger(InterpreterFactory.class); private Map cleanCl = From 43df28327171e1ae7c66472378471a0c6d7b26bc Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 11 Jun 2017 23:51:06 +0900 Subject: [PATCH 25/52] Removed yarn/log4j --- conf/yarn/log4j.properties | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 conf/yarn/log4j.properties diff --git a/conf/yarn/log4j.properties b/conf/yarn/log4j.properties deleted file mode 100644 index 8daee59d60d..00000000000 --- a/conf/yarn/log4j.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -log4j.rootLogger = INFO, stdout - -log4j.appender.stdout = org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout = org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n From 491654df0a2a68b3bb23c2e805fc20f7e3f02b0d Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 11 Jun 2017 23:54:31 +0900 Subject: [PATCH 26/52] Chagned the scope of hadoop libraries into cluster-yarn --- zeppelin-cluster/yarn/pom.xml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml index ff5f145e7b2..6f41affde95 100644 --- a/zeppelin-cluster/yarn/pom.xml +++ b/zeppelin-cluster/yarn/pom.xml @@ -44,11 +44,13 @@ org.apache.hadoop hadoop-yarn-client + provided org.apache.hadoop hadoop-yarn-common + provided org.codehaus.jackson @@ -92,12 +94,13 @@ org.apache.hadoop hadoop-yarn-api - ${hadoop.version} + provided org.apache.hadoop hadoop-common + provided javax.servlet @@ -145,6 +148,7 @@ org.apache.hadoop hadoop-hdfs + provided javax.servlet From ae836d1ddeca900b94722dfa9fa5ea929f55702a Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 12 Jun 2017 00:23:10 +0900 Subject: [PATCH 27/52] Fixed profiles in travis --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5a728f72e26..e0134b5aeba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ matrix: # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk7" - env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" + env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" # Test interpreter modules - jdk: "oraclejdk7" @@ -63,19 +63,19 @@ matrix: # Test spark module for 2.1.0 with scala 2.11, livy - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" + env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" # Test spark module for 2.0.2 with scala 2.11 - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 1.6.3 with scala 2.10 - jdk: "oraclejdk7" - env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 1.6.3 with scala 2.11 - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test python/pyspark with python 2, livy 0.2 - sudo: required From 5ddcc07ef6d268cf7cb7211218ce27476ad157bc Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 12 Jun 2017 01:02:33 +0900 Subject: [PATCH 28/52] Fixed profiles in travis --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index e0134b5aeba..bc3bd243cd0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ matrix: # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk7" - env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" + env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-cluster/common,zeppelin-cluster/yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" # Test interpreter modules - jdk: "oraclejdk7" @@ -63,19 +63,19 @@ matrix: # Test spark module for 2.1.0 with scala 2.11, livy - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" + env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster/common,zeppelin-cluster/yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" # Test spark module for 2.0.2 with scala 2.11 - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster/common,zeppelin-cluster/yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 1.6.3 with scala 2.10 - jdk: "oraclejdk7" - env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster/common,zeppelin-cluster/yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 1.6.3 with scala 2.11 - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster-common,zeppelin-cluster-yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-cluster/common,zeppelin-cluster/yarn,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test python/pyspark with python 2, livy 0.2 - sudo: required From b933f74364813a9e0b23a5ab69a29ad76fbe57c4 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 12 Jun 2017 01:56:06 +0900 Subject: [PATCH 29/52] Caught up error --- .../org/apache/zeppelin/server/ZeppelinServer.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index d50e8b02f82..6fe69178467 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -88,7 +88,7 @@ public class ZeppelinServer extends Application { public static Helium helium; private final InterpreterSettingManager interpreterSettingManager; - private final ClusterManager clusterManager; + private ClusterManager clusterManager; private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; private SearchService noteSearchService; @@ -149,8 +149,14 @@ public ZeppelinServer() throws Exception { this.schedulerFactory = new SchedulerFactory(); this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - this.clusterManager = (ClusterManager) Class.forName("org.apache.zeppelin.cluster.yarn.Client") - .getConstructor(ZeppelinConfiguration.class).newInstance(conf); + try { + this.clusterManager = (ClusterManager) Class + .forName("org.apache.zeppelin.cluster.yarn.Client") + .getConstructor(ZeppelinConfiguration.class).newInstance(conf); + } catch (Exception e) { + // TODO(jl): To be fixed + this.clusterManager = null; + } this.replFactory = new InterpreterFactory(conf, notebookWsServer, notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(), interpreterSettingManager, clusterManager); From 95e22e5f9010ef8128b7e43acc2ec5754f03e5a6 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 12 Jun 2017 02:52:33 +0900 Subject: [PATCH 30/52] Caught up error --- .../main/java/org/apache/zeppelin/server/ZeppelinServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 6fe69178467..efbd201912c 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -153,7 +153,7 @@ public ZeppelinServer() throws Exception { this.clusterManager = (ClusterManager) Class .forName("org.apache.zeppelin.cluster.yarn.Client") .getConstructor(ZeppelinConfiguration.class).newInstance(conf); - } catch (Exception e) { + } catch (Throwable t) { // TODO(jl): To be fixed this.clusterManager = null; } From 83dd41c7cf640477e92b2e9b53dfcb816e7ed001 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 12 Jun 2017 22:57:28 +0900 Subject: [PATCH 31/52] Added HADOOP_CONF_DIR in classpath --- bin/common.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/common.sh b/bin/common.sh index c7100c7d022..1a3bd9befcb 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -54,6 +54,10 @@ fi ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}" +if [[ -z "${HADOOP_CONF_DIR}" ]]; then + ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" +fi + function addEachJarInDir(){ if [[ -d "${1}" ]]; then for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do From f002cd646ac266deab491f277ac46bae126d8f5a Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 18 Jun 2017 23:33:58 +0900 Subject: [PATCH 32/52] Moved yarn related files from zengine to cluster-manager/yarn --- zeppelin-cluster/common/pom.xml | 1 + zeppelin-cluster/yarn/pom.xml | 8 +++++++ .../yarn/RemoteInterpreterYarnProcess.java | 11 ++++++---- .../src/assemble/distribution.xml | 22 +++++++++++++++++++ .../zeppelin/server/ZeppelinServer.java | 3 +-- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/zeppelin-cluster/common/pom.xml b/zeppelin-cluster/common/pom.xml index b025b113648..3769db32938 100644 --- a/zeppelin-cluster/common/pom.xml +++ b/zeppelin-cluster/common/pom.xml @@ -39,6 +39,7 @@ org.apache.zeppelin zeppelin-interpreter ${project.version} + provided diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml index 6f41affde95..4bc5c1fc319 100644 --- a/zeppelin-cluster/yarn/pom.xml +++ b/zeppelin-cluster/yarn/pom.xml @@ -35,10 +35,18 @@ Zeppelin Yarn Cluster + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + provided + + org.apache.zeppelin zeppelin-cluster-common ${project.version} + provided diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index f565727855e..583b10f7428 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -123,12 +123,14 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { this.interpreterLibPaths = Lists.newArrayList( Paths.get(homeDir, "zeppelin-interpreter", "target"), Paths.get(homeDir, "zeppelin-interpreter", "target", "lib"), - Paths.get(homeDir, "zeppelin-zengine", "target"), - Paths.get(homeDir, "zeppelin-zengine", "target", "lib"), + Paths.get(homeDir, "zeppelin-cluster", "common", "target"), + Paths.get(homeDir, "zeppelin-cluster", "common", "target", "lib"), + Paths.get(homeDir, "zeppelin-cluster", "yarn", "target"), + Paths.get(homeDir, "zeppelin-cluster", "yarn", "target", "lib"), Paths.get(homeDir, "conf", "yarn", "log4j.properties"), Paths.get(homeDir, "lib", "interpreter"), - Paths.get(homeDir, "lib", "zeppelin-zengine-0.8.0-SNAPSHOT.jar"), - Paths.get(homeDir, "lib", "zengine")); + Paths.get(homeDir, "lib", "cluster", "common"), + Paths.get(homeDir, "lib", "cluster", "yarn")); } @Override @@ -173,6 +175,7 @@ public void start(String userName, Boolean isUserImpersonate) { classpathStrings.add(0, ApplicationConstants.Environment.CLASSPATH.$$()); classpathStrings.add("./log4j.properties"); classpathStrings.add(System.getenv("HADOOP_CONF_DIR")); + classpathStrings.add("`hadoop classpath 2> /dev/null`"); String classpathEnv = Joiner.on(ApplicationConstants.CLASS_PATH_SEPARATOR).join(classpathStrings); diff --git a/zeppelin-distribution/src/assemble/distribution.xml b/zeppelin-distribution/src/assemble/distribution.xml index 5c369e256a3..e3f1677855d 100644 --- a/zeppelin-distribution/src/assemble/distribution.xml +++ b/zeppelin-distribution/src/assemble/distribution.xml @@ -44,11 +44,25 @@ ${project.groupId}:zeppelin-interpreter + + /lib/cluster/common + + ${project.groupId}:zeppelin-cluster-common + + + + /lib/cluster/yarn + + ${project.groupId}:zeppelin-cluster-yarn + + /lib ${project.groupId}:zeppelin-interpreter ${project.groupId}:zeppelin-web + ${project.groupId}:zeppelin-cluster-common + ${project.groupId}:zeppelin-cluster-yarn false true @@ -95,6 +109,14 @@ /lib/interpreter ../zeppelin-interpreter/target/lib + + /lib/cluster/common + ../zeppelin-cluster/common/target/lib + + + /lib/cluster/yarn + ../zeppelin-cluster/yarn/target/lib + /lib/node_modules/zeppelin-vis ../zeppelin-web/src/app/visualization diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index efbd201912c..9ad9a3a3a58 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -151,8 +151,7 @@ public ZeppelinServer() throws Exception { new InterpreterOption(true)); try { this.clusterManager = (ClusterManager) Class - .forName("org.apache.zeppelin.cluster.yarn.Client") - .getConstructor(ZeppelinConfiguration.class).newInstance(conf); + .forName("org.apache.zeppelin.cluster.yarn.Client").getConstructor().newInstance(); } catch (Throwable t) { // TODO(jl): To be fixed this.clusterManager = null; From ede01631fae4b4bc954f16227d9cddfd8371d6c0 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 4 May 2017 01:10:52 +0900 Subject: [PATCH 33/52] POC in my local --- .../yarn/RemoteInterpreterYarnProcess.java | 0 .../yarn/YarnRemoteInterpreterServer.java | 133 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java new file mode 100644 index 00000000000..e69de29bb2d diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java new file mode 100644 index 00000000000..8d4c0672ce0 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.cluster.yarn; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; + +import static org.apache.hadoop.yarn.api.records.FinalApplicationStatus.SUCCEEDED; + +/** + * + */ +public class YarnRemoteInterpreterServer extends RemoteInterpreterServer { + private static final Logger logger = LoggerFactory.getLogger(YarnRemoteInterpreterServer.class); + + private Configuration configuration; + private AMRMClientAsync amrmClientAsync; + + public YarnRemoteInterpreterServer(int port) throws TTransportException { + super(port); + configuration = new Configuration(); + } + + public static void main(String[] args) throws Exception { + int port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + YarnRemoteInterpreterServer yarnRemoteInterpreterServer = new YarnRemoteInterpreterServer(port); + yarnRemoteInterpreterServer.start(); + + yarnRemoteInterpreterServer.join(); + System.exit(0); + } + + @Override + public void run() { + logger.info("Start AMRMClient"); + amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(1000, new AMRMCallbackHandler()); + amrmClientAsync.init(configuration); + amrmClientAsync.start(); + + try { + String host = InetAddress.getLocalHost().getHostName(); + logger.info("Host: {}, port: {}", host, getPort()); + amrmClientAsync + .registerApplicationMaster(host, getPort(), "http://localhost:8080"); + } catch (YarnException | IOException e) { + logger.error("Error on registering application. Will shutdown"); + try { + shutdown(); + } catch (TException e1) { + logger.error("Never reach at", e1); + } + } + + super.run(); + } + + @Override + public void shutdown() throws TException { + try { + amrmClientAsync.unregisterApplicationMaster(SUCCEEDED, "Shutdown successfully", null); + } catch (IOException | YarnException e) { + logger.error("Error while unregistering application", e); + } + super.shutdown(); + + } + + private class AMRMCallbackHandler implements AMRMClientAsync.CallbackHandler { + @Override + public void onContainersCompleted(List list) { + logger.error("Never called"); + } + + @Override + public void onContainersAllocated(List list) { + logger.error("Never called"); + } + + @Override + public void onShutdownRequest() { + try { + shutdown(); + } catch (TException e) { + logger.error("Error while shutting down", e); + } + } + + @Override + public void onNodesUpdated(List list) { + logger.debug("onNoodesUpdated. {}", list); + } + + @Override + public float getProgress() { + return 0.5f; + } + + @Override + public void onError(Throwable throwable) { + logger.error("Error on Application", throwable); + } + } +} From e9b37ffd0432420439d436900ba8989812cea64b Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Tue, 27 Jun 2017 09:59:14 +0900 Subject: [PATCH 34/52] Refactored ClusterManager structure Introduced ClusterManagerFactory --- .../cluster/ClusterManagerFactory.java | 99 +++++++++++++ .../yarn/YarnRemoteInterpreterServer.java | 4 +- .../yarn/RemoteInterpreterYarnProcess.java | 0 .../yarn/YarnRemoteInterpreterServer.java | 133 ------------------ .../zeppelin/conf/ZeppelinConfiguration.java | 4 + 5 files changed, 105 insertions(+), 135 deletions(-) create mode 100644 zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java delete mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java delete mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java new file mode 100644 index 00000000000..6162175a3c8 --- /dev/null +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java @@ -0,0 +1,99 @@ +package org.apache.zeppelin.cluster; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class ClusterManagerFactory { + + private static final Logger logger = LoggerFactory.getLogger(ClusterManagerFactory.class); + private static final String CLUSTER_COMMON_DIR_NAME = "common"; + + private final String zeppelinHome; + private final String defaultClusterKey; + private final Map clusterManagerMap; + + private boolean initialized; + + public ClusterManagerFactory(String zeppelinHome, String defaultClusterKey) { + this.zeppelinHome = zeppelinHome; + this.defaultClusterKey = defaultClusterKey; + this.clusterManagerMap = Maps.newHashMap(); + this.initialized = false; + } + + public ClusterManager getClusterManager() { + return clusterManagerMap.get(defaultClusterKey); + } + + public ClusterManager getClusterManager(String key) { + if (!initialized) { + init(); + } + if (!clusterManagerMap.containsKey(key)) { + logger.info("Not supported. {}", key); + return null; + } + + return clusterManagerMap.get(key); + } + + private void init() { + // Find clusters inside source tree + logger.info("ZeppelinHome: {}", Paths.get(zeppelinHome).toAbsolutePath().toString()); + try { + for (Path p : Files + .newDirectoryStream(Paths.get(zeppelinHome, "zeppelin-cluster"), new Filter() { + @Override + public boolean accept(Path entry) throws IOException { + return !entry.toString().endsWith(CLUSTER_COMMON_DIR_NAME); + } + })) { + String key = p.getFileName().toString(); + List jars = Lists.newArrayList(); + for (Path childPath : Files + .newDirectoryStream(Paths.get(p.toString(), "target"), new Filter() { + @Override + public boolean accept(Path entry) throws IOException { + return entry.toString().endsWith(".jar"); + } + })) { + jars.add(childPath.toUri().toURL()); + } + + ClassLoader classLoader = new URLClassLoader(jars.toArray(new URL[0]), + ClassLoader.getSystemClassLoader()); + try { + Class clazz = classLoader.loadClass(ClusterManager.class.getName()); + Object cm = clazz.getConstructor().newInstance(); + logger.info("Class loaded. {}", clazz.getName()); + if (!clusterManagerMap.containsKey(key)) { + clusterManagerMap.put(key, (ClusterManager) cm); + } + } catch (ClassNotFoundException | IllegalAccessException | InstantiationException + | NoSuchMethodException | InvocationTargetException e) { + // Do nothing + } + + } + } catch (IOException e) { + logger.info("erroor while reading directory", e); + } + + initialized = true; + } +} diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java index 8d4c0672ce0..ce5aea38177 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java @@ -46,8 +46,8 @@ public class YarnRemoteInterpreterServer extends RemoteInterpreterServer { private Configuration configuration; private AMRMClientAsync amrmClientAsync; - public YarnRemoteInterpreterServer(int port) throws TTransportException { - super(port); + public YarnRemoteInterpreterServer(int port) throws TTransportException, IOException { + super(null, port); configuration = new Configuration(); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java deleted file mode 100644 index 8d4c0672ce0..00000000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.cluster.yarn; - -import java.io.IOException; -import java.net.InetAddress; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.thrift.TException; -import org.apache.thrift.transport.TTransportException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; - -import static org.apache.hadoop.yarn.api.records.FinalApplicationStatus.SUCCEEDED; - -/** - * - */ -public class YarnRemoteInterpreterServer extends RemoteInterpreterServer { - private static final Logger logger = LoggerFactory.getLogger(YarnRemoteInterpreterServer.class); - - private Configuration configuration; - private AMRMClientAsync amrmClientAsync; - - public YarnRemoteInterpreterServer(int port) throws TTransportException { - super(port); - configuration = new Configuration(); - } - - public static void main(String[] args) throws Exception { - int port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); - YarnRemoteInterpreterServer yarnRemoteInterpreterServer = new YarnRemoteInterpreterServer(port); - yarnRemoteInterpreterServer.start(); - - yarnRemoteInterpreterServer.join(); - System.exit(0); - } - - @Override - public void run() { - logger.info("Start AMRMClient"); - amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(1000, new AMRMCallbackHandler()); - amrmClientAsync.init(configuration); - amrmClientAsync.start(); - - try { - String host = InetAddress.getLocalHost().getHostName(); - logger.info("Host: {}, port: {}", host, getPort()); - amrmClientAsync - .registerApplicationMaster(host, getPort(), "http://localhost:8080"); - } catch (YarnException | IOException e) { - logger.error("Error on registering application. Will shutdown"); - try { - shutdown(); - } catch (TException e1) { - logger.error("Never reach at", e1); - } - } - - super.run(); - } - - @Override - public void shutdown() throws TException { - try { - amrmClientAsync.unregisterApplicationMaster(SUCCEEDED, "Shutdown successfully", null); - } catch (IOException | YarnException e) { - logger.error("Error while unregistering application", e); - } - super.shutdown(); - - } - - private class AMRMCallbackHandler implements AMRMClientAsync.CallbackHandler { - @Override - public void onContainersCompleted(List list) { - logger.error("Never called"); - } - - @Override - public void onContainersAllocated(List list) { - logger.error("Never called"); - } - - @Override - public void onShutdownRequest() { - try { - shutdown(); - } catch (TException e) { - logger.error("Error while shutting down", e); - } - } - - @Override - public void onNodesUpdated(List list) { - logger.debug("onNoodesUpdated. {}", list); - } - - @Override - public float getProgress() { - return 0.5f; - } - - @Override - public void onError(Throwable throwable) { - logger.error("Error on Application", throwable); - } - } -} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index e3df78a46da..793e244d257 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -514,6 +514,10 @@ public String getJettyName() { return getString(ConfVars.ZEPPELIN_SERVER_JETTY_NAME); } + public String getZeppelinDefaultCluster() { + return getString(ConfVars.ZEPPELIN_CLUSTER_DEFAULT); + } + public Map dumpConfigurations(ZeppelinConfiguration conf, ConfigurationKeyPredicate predicate) { Map configurations = new HashMap<>(); From 1d43a52384d5b630362889917baa142cc69c72cb Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Fri, 30 Jun 2017 17:48:30 +0900 Subject: [PATCH 35/52] Refactored --- .gitignore | 6 + bin/common.sh | 2 +- bin/interpreter.sh | 5 + bin/zeppelin-daemon.sh | 3 + pom.xml | 2 +- zeppelin-cluster/common/pom.xml | 5 + .../cluster/ClusterManagerFactory.java | 124 ++++++++++++++---- .../cluster/ClusterManagerFactoryTest.java | 35 +++++ .../src/test/resources/log4j.properties | 25 ++++ zeppelin-cluster/yarn/pom.xml | 29 +++- .../yarn/RemoteInterpreterYarnProcess.java | 12 +- .../src/main/resources/clustermanager-class | 1 + .../zeppelin/server/ZeppelinServer.java | 13 +- zeppelin-zengine/pom.xml | 6 - .../interpreter/InterpreterFactory.java | 9 +- .../interpreter/remote/RemoteInterpreter.java | 43 +++--- 16 files changed, 238 insertions(+), 82 deletions(-) create mode 100644 zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java create mode 100644 zeppelin-cluster/common/src/test/resources/log4j.properties create mode 100644 zeppelin-cluster/yarn/src/main/resources/clustermanager-class diff --git a/.gitignore b/.gitignore index 773edc80676..5f2f482b0f4 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,9 @@ tramp # Git properties **/git.properties + +# Yarn related +**/core-site.xml +**/hdfs-site.xml +**/yarn-site.xml +**/capacity-scheduler.xml diff --git a/bin/common.sh b/bin/common.sh index 1a3bd9befcb..293cda5317c 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -54,7 +54,7 @@ fi ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}" -if [[ -z "${HADOOP_CONF_DIR}" ]]; then +if [[ -n "${HADOOP_CONF_DIR}" ]]; then ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" fi diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 33f2738def0..b78d94244fa 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -85,6 +85,11 @@ fi addJarInDirForIntp "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" addJarInDirForIntp "${ZEPPELIN_HOME}/lib/interpreter" addJarInDirForIntp "${INTERPRETER_DIR}" +addJarInDirForIntp "${ZEPPELIN_HOME}/zeppelin-cluster/common/target/lib" +addJarInDirForIntp "${ZEPPELIN_HOME}/zeppelin-cluster/yarn/target/lib" +addJarInDirForIntp "${ZEPPELIN_HOME}/lib/cluster/common" +addJarInDirForIntp "${ZEPPELIN_HOME}/lib/cluster/yarn" + HOSTNAME=$(hostname) ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer diff --git a/bin/zeppelin-daemon.sh b/bin/zeppelin-daemon.sh index e88c26fc43c..33f3a090b52 100755 --- a/bin/zeppelin-daemon.sh +++ b/bin/zeppelin-daemon.sh @@ -73,10 +73,13 @@ fi addJarInDir "${ZEPPELIN_HOME}" addJarInDir "${ZEPPELIN_HOME}/lib" addJarInDir "${ZEPPELIN_HOME}/lib/interpreter" +addJarInDir "${ZEPPELIN_HOME}/lib/cluster/common" addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib" addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-cluster/common/target" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-cluster/common/target/lib" CLASSPATH+=":${ZEPPELIN_CLASSPATH}" diff --git a/pom.xml b/pom.xml index 7f1a480ec6c..d42ab9c294a 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ 0.2.1 15.0 9.2.15.v20160210 - 2.7.3 + 2.6.0 4.4.1 4.5.1 4.0.2 diff --git a/zeppelin-cluster/common/pom.xml b/zeppelin-cluster/common/pom.xml index 3769db32938..31739cd2221 100644 --- a/zeppelin-cluster/common/pom.xml +++ b/zeppelin-cluster/common/pom.xml @@ -41,6 +41,11 @@ ${project.version} provided + + + junit + junit + diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java index 6162175a3c8..1211005b41c 100644 --- a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java @@ -1,8 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.cluster; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; import java.lang.reflect.InvocationTargetException; import java.net.URL; import java.net.URLClassLoader; @@ -12,6 +31,7 @@ import java.nio.file.Paths; import java.util.List; import java.util.Map; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +42,7 @@ public class ClusterManagerFactory { private static final Logger logger = LoggerFactory.getLogger(ClusterManagerFactory.class); private static final String CLUSTER_COMMON_DIR_NAME = "common"; + private static final String CLUSTER_CLASS_NAME_FILE = "clustermanager-class"; private final String zeppelinHome; private final String defaultClusterKey; @@ -37,13 +58,18 @@ public ClusterManagerFactory(String zeppelinHome, String defaultClusterKey) { } public ClusterManager getClusterManager() { - return clusterManagerMap.get(defaultClusterKey); + return getClusterManager(defaultClusterKey); } public ClusterManager getClusterManager(String key) { if (!initialized) { init(); } + + if (null == key) { + return getClusterManager(); + } + if (!clusterManagerMap.containsKey(key)) { logger.info("Not supported. {}", key); return null; @@ -52,48 +78,90 @@ public ClusterManager getClusterManager(String key) { return clusterManagerMap.get(key); } - private void init() { + synchronized void init() { + if (initialized) { + return; + } // Find clusters inside source tree logger.info("ZeppelinHome: {}", Paths.get(zeppelinHome).toAbsolutePath().toString()); try { for (Path p : Files - .newDirectoryStream(Paths.get(zeppelinHome, "zeppelin-cluster"), new Filter() { - @Override - public boolean accept(Path entry) throws IOException { - return !entry.toString().endsWith(CLUSTER_COMMON_DIR_NAME); - } - })) { + .newDirectoryStream(Paths.get(zeppelinHome, "zeppelin-cluster"), new Filter() { + @Override + public boolean accept(Path entry) throws IOException { + return !entry.toString().endsWith(CLUSTER_COMMON_DIR_NAME); + } + })) { String key = p.getFileName().toString(); List jars = Lists.newArrayList(); - for (Path childPath : Files - .newDirectoryStream(Paths.get(p.toString(), "target"), new Filter() { + addJarsFromPath(Paths.get(p.toString(), "target"), jars); + addJarsFromPath(Paths.get(p.toString(), "target", "lib"), jars); + + registerClusterManager(key, jars); + } + + for (Path p : Files + .newDirectoryStream(Paths.get(zeppelinHome, "lib", "cluster"), new Filter() { @Override public boolean accept(Path entry) throws IOException { - return entry.toString().endsWith(".jar"); + return !entry.toString().endsWith(CLUSTER_COMMON_DIR_NAME); } })) { - jars.add(childPath.toUri().toURL()); - } - - ClassLoader classLoader = new URLClassLoader(jars.toArray(new URL[0]), - ClassLoader.getSystemClassLoader()); - try { - Class clazz = classLoader.loadClass(ClusterManager.class.getName()); - Object cm = clazz.getConstructor().newInstance(); - logger.info("Class loaded. {}", clazz.getName()); - if (!clusterManagerMap.containsKey(key)) { - clusterManagerMap.put(key, (ClusterManager) cm); - } - } catch (ClassNotFoundException | IllegalAccessException | InstantiationException - | NoSuchMethodException | InvocationTargetException e) { - // Do nothing - } + String key = p.getFileName().toString(); + List jars = Lists.newArrayList(); + addJarsFromPath(p, jars); + registerClusterManager(key, jars); } } catch (IOException e) { - logger.info("erroor while reading directory", e); + logger.debug("error while reading directory", e); } initialized = true; } + + private void registerClusterManager(String key, List jars) { + ClassLoader parentClassLoader = ClassLoader.getSystemClassLoader(); + ClassLoader classLoader = new URLClassLoader(jars.toArray(new URL[0]), parentClassLoader); + String className = null; + + try (InputStream classNameInputstream = + classLoader.getResourceAsStream(CLUSTER_CLASS_NAME_FILE); + StringWriter classNameWriter = new StringWriter()) { + if (null != classNameInputstream) { + IOUtils.copy(classNameInputstream, classNameWriter); + className = classNameWriter.toString(); + Class clazz = classLoader.loadClass(className); + Object cm = clazz.getConstructor().newInstance(); + logger.info("Class loaded. {}", clazz.getName()); + if (!clusterManagerMap.containsKey(key)) { + clusterManagerMap.put(key, (ClusterManager) cm); + } + } else { + logger.debug("No resources {} in {}", CLUSTER_COMMON_DIR_NAME, jars); + } + } catch (ClassNotFoundException | IllegalAccessException | InstantiationException + | NoSuchMethodException | InvocationTargetException | IOException e) { + if (null != className) { + logger.error("class name is defined {}, but witch doesn't exist", className); + } + } catch (NoClassDefFoundError e) { + logger.info("{} exists but insufficient dependencies. jars: {}", className, jars); + } + } + + public void addJarsFromPath(Path dir, List jars) { + try { + for (Path p : Files.newDirectoryStream(dir, new Filter() { + @Override + public boolean accept(Path entry) throws IOException { + return entry.toString().endsWith(".jar"); + } + })) { + jars.add(p.toUri().toURL()); + } + } catch (IOException e) { + logger.error("Error occurs while reading {}", dir.toAbsolutePath().toString()); + } + } } diff --git a/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java b/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java new file mode 100644 index 00000000000..30d8abdb199 --- /dev/null +++ b/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.cluster; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * + */ +public class ClusterManagerFactoryTest { + @Test + public void initTest() { + System.out.println(System.getenv("CLASSPATH")); + ClusterManagerFactory cmf = new ClusterManagerFactory("../../", "local"); + // TODO(jl): Fix order of building module for testing it + //assertTrue("One of cluster manager will exists", null != cmf.getClusterManager("yarn") || null != cmf.getClusterManager("mesos")); + } +} diff --git a/zeppelin-cluster/common/src/test/resources/log4j.properties b/zeppelin-cluster/common/src/test/resources/log4j.properties new file mode 100644 index 00000000000..2d8eb8656ba --- /dev/null +++ b/zeppelin-cluster/common/src/test/resources/log4j.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n \ No newline at end of file diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml index 4bc5c1fc319..dce39e523e8 100644 --- a/zeppelin-cluster/yarn/pom.xml +++ b/zeppelin-cluster/yarn/pom.xml @@ -52,13 +52,11 @@ org.apache.hadoop hadoop-yarn-client - provided org.apache.hadoop hadoop-yarn-common - provided org.codehaus.jackson @@ -102,13 +100,11 @@ org.apache.hadoop hadoop-yarn-api - provided org.apache.hadoop hadoop-common - provided javax.servlet @@ -156,7 +152,6 @@ org.apache.hadoop hadoop-hdfs - provided javax.servlet @@ -210,4 +205,28 @@ + + + + ide + + + + src/main/resources + + + + + + + + + + src/main/resources + + clustermanager-class + + + + diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index 583b10f7428..8eb8841ff5c 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -193,13 +193,13 @@ public void start(String userName, Boolean isUserImpersonate) { FileSystem fileSystem = FileSystem.get(configuration); - Path interpreterDir = getInterpreterRelativePath(group); + Path interpreterDir = getInterpreterRelativePath(); List interpreterPaths = getPathsFromDirPath(interpreterDir); if (isSparkInterpreter()) { // For pyspark interpreterDir = isSparkHomeSet() ? Paths.get(this.env.get("SPARK_HOME"), "python", "lib") - : getInterpreterRelativePath(group, "pyspark"); + : getInterpreterRelativePath("pyspark"); List pythonLibPath = getPathsFromDirPath(interpreterDir); interpreterPaths.addAll(pythonLibPath); @@ -214,7 +214,7 @@ public void start(String userName, Boolean isUserImpersonate) { env.put("PYSPARK_ARCHIVES_PATH", Joiner.on(",").join(pythonLibPaths)); interpreterDir = isSparkHomeSet() ? Paths.get(this.env.get("SPARK_HOME"), "jars") - : getInterpreterRelativePath(group, "dep"); + : getInterpreterRelativePath("dep"); List jarPaths = getPathsFromDirPath(interpreterDir); interpreterPaths.addAll(jarPaths); @@ -238,7 +238,7 @@ public void start(String userName, Boolean isUserImpersonate) { } properties.setProperty("spark.yarn.archive", dstPath); } else if (isPythonInterpreter()) { - interpreterDir = getInterpreterRelativePath(group); + interpreterDir = getInterpreterRelativePath(); List pythonLibPath = getPathsFromDirPath(interpreterDir); interpreterPaths.addAll(pythonLibPath); @@ -255,7 +255,7 @@ public void start(String userName, Boolean isUserImpersonate) { // For pyspark and python List zeppelinPythonLibPaths = getPathsFromDirPath( - getInterpreterRelativePath("lib", "python")); + Paths.get(this.interpreterDir, "lib", "python")); interpreterPaths.addAll(zeppelinPythonLibPaths); List distributedZeppelinPythonLibPaths = Lists.newArrayList(); for (Path p : zeppelinPythonLibPaths) { @@ -406,7 +406,7 @@ private boolean isSparkHomeSet() { } private boolean isHadoopConfSet() { - return this.env.containsKey("HADOOP_CONF_DIR"); + return null != System.getenv("HADOOP_CONF_DIR"); } private int convertSparkMemoryFormat(String memoryFormat) { diff --git a/zeppelin-cluster/yarn/src/main/resources/clustermanager-class b/zeppelin-cluster/yarn/src/main/resources/clustermanager-class new file mode 100644 index 00000000000..871c35ee9be --- /dev/null +++ b/zeppelin-cluster/yarn/src/main/resources/clustermanager-class @@ -0,0 +1 @@ +org.apache.zeppelin.cluster.yarn.Client \ No newline at end of file diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 9ad9a3a3a58..7179b900571 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -28,6 +28,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.shiro.web.env.EnvironmentLoaderListener; import org.apache.shiro.web.servlet.ShiroFilter; +import org.apache.zeppelin.cluster.ClusterManagerFactory; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; @@ -88,7 +89,7 @@ public class ZeppelinServer extends Application { public static Helium helium; private final InterpreterSettingManager interpreterSettingManager; - private ClusterManager clusterManager; + private ClusterManagerFactory clusterManagerFactory; private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; private SearchService noteSearchService; @@ -149,16 +150,10 @@ public ZeppelinServer() throws Exception { this.schedulerFactory = new SchedulerFactory(); this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - try { - this.clusterManager = (ClusterManager) Class - .forName("org.apache.zeppelin.cluster.yarn.Client").getConstructor().newInstance(); - } catch (Throwable t) { - // TODO(jl): To be fixed - this.clusterManager = null; - } + this.clusterManagerFactory = new ClusterManagerFactory(conf.getHome(), "local"); this.replFactory = new InterpreterFactory(conf, notebookWsServer, notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(), - interpreterSettingManager, clusterManager); + interpreterSettingManager, clusterManagerFactory); this.notebookRepo = new NotebookRepoSync(conf); this.noteSearchService = new LuceneSearch(); this.notebookAuthorization = NotebookAuthorization.init(conf); diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 2eec26d3c58..d3879564d55 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -65,12 +65,6 @@ ${project.version} - - org.apache.zeppelin - zeppelin-cluster-yarn - ${project.version} - - org.slf4j slf4j-api diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 536d21245de..1a0415952e0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -47,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import org.apache.zeppelin.cluster.ClusterManagerFactory; /** * Manage interpreters. @@ -65,7 +66,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; private final ApplicationEventListener appEventListener; - private final ClusterManager clusterManager; + private final ClusterManagerFactory clusterManagerFactory; private boolean shiroEnabled; @@ -78,7 +79,7 @@ public InterpreterFactory(ZeppelinConfiguration conf, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appEventListener, DependencyResolver depResolver, boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager, - ClusterManager clusterManager) + ClusterManagerFactory clusterManagerFactory) throws InterpreterException, IOException, RepositoryException { this.conf = conf; this.angularObjectRegistryListener = angularObjectRegistryListener; @@ -90,7 +91,7 @@ public InterpreterFactory(ZeppelinConfiguration conf, //TODO(jl): Fix it not to use InterpreterGroupFactory interpreterSettingManager.setInterpreterGroupFactory(this); - this.clusterManager = clusterManager; + this.clusterManagerFactory = clusterManagerFactory; logger.info("shiroEnabled: {}", shiroEnabled); } @@ -287,7 +288,7 @@ Interpreter createRemoteRepl(String interpreterPath, String interpreterSessionKe interpreterRunnerPath, interpreterPath, localRepoPath, connectTimeout, maxPoolSize, remoteInterpreterProcessListener, appEventListener, userName, isUserImpersonate, conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName, - clusterManager, interpreterGroupStr); + clusterManagerFactory, interpreterGroupStr); remoteInterpreter.addEnv(env); return new LazyOpenInterpreter(remoteInterpreter); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index c672fee88c3..eb14a7977dd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -21,6 +21,7 @@ import org.apache.thrift.TException; import org.apache.zeppelin.cluster.ClusterManager; +import org.apache.zeppelin.cluster.ClusterManagerFactory; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; @@ -67,7 +68,7 @@ public class RemoteInterpreter extends Interpreter { private Boolean isUserImpersonate; private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT; private String interpreterGroupName; - private ClusterManager clusterManager; + private ClusterManagerFactory clusterManagerFactory; private String group; /** @@ -77,7 +78,8 @@ public RemoteInterpreter(Properties property, String sessionKey, String classNam String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout, int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appListener, String userName, Boolean isUserImpersonate, - int outputLimit, String interpreterGroupName, ClusterManager clusterManager, String group) { + int outputLimit, String interpreterGroupName, ClusterManagerFactory clusterManagerFactory, + String group) { super(property); this.sessionKey = sessionKey; this.className = className; @@ -95,7 +97,7 @@ public RemoteInterpreter(Properties property, String sessionKey, String classNam this.isUserImpersonate = isUserImpersonate; this.outputLimit = outputLimit; this.interpreterGroupName = interpreterGroupName; - this.clusterManager = clusterManager; + this.clusterManagerFactory = clusterManagerFactory; this.group = group; } @@ -184,25 +186,22 @@ public RemoteInterpreterProcess getInterpreterProcess() { port); } else { String clusterManagerKey = getProperty(Constants.ZEPPELIN_CLUSTER_MANAGER_KEY); - if (null == clusterManagerKey) { - //TODO(jl): Set defult cluster manager - clusterManagerKey = Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL; - } - switch (clusterManagerKey) { - case Constants.ZEPPELIN_CLUSTER_MANAGER_YARN: - remoteProcess = clusterManager - .createInterpreter(sessionKey, interpreterGroupName, group, env, property, - connectTimeout, remoteInterpreterProcessListener, applicationEventListener, - homeDir, interpreterPath); - break; - case Constants.ZEPPELIN_CLUSTER_MANAGER_LOCAL: - default: - // Default is local process - // create new remote process - remoteProcess = new RemoteInterpreterManagedProcess( - interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, - remoteInterpreterProcessListener, applicationEventListener, - interpreterGroupName); + ClusterManager clusterManager = + clusterManagerFactory.getClusterManager(clusterManagerKey); + + //TODO(jl): Fix the parameter list to unify all methods + if (null != clusterManager) { + remoteProcess = clusterManager + .createInterpreter(sessionKey, interpreterGroupName, group, env, property, + connectTimeout, remoteInterpreterProcessListener, applicationEventListener, + homeDir, interpreterPath); + } else { + // Default is local process + // create new remote process + remoteProcess = new RemoteInterpreterManagedProcess( + interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, + remoteInterpreterProcessListener, applicationEventListener, + interpreterGroupName); } } From 9be39a2922592141b862bfb01c37ed74ad01ac04 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Fri, 30 Jun 2017 17:58:26 +0900 Subject: [PATCH 36/52] Fixed rat issue --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index d42ab9c294a..9d1f4998435 100644 --- a/pom.xml +++ b/pom.xml @@ -1043,6 +1043,7 @@ **/src/fonts/Source-Code-Pro* **/src/fonts/source-code-pro* **/src/**/**.test.js + **/clustermanager-class **/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java From 76c1ebd82b57365c183b44538573fcb22b3b5315 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 2 Jul 2017 13:49:51 +0900 Subject: [PATCH 37/52] Removed unrelated changes --- bin/interpreter.sh | 13 +- .../zeppelin/helium/ZeppelinDevServer.java | 4 +- pom.xml | 34 +- .../interpreter/remote/ClientFactory.java | 4 - .../remote/RemoteInterpreterServer.java | 53 +- .../remote/RemoteInterpreterUtils.java | 31 - .../interpreter/thrift/CallbackInfo.java | 518 ----------- .../RemoteInterpreterCallbackService.java | 879 ------------------ .../thrift/RemoteInterpreterService.thrift | 9 - .../remote/RemoteInterpreterServerTest.java | 4 +- .../RemoteInterpreterManagedProcess.java | 94 +- .../remote/RemoteInterpreterProcessTest.java | 4 +- 12 files changed, 44 insertions(+), 1603 deletions(-) delete mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/CallbackInfo.java delete mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterCallbackService.java diff --git a/bin/interpreter.sh b/bin/interpreter.sh index b78d94244fa..11806a76975 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -23,7 +23,7 @@ function usage() { echo "usage) $0 -p -d -l -g " } -while getopts "hc:p:d:l:v:u:g:" o; do +while getopts "hp:d:l:v:u:g:" o; do case ${o} in h) usage @@ -32,11 +32,8 @@ while getopts "hc:p:d:l:v:u:g:" o; do d) INTERPRETER_DIR=${OPTARG} ;; - c) - CALLBACK_HOST=${OPTARG} # This will be used callback host - ;; p) - PORT=${OPTARG} # This will be used callback port + PORT=${OPTARG} ;; l) LOCAL_INTERPRETER_REPO=${OPTARG} @@ -210,12 +207,12 @@ fi if [[ -n "${SPARK_SUBMIT}" ]]; then if [[ -n "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" != "false" ]]; then - INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${PORT}` else - INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT}` fi else - INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ` + INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} ` fi if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ -n "${suid}" || -z "${SPARK_SUBMIT}" ]]; then diff --git a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java index 3a15e8be0dd..21ce283abba 100644 --- a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java +++ b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java @@ -38,8 +38,8 @@ public class ZeppelinDevServer extends private DevInterpreter interpreter = null; private InterpreterOutput out; - public ZeppelinDevServer(int port) throws TException, IOException { - super(null, port); + public ZeppelinDevServer(int port) throws TException { + super(port); } @Override diff --git a/pom.xml b/pom.xml index 9d1f4998435..ea08421e12c 100644 --- a/pom.xml +++ b/pom.xml @@ -80,7 +80,6 @@ scio zeppelin-web zeppelin-server - zeppelin-cluster/yarn zeppelin-jupyter zeppelin-distribution @@ -268,38 +267,6 @@ ${shiro.version} - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - - - - org.apache.hadoop - hadoop-yarn-api - ${hadoop.version} - - - - org.apache.hadoop - hadoop-yarn-common - ${hadoop.version} - - - - org.apache.hadoop - hadoop-yarn-client - ${hadoop.version} - - - junit @@ -356,6 +323,7 @@ ${powermock.version} test + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java index c0fc6d79a49..b2cb78f810e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java @@ -30,14 +30,11 @@ import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * */ public class ClientFactory extends BasePooledObjectFactory{ - private final Logger logger = LoggerFactory.getLogger(ClientFactory.class); private String host; private int port; Map clientSocketMap = new HashMap<>(); @@ -49,7 +46,6 @@ public ClientFactory(String host, int port) { @Override public Client create() throws Exception { - logger.info("Create. host: {}, port: {}", host, port); TSocket transport = new TSocket(host, port); try { transport.open(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 51fa6b68381..b1686446fcb 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -72,7 +72,6 @@ import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.RemoteWorksController; import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource; -import org.apache.zeppelin.interpreter.thrift.CallbackInfo; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.interpreter.thrift.RemoteApplicationResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; @@ -111,9 +110,6 @@ public class RemoteInterpreterServer Gson gson = new Gson(); RemoteInterpreterService.Processor processor; - private String callbackHost; - private int callbackPort; - private String host; private int port; protected TThreadPoolServer server; @@ -135,15 +131,7 @@ public RemoteInterpreterServer(int port) throws TTransportException { this.port = port; processor = new RemoteInterpreterService.Processor<>(this); - TServerSocket serverTransport; - if (null == callbackHost) { - // Dev Interpreter - serverTransport = new TServerSocket(port); - } else { - this.port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); - this.host = RemoteInterpreterUtils.findAvailableHostname(); - serverTransport = new TServerSocket(this.port); - } + TServerSocket serverTransport = new TServerSocket(port); server = new TThreadPoolServer( new TThreadPoolServer.Args(serverTransport).processor(processor)); remoteWorksResponsePool = Collections.synchronizedMap(new HashMap()); @@ -152,36 +140,6 @@ public RemoteInterpreterServer(int port) throws TTransportException { @Override public void run() { - if (null != callbackHost) { - new Thread(new Runnable() { - boolean interrupted = false; - @Override - public void run() { - while (!interrupted && !server.isServing()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - interrupted = true; - } - } - - if (!interrupted) { - CallbackInfo callbackInfo = new CallbackInfo(host, port); - try { - RemoteInterpreterUtils - .registerInterpreter(callbackHost, callbackPort, callbackInfo); - } catch (TException e) { - logger.error("Error while registering interpreter: {}", callbackInfo, e); - try { - shutdown(); - } catch (TException e1) { - logger.warn("Exception occurs while shutting down", e1); - } - } - } - } - }).start(); - } logger.info("Starting remote interpreter server on port {}", port); server.serve(); } @@ -228,15 +186,12 @@ public boolean isRunning() { public static void main(String[] args) - throws TTransportException, InterruptedException, Exception { - String callbackHost = null; + throws TTransportException, InterruptedException { int port = Constants.ZEPPELIN_INTERPRETER_DEFAUlT_PORT; if (args.length > 0) { - callbackHost = args[0]; - port = Integer.parseInt(args[1]); + port = Integer.parseInt(args[0]); } - RemoteInterpreterServer remoteInterpreterServer = - new RemoteInterpreterServer(callbackHost, port); + RemoteInterpreterServer remoteInterpreterServer = new RemoteInterpreterServer(port); remoteInterpreterServer.start(); remoteInterpreterServer.join(); System.exit(0); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java index d5e0cf14667..32354af3421 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java @@ -54,24 +54,6 @@ public static int findRandomAvailablePortOnAllLocalInterfaces() throws IOExcepti return port; } - public static String findAvailableHostname() throws UnknownHostException, SocketException { - InetAddress address = InetAddress.getLocalHost(); - if (address.isLoopbackAddress()) { - for (NetworkInterface networkInterface : Collections - .list(NetworkInterface.getNetworkInterfaces())) { - if (!networkInterface.isLoopback()) { - for (InterfaceAddress interfaceAddress : networkInterface.getInterfaceAddresses()) { - InetAddress a = interfaceAddress.getAddress(); - if (a instanceof Inet4Address) { - return a.getHostAddress(); - } - } - } - } - } - return address.getHostName(); - } - public static boolean checkIfRemoteEndpointAccessible(String host, int port) { try { Socket discover = new Socket(); @@ -112,17 +94,4 @@ public static boolean isEnvString(String key) { return key.matches("^[A-Z_0-9]*"); } - - public static void registerInterpreter(String callbackHost, int callbackPort, - final CallbackInfo callbackInfo) throws TException { - LOGGER.info("callbackHost: {}, callbackPort: {}, callbackInfo: {}", callbackHost, callbackPort, - callbackInfo); - try (TTransport transport = new TSocket(callbackHost, callbackPort)) { - transport.open(); - TProtocol protocol = new TBinaryProtocol(transport); - RemoteInterpreterCallbackService.Client client = new RemoteInterpreterCallbackService.Client( - protocol); - client.callback(callbackInfo); - } - } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/CallbackInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/CallbackInfo.java deleted file mode 100644 index b0c7e9acb57..00000000000 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/CallbackInfo.java +++ /dev/null @@ -1,518 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Autogenerated by Thrift Compiler (0.9.2) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -package org.apache.zeppelin.interpreter.thrift; - -import org.apache.thrift.scheme.IScheme; -import org.apache.thrift.scheme.SchemeFactory; -import org.apache.thrift.scheme.StandardScheme; - -import org.apache.thrift.scheme.TupleScheme; -import org.apache.thrift.protocol.TTupleProtocol; -import org.apache.thrift.protocol.TProtocolException; -import org.apache.thrift.EncodingUtils; -import org.apache.thrift.TException; -import org.apache.thrift.async.AsyncMethodCallback; -import org.apache.thrift.server.AbstractNonblockingServer.*; -import java.util.List; -import java.util.ArrayList; -import java.util.Map; -import java.util.HashMap; -import java.util.EnumMap; -import java.util.Set; -import java.util.HashSet; -import java.util.EnumSet; -import java.util.Collections; -import java.util.BitSet; -import java.nio.ByteBuffer; -import java.util.Arrays; -import javax.annotation.Generated; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-6-17") -public class CallbackInfo implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CallbackInfo"); - - private static final org.apache.thrift.protocol.TField HOST_FIELD_DESC = new org.apache.thrift.protocol.TField("host", org.apache.thrift.protocol.TType.STRING, (short)1); - private static final org.apache.thrift.protocol.TField PORT_FIELD_DESC = new org.apache.thrift.protocol.TField("port", org.apache.thrift.protocol.TType.I32, (short)2); - - private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); - static { - schemes.put(StandardScheme.class, new CallbackInfoStandardSchemeFactory()); - schemes.put(TupleScheme.class, new CallbackInfoTupleSchemeFactory()); - } - - public String host; // required - public int port; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - HOST((short)1, "host"), - PORT((short)2, "port"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // HOST - return HOST; - case 2: // PORT - return PORT; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); - } - - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - public short getThriftFieldId() { - return _thriftId; - } - - public String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - private static final int __PORT_ISSET_ID = 0; - private byte __isset_bitfield = 0; - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.HOST, new org.apache.thrift.meta_data.FieldMetaData("host", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - tmpMap.put(_Fields.PORT, new org.apache.thrift.meta_data.FieldMetaData("port", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); - metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CallbackInfo.class, metaDataMap); - } - - public CallbackInfo() { - } - - public CallbackInfo( - String host, - int port) - { - this(); - this.host = host; - this.port = port; - setPortIsSet(true); - } - - /** - * Performs a deep copy on other. - */ - public CallbackInfo(CallbackInfo other) { - __isset_bitfield = other.__isset_bitfield; - if (other.isSetHost()) { - this.host = other.host; - } - this.port = other.port; - } - - public CallbackInfo deepCopy() { - return new CallbackInfo(this); - } - - @Override - public void clear() { - this.host = null; - setPortIsSet(false); - this.port = 0; - } - - public String getHost() { - return this.host; - } - - public CallbackInfo setHost(String host) { - this.host = host; - return this; - } - - public void unsetHost() { - this.host = null; - } - - /** Returns true if field host is set (has been assigned a value) and false otherwise */ - public boolean isSetHost() { - return this.host != null; - } - - public void setHostIsSet(boolean value) { - if (!value) { - this.host = null; - } - } - - public int getPort() { - return this.port; - } - - public CallbackInfo setPort(int port) { - this.port = port; - setPortIsSet(true); - return this; - } - - public void unsetPort() { - __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __PORT_ISSET_ID); - } - - /** Returns true if field port is set (has been assigned a value) and false otherwise */ - public boolean isSetPort() { - return EncodingUtils.testBit(__isset_bitfield, __PORT_ISSET_ID); - } - - public void setPortIsSet(boolean value) { - __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __PORT_ISSET_ID, value); - } - - public void setFieldValue(_Fields field, Object value) { - switch (field) { - case HOST: - if (value == null) { - unsetHost(); - } else { - setHost((String)value); - } - break; - - case PORT: - if (value == null) { - unsetPort(); - } else { - setPort((Integer)value); - } - break; - - } - } - - public Object getFieldValue(_Fields field) { - switch (field) { - case HOST: - return getHost(); - - case PORT: - return Integer.valueOf(getPort()); - - } - throw new IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); - } - - switch (field) { - case HOST: - return isSetHost(); - case PORT: - return isSetPort(); - } - throw new IllegalStateException(); - } - - @Override - public boolean equals(Object that) { - if (that == null) - return false; - if (that instanceof CallbackInfo) - return this.equals((CallbackInfo)that); - return false; - } - - public boolean equals(CallbackInfo that) { - if (that == null) - return false; - - boolean this_present_host = true && this.isSetHost(); - boolean that_present_host = true && that.isSetHost(); - if (this_present_host || that_present_host) { - if (!(this_present_host && that_present_host)) - return false; - if (!this.host.equals(that.host)) - return false; - } - - boolean this_present_port = true; - boolean that_present_port = true; - if (this_present_port || that_present_port) { - if (!(this_present_port && that_present_port)) - return false; - if (this.port != that.port) - return false; - } - - return true; - } - - @Override - public int hashCode() { - List list = new ArrayList(); - - boolean present_host = true && (isSetHost()); - list.add(present_host); - if (present_host) - list.add(host); - - boolean present_port = true; - list.add(present_port); - if (present_port) - list.add(port); - - return list.hashCode(); - } - - @Override - public int compareTo(CallbackInfo other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = Boolean.valueOf(isSetHost()).compareTo(other.isSetHost()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetHost()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.host, other.host); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = Boolean.valueOf(isSetPort()).compareTo(other.isSetPort()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetPort()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.port, other.port); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - schemes.get(iprot.getScheme()).getScheme().read(iprot, this); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - schemes.get(oprot.getScheme()).getScheme().write(oprot, this); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("CallbackInfo("); - boolean first = true; - - sb.append("host:"); - if (this.host == null) { - sb.append("null"); - } else { - sb.append(this.host); - } - first = false; - if (!first) sb.append(", "); - sb.append("port:"); - sb.append(this.port); - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { - try { - // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. - __isset_bitfield = 0; - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class CallbackInfoStandardSchemeFactory implements SchemeFactory { - public CallbackInfoStandardScheme getScheme() { - return new CallbackInfoStandardScheme(); - } - } - - private static class CallbackInfoStandardScheme extends StandardScheme { - - public void read(org.apache.thrift.protocol.TProtocol iprot, CallbackInfo struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // HOST - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.host = iprot.readString(); - struct.setHostIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // PORT - if (schemeField.type == org.apache.thrift.protocol.TType.I32) { - struct.port = iprot.readI32(); - struct.setPortIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot, CallbackInfo struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.host != null) { - oprot.writeFieldBegin(HOST_FIELD_DESC); - oprot.writeString(struct.host); - oprot.writeFieldEnd(); - } - oprot.writeFieldBegin(PORT_FIELD_DESC); - oprot.writeI32(struct.port); - oprot.writeFieldEnd(); - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class CallbackInfoTupleSchemeFactory implements SchemeFactory { - public CallbackInfoTupleScheme getScheme() { - return new CallbackInfoTupleScheme(); - } - } - - private static class CallbackInfoTupleScheme extends TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, CallbackInfo struct) throws org.apache.thrift.TException { - TTupleProtocol oprot = (TTupleProtocol) prot; - BitSet optionals = new BitSet(); - if (struct.isSetHost()) { - optionals.set(0); - } - if (struct.isSetPort()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); - if (struct.isSetHost()) { - oprot.writeString(struct.host); - } - if (struct.isSetPort()) { - oprot.writeI32(struct.port); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, CallbackInfo struct) throws org.apache.thrift.TException { - TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(2); - if (incoming.get(0)) { - struct.host = iprot.readString(); - struct.setHostIsSet(true); - } - if (incoming.get(1)) { - struct.port = iprot.readI32(); - struct.setPortIsSet(true); - } - } - } - -} - diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterCallbackService.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterCallbackService.java deleted file mode 100644 index 6ef08f6b0e8..00000000000 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterCallbackService.java +++ /dev/null @@ -1,879 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Autogenerated by Thrift Compiler (0.9.2) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -package org.apache.zeppelin.interpreter.thrift; - -import org.apache.thrift.scheme.IScheme; -import org.apache.thrift.scheme.SchemeFactory; -import org.apache.thrift.scheme.StandardScheme; - -import org.apache.thrift.scheme.TupleScheme; -import org.apache.thrift.protocol.TTupleProtocol; -import org.apache.thrift.protocol.TProtocolException; -import org.apache.thrift.EncodingUtils; -import org.apache.thrift.TException; -import org.apache.thrift.async.AsyncMethodCallback; -import org.apache.thrift.server.AbstractNonblockingServer.*; -import java.util.List; -import java.util.ArrayList; -import java.util.Map; -import java.util.HashMap; -import java.util.EnumMap; -import java.util.Set; -import java.util.HashSet; -import java.util.EnumSet; -import java.util.Collections; -import java.util.BitSet; -import java.nio.ByteBuffer; -import java.util.Arrays; -import javax.annotation.Generated; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-6-17") -public class RemoteInterpreterCallbackService { - - public interface Iface { - - public void callback(CallbackInfo callbackInfo) throws org.apache.thrift.TException; - - } - - public interface AsyncIface { - - public void callback(CallbackInfo callbackInfo, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - - } - - public static class Client extends org.apache.thrift.TServiceClient implements Iface { - public static class Factory implements org.apache.thrift.TServiceClientFactory { - public Factory() {} - public Client getClient(org.apache.thrift.protocol.TProtocol prot) { - return new Client(prot); - } - public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { - return new Client(iprot, oprot); - } - } - - public Client(org.apache.thrift.protocol.TProtocol prot) - { - super(prot, prot); - } - - public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) { - super(iprot, oprot); - } - - public void callback(CallbackInfo callbackInfo) throws org.apache.thrift.TException - { - send_callback(callbackInfo); - recv_callback(); - } - - public void send_callback(CallbackInfo callbackInfo) throws org.apache.thrift.TException - { - callback_args args = new callback_args(); - args.setCallbackInfo(callbackInfo); - sendBase("callback", args); - } - - public void recv_callback() throws org.apache.thrift.TException - { - callback_result result = new callback_result(); - receiveBase(result, "callback"); - return; - } - - } - public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { - public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { - private org.apache.thrift.async.TAsyncClientManager clientManager; - private org.apache.thrift.protocol.TProtocolFactory protocolFactory; - public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) { - this.clientManager = clientManager; - this.protocolFactory = protocolFactory; - } - public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) { - return new AsyncClient(protocolFactory, clientManager, transport); - } - } - - public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) { - super(protocolFactory, clientManager, transport); - } - - public void callback(CallbackInfo callbackInfo, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - checkReady(); - callback_call method_call = new callback_call(callbackInfo, resultHandler, this, ___protocolFactory, ___transport); - this.___currentMethod = method_call; - ___manager.call(method_call); - } - - public static class callback_call extends org.apache.thrift.async.TAsyncMethodCall { - private CallbackInfo callbackInfo; - public callback_call(CallbackInfo callbackInfo, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { - super(client, protocolFactory, transport, resultHandler, false); - this.callbackInfo = callbackInfo; - } - - public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { - prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("callback", org.apache.thrift.protocol.TMessageType.CALL, 0)); - callback_args args = new callback_args(); - args.setCallbackInfo(callbackInfo); - args.write(prot); - prot.writeMessageEnd(); - } - - public void getResult() throws org.apache.thrift.TException { - if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { - throw new IllegalStateException("Method call not finished!"); - } - org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); - org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); - (new Client(prot)).recv_callback(); - } - } - - } - - public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { - private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName()); - public Processor(I iface) { - super(iface, getProcessMap(new HashMap>())); - } - - protected Processor(I iface, Map> processMap) { - super(iface, getProcessMap(processMap)); - } - - private static Map> getProcessMap(Map> processMap) { - processMap.put("callback", new callback()); - return processMap; - } - - public static class callback extends org.apache.thrift.ProcessFunction { - public callback() { - super("callback"); - } - - public callback_args getEmptyArgsInstance() { - return new callback_args(); - } - - protected boolean isOneway() { - return false; - } - - public callback_result getResult(I iface, callback_args args) throws org.apache.thrift.TException { - callback_result result = new callback_result(); - iface.callback(args.callbackInfo); - return result; - } - } - - } - - public static class AsyncProcessor extends org.apache.thrift.TBaseAsyncProcessor { - private static final Logger LOGGER = LoggerFactory.getLogger(AsyncProcessor.class.getName()); - public AsyncProcessor(I iface) { - super(iface, getProcessMap(new HashMap>())); - } - - protected AsyncProcessor(I iface, Map> processMap) { - super(iface, getProcessMap(processMap)); - } - - private static Map> getProcessMap(Map> processMap) { - processMap.put("callback", new callback()); - return processMap; - } - - public static class callback extends org.apache.thrift.AsyncProcessFunction { - public callback() { - super("callback"); - } - - public callback_args getEmptyArgsInstance() { - return new callback_args(); - } - - public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { - final org.apache.thrift.AsyncProcessFunction fcall = this; - return new AsyncMethodCallback() { - public void onComplete(Void o) { - callback_result result = new callback_result(); - try { - fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); - return; - } catch (Exception e) { - LOGGER.error("Exception writing to internal frame buffer", e); - } - fb.close(); - } - public void onError(Exception e) { - byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; - org.apache.thrift.TBase msg; - callback_result result = new callback_result(); - { - msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; - msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); - } - try { - fcall.sendResponse(fb,msg,msgType,seqid); - return; - } catch (Exception ex) { - LOGGER.error("Exception writing to internal frame buffer", ex); - } - fb.close(); - } - }; - } - - protected boolean isOneway() { - return false; - } - - public void start(I iface, callback_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { - iface.callback(args.callbackInfo,resultHandler); - } - } - - } - - public static class callback_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("callback_args"); - - private static final org.apache.thrift.protocol.TField CALLBACK_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("callbackInfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); - - private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); - static { - schemes.put(StandardScheme.class, new callback_argsStandardSchemeFactory()); - schemes.put(TupleScheme.class, new callback_argsTupleSchemeFactory()); - } - - public CallbackInfo callbackInfo; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - CALLBACK_INFO((short)1, "callbackInfo"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // CALLBACK_INFO - return CALLBACK_INFO; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); - } - - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - public short getThriftFieldId() { - return _thriftId; - } - - public String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.CALLBACK_INFO, new org.apache.thrift.meta_data.FieldMetaData("callbackInfo", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CallbackInfo.class))); - metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(callback_args.class, metaDataMap); - } - - public callback_args() { - } - - public callback_args( - CallbackInfo callbackInfo) - { - this(); - this.callbackInfo = callbackInfo; - } - - /** - * Performs a deep copy on other. - */ - public callback_args(callback_args other) { - if (other.isSetCallbackInfo()) { - this.callbackInfo = new CallbackInfo(other.callbackInfo); - } - } - - public callback_args deepCopy() { - return new callback_args(this); - } - - @Override - public void clear() { - this.callbackInfo = null; - } - - public CallbackInfo getCallbackInfo() { - return this.callbackInfo; - } - - public callback_args setCallbackInfo(CallbackInfo callbackInfo) { - this.callbackInfo = callbackInfo; - return this; - } - - public void unsetCallbackInfo() { - this.callbackInfo = null; - } - - /** Returns true if field callbackInfo is set (has been assigned a value) and false otherwise */ - public boolean isSetCallbackInfo() { - return this.callbackInfo != null; - } - - public void setCallbackInfoIsSet(boolean value) { - if (!value) { - this.callbackInfo = null; - } - } - - public void setFieldValue(_Fields field, Object value) { - switch (field) { - case CALLBACK_INFO: - if (value == null) { - unsetCallbackInfo(); - } else { - setCallbackInfo((CallbackInfo)value); - } - break; - - } - } - - public Object getFieldValue(_Fields field) { - switch (field) { - case CALLBACK_INFO: - return getCallbackInfo(); - - } - throw new IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); - } - - switch (field) { - case CALLBACK_INFO: - return isSetCallbackInfo(); - } - throw new IllegalStateException(); - } - - @Override - public boolean equals(Object that) { - if (that == null) - return false; - if (that instanceof callback_args) - return this.equals((callback_args)that); - return false; - } - - public boolean equals(callback_args that) { - if (that == null) - return false; - - boolean this_present_callbackInfo = true && this.isSetCallbackInfo(); - boolean that_present_callbackInfo = true && that.isSetCallbackInfo(); - if (this_present_callbackInfo || that_present_callbackInfo) { - if (!(this_present_callbackInfo && that_present_callbackInfo)) - return false; - if (!this.callbackInfo.equals(that.callbackInfo)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - List list = new ArrayList(); - - boolean present_callbackInfo = true && (isSetCallbackInfo()); - list.add(present_callbackInfo); - if (present_callbackInfo) - list.add(callbackInfo); - - return list.hashCode(); - } - - @Override - public int compareTo(callback_args other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = Boolean.valueOf(isSetCallbackInfo()).compareTo(other.isSetCallbackInfo()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetCallbackInfo()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.callbackInfo, other.callbackInfo); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - schemes.get(iprot.getScheme()).getScheme().read(iprot, this); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - schemes.get(oprot.getScheme()).getScheme().write(oprot, this); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("callback_args("); - boolean first = true; - - sb.append("callbackInfo:"); - if (this.callbackInfo == null) { - sb.append("null"); - } else { - sb.append(this.callbackInfo); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - if (callbackInfo != null) { - callbackInfo.validate(); - } - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class callback_argsStandardSchemeFactory implements SchemeFactory { - public callback_argsStandardScheme getScheme() { - return new callback_argsStandardScheme(); - } - } - - private static class callback_argsStandardScheme extends StandardScheme { - - public void read(org.apache.thrift.protocol.TProtocol iprot, callback_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // CALLBACK_INFO - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.callbackInfo = new CallbackInfo(); - struct.callbackInfo.read(iprot); - struct.setCallbackInfoIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot, callback_args struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.callbackInfo != null) { - oprot.writeFieldBegin(CALLBACK_INFO_FIELD_DESC); - struct.callbackInfo.write(oprot); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class callback_argsTupleSchemeFactory implements SchemeFactory { - public callback_argsTupleScheme getScheme() { - return new callback_argsTupleScheme(); - } - } - - private static class callback_argsTupleScheme extends TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, callback_args struct) throws org.apache.thrift.TException { - TTupleProtocol oprot = (TTupleProtocol) prot; - BitSet optionals = new BitSet(); - if (struct.isSetCallbackInfo()) { - optionals.set(0); - } - oprot.writeBitSet(optionals, 1); - if (struct.isSetCallbackInfo()) { - struct.callbackInfo.write(oprot); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, callback_args struct) throws org.apache.thrift.TException { - TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(1); - if (incoming.get(0)) { - struct.callbackInfo = new CallbackInfo(); - struct.callbackInfo.read(iprot); - struct.setCallbackInfoIsSet(true); - } - } - } - - } - - public static class callback_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("callback_result"); - - - private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); - static { - schemes.put(StandardScheme.class, new callback_resultStandardSchemeFactory()); - schemes.put(TupleScheme.class, new callback_resultTupleSchemeFactory()); - } - - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { -; - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); - } - - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - public short getThriftFieldId() { - return _thriftId; - } - - public String getFieldName() { - return _fieldName; - } - } - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(callback_result.class, metaDataMap); - } - - public callback_result() { - } - - /** - * Performs a deep copy on other. - */ - public callback_result(callback_result other) { - } - - public callback_result deepCopy() { - return new callback_result(this); - } - - @Override - public void clear() { - } - - public void setFieldValue(_Fields field, Object value) { - switch (field) { - } - } - - public Object getFieldValue(_Fields field) { - switch (field) { - } - throw new IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); - } - - switch (field) { - } - throw new IllegalStateException(); - } - - @Override - public boolean equals(Object that) { - if (that == null) - return false; - if (that instanceof callback_result) - return this.equals((callback_result)that); - return false; - } - - public boolean equals(callback_result that) { - if (that == null) - return false; - - return true; - } - - @Override - public int hashCode() { - List list = new ArrayList(); - - return list.hashCode(); - } - - @Override - public int compareTo(callback_result other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - return 0; - } - - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - schemes.get(iprot.getScheme()).getScheme().read(iprot, this); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - schemes.get(oprot.getScheme()).getScheme().write(oprot, this); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("callback_result("); - boolean first = true; - - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class callback_resultStandardSchemeFactory implements SchemeFactory { - public callback_resultStandardScheme getScheme() { - return new callback_resultStandardScheme(); - } - } - - private static class callback_resultStandardScheme extends StandardScheme { - - public void read(org.apache.thrift.protocol.TProtocol iprot, callback_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot, callback_result struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class callback_resultTupleSchemeFactory implements SchemeFactory { - public callback_resultTupleScheme getScheme() { - return new callback_resultTupleScheme(); - } - } - - private static class callback_resultTupleScheme extends TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, callback_result struct) throws org.apache.thrift.TException { - TTupleProtocol oprot = (TTupleProtocol) prot; - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, callback_result struct) throws org.apache.thrift.TException { - TTupleProtocol iprot = (TTupleProtocol) prot; - } - } - - } - -} diff --git a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift index f20fb9038e6..f2eb13f23d8 100644 --- a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift +++ b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift @@ -88,11 +88,6 @@ struct InterpreterCompletion { 3: string meta } -struct CallbackInfo { - 1: string host, - 2: i32 port -} - service RemoteInterpreterService { void createInterpreter(1: string intpGroupId, 2: string sessionKey, 3: string className, 4: map properties, 5: string userName); @@ -136,7 +131,3 @@ service RemoteInterpreterService { void onReceivedZeppelinResource(1: string object); } - -service RemoteInterpreterCallbackService { - void callback(1: CallbackInfo callbackInfo); -} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java index 6efe4560777..a4b3a2573ba 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java @@ -42,7 +42,7 @@ public void tearDown() throws Exception { @Test public void testStartStop() throws InterruptedException, IOException, TException { - RemoteInterpreterServer server = new RemoteInterpreterServer("localhost", + RemoteInterpreterServer server = new RemoteInterpreterServer( RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces()); assertEquals(false, server.isRunning()); @@ -90,7 +90,7 @@ public void run() { @Test public void testStartStopWithQueuedEvents() throws InterruptedException, IOException, TException { - RemoteInterpreterServer server = new RemoteInterpreterServer("localhost", + RemoteInterpreterServer server = new RemoteInterpreterServer( RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces()); assertEquals(false, server.isRunning()); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index 58af4bf0de1..ae72f2dad76 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -53,12 +53,9 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess RemoteInterpreterManagedProcess.class); private final String interpreterRunner; - private CountDownLatch hostPortLatch; private DefaultExecutor executor; private ExecuteWatchdog watchdog; boolean running = false; - TServer callbackServer; - private String host = null; private int port = -1; private final String interpreterDir; private final String localRepoDir; @@ -82,7 +79,6 @@ public RemoteInterpreterManagedProcess( this.interpreterDir = intpDir; this.localRepoDir = localRepoDir; this.interpreterGroupName = interpreterGroupName; - this.hostPortLatch = new CountDownLatch(1); } RemoteInterpreterManagedProcess(String intpRunner, @@ -99,7 +95,6 @@ public RemoteInterpreterManagedProcess( this.interpreterDir = intpDir; this.localRepoDir = localRepoDir; this.interpreterGroupName = interpreterGroupName; - this.hostPortLatch = new CountDownLatch(1); } @Override @@ -115,64 +110,17 @@ public int getPort() { @Override public void start(String userName, Boolean isUserImpersonate) { // start server process - final String callbackHost; - final int callbackPort; try { - callbackHost = RemoteInterpreterUtils.findAvailableHostname(); - callbackPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); } catch (IOException e1) { throw new InterpreterException(e1); } - logger.info("Thrift server for callback will start. Port: {}", callbackPort); - try { - callbackServer = new TThreadPoolServer( - new TThreadPoolServer.Args(new TServerSocket(callbackPort)).processor( - new RemoteInterpreterCallbackService.Processor<>( - new RemoteInterpreterCallbackService.Iface() { - @Override - public void callback(CallbackInfo callbackInfo) throws TException { - logger.info("Registered: {}", callbackInfo); - host = callbackInfo.getHost(); - port = callbackInfo.getPort(); - hostPortLatch.countDown(); - } - }))); - // Start thrift server to receive callbackInfo from RemoteInterpreterServer; - new Thread(new Runnable() { - @Override - public void run() { - callbackServer.serve(); - } - }).start(); - - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - if (callbackServer.isServing()) { - callbackServer.stop(); - } - } - })); - - while (!callbackServer.isServing()) { - logger.debug("callbackServer is not serving"); - Thread.sleep(500); - } - logger.debug("callbackServer is serving now"); - } catch (TTransportException e) { - logger.error("callback server error.", e); - } catch (InterruptedException e) { - logger.warn("", e); - } - CommandLine cmdLine = CommandLine.parse(interpreterRunner); cmdLine.addArgument("-d", false); cmdLine.addArgument(interpreterDir, false); - cmdLine.addArgument("-c", false); - cmdLine.addArgument(callbackHost, false); cmdLine.addArgument("-p", false); - cmdLine.addArgument(Integer.toString(callbackPort), false); + cmdLine.addArgument(Integer.toString(port), false); if (isUserImpersonate && !userName.equals("anonymous")) { cmdLine.addArgument("-u", false); cmdLine.addArgument(userName, false); @@ -204,24 +152,38 @@ public void run() { throw new InterpreterException(e); } - try { - hostPortLatch.await(getConnectTimeout() * 2, TimeUnit.MILLISECONDS); - // Check if not running - if (null == host || -1 == port) { - hostPortLatch = new CountDownLatch(1); - callbackServer.stop(); - throw new InterpreterException("Cannot run interpreter"); + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < getConnectTimeout()) { + if (!running) { + try { + cmdOut.flush(); + } catch (IOException e) { + // nothing to do + } + throw new InterpreterException(new String(cmdOut.toByteArray())); + } + + try { + if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) { + break; + } else { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + logger.error("Exception in RemoteInterpreterProcess while synchronized reference " + + "Thread.sleep", e); + } + } + } catch (Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("Remote interpreter not yet accessible at localhost:" + port); + } } - } catch (InterruptedException e) { - logger.error("Remote interpreter is not accessible"); } processOutput.setOutputStream(null); } public void stop() { - if (callbackServer.isServing()) { - callbackServer.stop(); - } if (isRunning()) { logger.info("kill interpreter process"); watchdog.destroyProcess(); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index ffbeac89191..dc3bae8b73c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -78,8 +78,8 @@ public void testClientFactory() throws Exception { } @Test - public void testStartStopRemoteInterpreter() throws TException, InterruptedException, IOException { - RemoteInterpreterServer server = new RemoteInterpreterServer(null, 3678); + public void testStartStopRemoteInterpreter() throws TException, InterruptedException { + RemoteInterpreterServer server = new RemoteInterpreterServer(3678); server.start(); boolean running = false; long startTime = System.currentTimeMillis(); From 78eb0ba6ad8aa20d2bf40becc9d04b7bb42f6831 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 2 Jul 2017 14:01:55 +0900 Subject: [PATCH 38/52] Removed unrelated changes --- pom.xml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pom.xml b/pom.xml index ea08421e12c..ae296352973 100644 --- a/pom.xml +++ b/pom.xml @@ -267,6 +267,33 @@ ${shiro.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-client + ${hadoop.version} + + junit From 75962109c5f26cacc3bab194dcb97705e350ec6e Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 2 Jul 2017 14:06:04 +0900 Subject: [PATCH 39/52] Removed unrelated changes --- .../remote/RemoteInterpreterUtils.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java index 32354af3421..de00185f535 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java @@ -19,27 +19,12 @@ import java.io.IOException; import java.net.ConnectException; -import java.net.Inet4Address; -import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.InterfaceAddress; -import java.net.NetworkInterface; import java.net.ServerSocket; import java.net.Socket; -import java.net.SocketException; -import java.net.UnknownHostException; -import java.util.Collections; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.zeppelin.interpreter.thrift.CallbackInfo; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterCallbackService; - /** * */ From b82c25d3d808200d0c8508f9faf2bfc77d2946bf Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 2 Jul 2017 14:26:02 +0900 Subject: [PATCH 40/52] Removed unrelated changes --- .../cluster/yarn/YarnRemoteInterpreterServer.java | 2 +- .../interpreter/remote/RemoteInterpreterServer.java | 3 +-- .../remote/RemoteInterpreterManagedProcess.java | 9 --------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java index ce5aea38177..0ccb33f55ba 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java @@ -47,7 +47,7 @@ public class YarnRemoteInterpreterServer extends RemoteInterpreterServer { private AMRMClientAsync amrmClientAsync; public YarnRemoteInterpreterServer(int port) throws TTransportException, IOException { - super(null, port); + super(port); configuration = new Configuration(); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index b1686446fcb..9912703b3e3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -185,8 +185,7 @@ public boolean isRunning() { } - public static void main(String[] args) - throws TTransportException, InterruptedException { + public static void main(String[] args) throws Exception { int port = Constants.ZEPPELIN_INTERPRETER_DEFAUlT_PORT; if (args.length > 0) { port = Integer.parseInt(args[0]); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index ae72f2dad76..3ed57e376e6 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -21,8 +21,6 @@ import java.io.IOException; import java.io.OutputStream; import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.ExecuteException; @@ -31,18 +29,11 @@ import org.apache.commons.exec.LogOutputStream; import org.apache.commons.exec.PumpStreamHandler; import org.apache.commons.exec.environment.EnvironmentUtils; -import org.apache.thrift.TException; -import org.apache.thrift.server.TServer; -import org.apache.thrift.server.TThreadPoolServer; -import org.apache.thrift.transport.TServerSocket; -import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.thrift.CallbackInfo; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterCallbackService; /** * This class manages start / stop of remote interpreter process From b46aa13c16e78f048fc0a2bd11e4a3f6cb0ab494 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 2 Jul 2017 15:39:50 +0900 Subject: [PATCH 41/52] Removed unrelated changes --- .../zeppelin/interpreter/remote/RemoteInterpreter.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index eb14a7977dd..5d85f32f474 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -186,11 +186,12 @@ public RemoteInterpreterProcess getInterpreterProcess() { port); } else { String clusterManagerKey = getProperty(Constants.ZEPPELIN_CLUSTER_MANAGER_KEY); - ClusterManager clusterManager = - clusterManagerFactory.getClusterManager(clusterManagerKey); + ClusterManager clusterManager; //TODO(jl): Fix the parameter list to unify all methods - if (null != clusterManager) { + if (null != clusterManagerFactory + && null != + (clusterManager = clusterManagerFactory.getClusterManager(clusterManagerKey))) { remoteProcess = clusterManager .createInterpreter(sessionKey, interpreterGroupName, group, env, property, connectTimeout, remoteInterpreterProcessListener, applicationEventListener, From 0d610cb88f9456d2ce8b2a52803b8a9d022d8c6a Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 2 Jul 2017 22:54:20 +0900 Subject: [PATCH 42/52] Added simple javadoc --- .../main/java/org/apache/zeppelin/cluster/ClusterManager.java | 2 +- .../org/apache/zeppelin/cluster/ClusterManagerFactory.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java index a4e80d5f45a..5e1d96d0de4 100644 --- a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -26,7 +26,7 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; /** - * + * This is an abstraction class for implementing cluster managing service. */ public abstract class ClusterManager { diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java index 1211005b41c..9e5abb4f1b9 100644 --- a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java @@ -36,7 +36,8 @@ import org.slf4j.LoggerFactory; /** - * + * This class handle to create and initializing actual clustermanagers. It uses reflection and + * URLClassloader to load and make an object of cluster manager. */ public class ClusterManagerFactory { From ded25e4136a43f0eea5a5ff7f1ede342488412ef Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 2 Jul 2017 22:58:12 +0900 Subject: [PATCH 43/52] Added simple javadoc for yarn related classes --- .../src/main/java/org/apache/zeppelin/cluster/yarn/Client.java | 2 +- .../zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java | 3 ++- .../zeppelin/interpreter/remote/RemoteInterpreterProcess.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 00f18a793e5..fe1939c3696 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -39,7 +39,7 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; /** - * + * This class is in charge of making yarn client and implementing cluster manager. */ public class Client extends ClusterManager { diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java index 0ccb33f55ba..fdf882d2ed0 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnRemoteInterpreterServer.java @@ -38,7 +38,8 @@ import static org.apache.hadoop.yarn.api.records.FinalApplicationStatus.SUCCEEDED; /** - * + * This class register its host and port to yarn application information. Those information will be + * used in RemoteInterpreterYarnProcess. Rest of them has a same function of RemoteInterpreterServer */ public class YarnRemoteInterpreterServer extends RemoteInterpreterServer { private static final Logger logger = LoggerFactory.getLogger(YarnRemoteInterpreterServer.class); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 2e8188a2fbd..ffbd8bf01c8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -27,7 +27,8 @@ import java.util.concurrent.atomic.AtomicInteger; /** - * Abstract class for interpreter process + * Abstract class for interpreter process. This class actually makes InterpreterProcess for yarn and + * pass actual data to interpreter by calling thrift methods. */ public abstract class RemoteInterpreterProcess { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class); From 81f1ec335a7d8154d249f4f16fab725eee9c5220 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Tue, 4 Jul 2017 00:17:37 +0900 Subject: [PATCH 44/52] Simplified pom.xml --- pom.xml | 14 +--- zeppelin-cluster/yarn/pom.xml | 128 ++++++++++++---------------------- 2 files changed, 45 insertions(+), 97 deletions(-) diff --git a/pom.xml b/pom.xml index ae296352973..fe4bbd51b6e 100644 --- a/pom.xml +++ b/pom.xml @@ -270,22 +270,12 @@ org.apache.hadoop - hadoop-common - ${hadoop.version} - - - org.apache.hadoop - hadoop-hdfs + hadoop-client ${hadoop.version} org.apache.hadoop - hadoop-yarn-api - ${hadoop.version} - - - org.apache.hadoop - hadoop-yarn-common + hadoop-common ${hadoop.version} diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml index dce39e523e8..6a27f09760b 100644 --- a/zeppelin-cluster/yarn/pom.xml +++ b/zeppelin-cluster/yarn/pom.xml @@ -51,36 +51,15 @@ org.apache.hadoop - hadoop-yarn-client - - - - org.apache.hadoop - hadoop-yarn-common + hadoop-client - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-jaxrs - - - org.codehaus.jackson - jackson-xc - - - org.apache.commons - commons-compress - - - org.codehaus.jackson - jackson-mapper-asl + com.jcraft + jsch com.sun.jersey - jersey-server + jersey-core com.sun.jersey @@ -88,90 +67,77 @@ com.sun.jersey - jersey-core + jersey-server javax.servlet servlet-api - - - - - org.apache.hadoop - hadoop-yarn-api - - - - org.apache.hadoop - hadoop-common - - javax.servlet - servlet-api + org.apache.commons + commons-compress - org.codehaus.jackson - jackson-core-asl + org.apache.hadoop + hadoop-mapreduce-client-app - org.codehaus.jackson - jackson-jaxrs + org.apache.hadoop + hadoop-mapreduce-client-core - org.codehaus.jackson - jackson-xc + org.apache.hadoop + hadoop-mapreduce-client-jobclient - com.jcraft - jsch + org.codehaus.jackson + jackson-core-asl - org.apache.commons - commons-compress + org.codehaus.jackson + jackson-jaxrs org.codehaus.jackson jackson-mapper-asl - com.sun.jersey - jersey-server + org.codehaus.jackson + jackson-xc - com.sun.jersey - jersey-core + org.mortbay.jetty + jetty-util - javax.servlet - servlet-api + xerces + xercesImpl + + + io.netty + netty - org.apache.hadoop - hadoop-hdfs + hadoop-yarn-client - javax.servlet - servlet-api - - - org.codehaus.jackson - jackson-core-asl + com.sun.jersey + jersey-core - org.codehaus.jackson - jackson-jaxrs + com.sun.jersey + jersey-json - org.codehaus.jackson - jackson-xc + com.sun.jersey + jersey-server - com.jcraft - jsch + javax.servlet + servlet-api org.apache.commons @@ -179,27 +145,19 @@ org.codehaus.jackson - jackson-mapper-asl - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core + jackson-core-asl - javax.servlet - servlet-api + org.codehaus.jackson + jackson-jaxrs - xerces - xercesImpl + org.codehaus.jackson + jackson-mapper-asl - io.netty - netty + org.mortbay.jetty + jetty-util From d35f0e1fc919442e19c389cf68d4f0555c3fef70 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 6 Jul 2017 00:39:28 +0900 Subject: [PATCH 45/52] Added yarn cluster test --- pom.xml | 8 ++ zeppelin-cluster/yarn/pom.xml | 43 +++++++ .../apache/zeppelin/cluster/yarn/Client.java | 9 +- .../yarn/RemoteInterpreterYarnProcess.java | 26 ++++- .../zeppelin/cluster/yarn/YarnClientTest.java | 108 ++++++++++++++++++ .../yarn/src/test/resources/log4j.properties | 25 ++++ 6 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java create mode 100644 zeppelin-cluster/yarn/src/test/resources/log4j.properties diff --git a/pom.xml b/pom.xml index fe4bbd51b6e..fa8e79528d6 100644 --- a/pom.xml +++ b/pom.xml @@ -292,6 +292,14 @@ test + + org.apache.hadoop + hadoop-yarn-server-tests + tests + ${hadoop.version} + test + + org.assertj assertj-core diff --git a/zeppelin-cluster/yarn/pom.xml b/zeppelin-cluster/yarn/pom.xml index 6a27f09760b..34668ffe0fb 100644 --- a/zeppelin-cluster/yarn/pom.xml +++ b/zeppelin-cluster/yarn/pom.xml @@ -34,6 +34,10 @@ Zeppelin Cluster: yarn Zeppelin Yarn Cluster + + 1.9 + + org.apache.zeppelin @@ -161,6 +165,45 @@ + + + junit + junit + + + org.apache.hadoop + hadoop-yarn-server-tests + tests + + + + + com.sun.jersey + jersey-core + ${jersey-1.version} + test + + + com.sun.jersey + jersey-json + ${jersey-1.version} + test + + + com.sun.jersey + jersey-server + ${jersey-1.version} + test + + + com.sun.jersey.contribs + jersey-guice + ${jersey-1.version} + test + diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index fe1939c3696..46544b8664b 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -60,10 +60,17 @@ public Client() { this.started = false; } + // For Testing + Client(Configuration configuration) { + this.configuration = configuration; + } + public synchronized void start() { if (!started) { // it will help when calling it multiple times from different threads logger.info("Start to initialize yarn client"); - this.configuration = new Configuration(); + if (null == this.configuration) { + this.configuration = new Configuration(); + } this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(configuration); this.yarnClient.start(); diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index 8eb8841ff5c..d54043a3d61 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -105,6 +106,8 @@ public class RemoteInterpreterYarnProcess extends RemoteInterpreterProcess { private String host = null; private int port = -1; + private String extraClasspath = null; + RemoteInterpreterYarnProcess(int connectTimeout, RemoteInterpreterProcessListener listener, ApplicationEventListener appListener, YarnClient yarnClient, String homeDir, String interpreterDir, Configuration configuration, String name, String group, @@ -174,8 +177,12 @@ public void start(String userName, Boolean isUserImpersonate) { classpathStrings.add(0, "./*"); classpathStrings.add(0, ApplicationConstants.Environment.CLASSPATH.$$()); classpathStrings.add("./log4j.properties"); - classpathStrings.add(System.getenv("HADOOP_CONF_DIR")); - classpathStrings.add("`hadoop classpath 2> /dev/null`"); + if (isHadoopConfSet()) { + classpathStrings.add(System.getenv("HADOOP_CONF_DIR")); + } + if (null != extraClasspath) { + classpathStrings.add(extraClasspath); + } String classpathEnv = Joiner.on(ApplicationConstants.CLASS_PATH_SEPARATOR).join(classpathStrings); @@ -295,7 +302,11 @@ public void start(String userName, Boolean isUserImpersonate) { List vargs = Lists.newArrayList(); - vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); + if (null != System.getenv("JAVA_HOME")) { + vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); + } else { + vargs.add("java"); + } int memory; int defaultMemory = Integer.valueOf(ZEPPELIN_YARN_MEMORY_DEFAULT); @@ -384,6 +395,9 @@ public void start(String userName, Boolean isUserImpersonate) { waitingInitialized.await(5, TimeUnit.MINUTES); if (oldState != RUNNING) { stop(); + ApplicationAttemptId applicationAttemptId = + yarnClient.getApplicationReport(applicationId).getCurrentApplicationAttemptId(); + logger.error(yarnClient.getApplicationAttemptReport(applicationAttemptId).getDiagnostics()); throw new InterpreterException("Failed to initialize yarn application: " + applicationId); } @@ -456,6 +470,12 @@ private void setRunning(boolean running) { this.isRunning = running; } + // For Testing + // It should be called before calling start() + void setExtraClasspath(String extraClasspath) { + this.extraClasspath = extraClasspath; + } + private class ApplicationMonitor implements Runnable { @Override diff --git a/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java b/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java new file mode 100644 index 00000000000..67bf439af91 --- /dev/null +++ b/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.cluster.yarn; + +import com.google.common.collect.Maps; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Created by jl on 04/07/2017. + */ +public class YarnClientTest { + private static MiniYARNCluster miniYARNCluster; + private static YarnConfiguration yarnConfiguration; + + @BeforeClass + public static void setupCluster() { + yarnConfiguration = new YarnConfiguration(); + yarnConfiguration.set("yarn.nodemanager.disk-health-checker" + + ".max-disk-utilization-per-disk-percentage", "100.0"); + yarnConfiguration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true"); + yarnConfiguration.set(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, "true"); + + miniYARNCluster = new MiniYARNCluster(YarnClientTest.class.getName(), 1, 1, 1); + miniYARNCluster.init(yarnConfiguration); + miniYARNCluster.start(); + + final CountDownLatch waitUntilLaunched = new CountDownLatch(1); + final AtomicInteger checkerRunning = new AtomicInteger(0); + + new Thread(new Runnable() { + @Override + public void run() { + while (0 < checkerRunning.get()) { + if (!"0".equals(miniYARNCluster.getConfig().get(YarnConfiguration.RM_ADDRESS).split(":")[1])) { + waitUntilLaunched.countDown(); + } + } + } + }).start(); + + try { + waitUntilLaunched.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // Do nothing + } + + if ("0".equals(miniYARNCluster.getConfig().get(YarnConfiguration.RM_ADDRESS).split(":")[1])) { + throw new RuntimeException("Cannot run MiniYarnCluster for testing yarn"); + } + } + + @AfterClass + public static void shutdownCluster() { + try { + miniYARNCluster.close(); + } catch (IOException e) { + // Do nothing + } + } + + @Test + public void clientTest() throws Exception { + System.out.println("RM_ADDRESS = " + miniYARNCluster.getConfig().get(YarnConfiguration.RM_ADDRESS)); + Client client = new Client(miniYARNCluster.getConfig()); + client.start(); + RemoteInterpreterProcess remoteInterpreterProcess = client.createInterpreter("id", "name", "fake", Maps.newHashMap(), new Properties(), 100000, null, null, "fakeHome", "fake"); + String parentClasspath = Paths.get(YarnRemoteInterpreterServer.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getParent().getParent().getParent().toAbsolutePath().toString(); + String commonClasspath = parentClasspath + "/common/target"; + String yarnClasspath = parentClasspath + "/yarn/target"; + System.out.println("parentClasspath: " + parentClasspath); + System.out.println("commonClasspath: " + commonClasspath); + System.out.println("yarnClasspath: " + yarnClasspath); + String interpreterClasspath = Paths.get(parentClasspath).getParent().toAbsolutePath().toString() + "/zeppelin-interpreter/target"; + ((RemoteInterpreterYarnProcess)remoteInterpreterProcess).setExtraClasspath(commonClasspath + "/classes" + ApplicationConstants.CLASS_PATH_SEPARATOR + commonClasspath + "/lib/*" + ApplicationConstants.CLASS_PATH_SEPARATOR + yarnClasspath + "/classes" + ApplicationConstants.CLASS_PATH_SEPARATOR + yarnClasspath + "/lib/*" + ApplicationConstants.CLASS_PATH_SEPARATOR + interpreterClasspath+ "/classes" + ApplicationConstants.CLASS_PATH_SEPARATOR + interpreterClasspath + "/lib/*"); + remoteInterpreterProcess.start("user", false); + assertNotNull("Host should not be null", remoteInterpreterProcess.getHost()); + assertFalse("Port should not be -1", -1 == remoteInterpreterProcess.getPort()); + } +} \ No newline at end of file diff --git a/zeppelin-cluster/yarn/src/test/resources/log4j.properties b/zeppelin-cluster/yarn/src/test/resources/log4j.properties new file mode 100644 index 00000000000..2d8eb8656ba --- /dev/null +++ b/zeppelin-cluster/yarn/src/test/resources/log4j.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n \ No newline at end of file From f2629c9099ea86dd94fe4fcca60a294e8bad6f93 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 6 Jul 2017 00:52:45 +0900 Subject: [PATCH 46/52] Fixed style --- .../interpreter/InterpreterFactory.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 1a0415952e0..c310aef9fab 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -19,20 +19,6 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; -import org.apache.commons.lang.NullArgumentException; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.dep.DependencyResolver; -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.display.AngularObjectRegistryListener; -import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.RepositoryException; - import java.io.File; import java.io.IOException; import java.lang.reflect.Constructor; @@ -47,7 +33,21 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import org.apache.commons.lang.NullArgumentException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.RepositoryException; + import org.apache.zeppelin.cluster.ClusterManagerFactory; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; /** * Manage interpreters. @@ -98,7 +98,7 @@ public InterpreterFactory(ZeppelinConfiguration conf, /** * @param id interpreterGroup id. Combination of interpreterSettingId + noteId/userId/shared - * depends on interpreter mode + * depends on interpreter mode */ @Override public InterpreterGroup createInterpreterGroup(String id, InterpreterOption option) From 0c08dc1277bd7349111e1f8f30703fe0b9fb4098 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 6 Jul 2017 19:13:41 +0900 Subject: [PATCH 47/52] Fixed classloader issue --- .../java/org/apache/zeppelin/cluster/ClusterManager.java | 5 +++++ .../org/apache/zeppelin/cluster/ClusterManagerFactory.java | 2 +- .../main/java/org/apache/zeppelin/cluster/yarn/Client.java | 7 +++++-- .../org/apache/zeppelin/cluster/yarn/YarnClientTest.java | 4 ++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java index 5e1d96d0de4..a57e6f8ead5 100644 --- a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -29,6 +29,11 @@ * This is an abstraction class for implementing cluster managing service. */ public abstract class ClusterManager { + protected final ClassLoader classLoader; + + public ClusterManager(ClassLoader classLoader) { + this.classLoader = classLoader; + } /** * Can throw `RuntimeException` diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java index 9e5abb4f1b9..0210f4458ef 100644 --- a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java @@ -133,7 +133,7 @@ private void registerClusterManager(String key, List jars) { IOUtils.copy(classNameInputstream, classNameWriter); className = classNameWriter.toString(); Class clazz = classLoader.loadClass(className); - Object cm = clazz.getConstructor().newInstance(); + Object cm = clazz.getConstructor(ClassLoader.class).newInstance(classLoader); logger.info("Class loaded. {}", clazz.getName()); if (!clusterManagerMap.containsKey(key)) { clusterManagerMap.put(key, (ClusterManager) cm); diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 46544b8664b..6cff7f40f25 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -56,12 +56,14 @@ public class Client extends ClusterManager { */ private Map idApplicationIdMap; - public Client() { + public Client(ClassLoader classLoader) { + super(classLoader); this.started = false; } // For Testing - Client(Configuration configuration) { + Client(ClassLoader classLoader, Configuration configuration) { + super(classLoader); this.configuration = configuration; } @@ -71,6 +73,7 @@ public synchronized void start() { if (null == this.configuration) { this.configuration = new Configuration(); } + this.configuration.setClassLoader(classLoader); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(configuration); this.yarnClient.start(); diff --git a/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java b/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java index 67bf439af91..767a3c03c01 100644 --- a/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java +++ b/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java @@ -35,7 +35,7 @@ import static org.junit.Assert.*; /** - * Created by jl on 04/07/2017. + * */ public class YarnClientTest { private static MiniYARNCluster miniYARNCluster; @@ -90,7 +90,7 @@ public static void shutdownCluster() { @Test public void clientTest() throws Exception { System.out.println("RM_ADDRESS = " + miniYARNCluster.getConfig().get(YarnConfiguration.RM_ADDRESS)); - Client client = new Client(miniYARNCluster.getConfig()); + Client client = new Client(ClassLoader.getSystemClassLoader(), miniYARNCluster.getConfig()); client.start(); RemoteInterpreterProcess remoteInterpreterProcess = client.createInterpreter("id", "name", "fake", Maps.newHashMap(), new Properties(), 100000, null, null, "fakeHome", "fake"); String parentClasspath = Paths.get(YarnRemoteInterpreterServer.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getParent().getParent().getParent().toAbsolutePath().toString(); From eeba4009d4371a48e2ec7069713fd47f6d83ad31 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 9 Jul 2017 23:48:32 +0900 Subject: [PATCH 48/52] Changed to use system classloader for all cluster managers --- bin/zeppelin-daemon.sh | 3 + .../zeppelin/cluster/ClusterManager.java | 8 +- .../cluster/ClusterManagerFactory.java | 105 ++++-------------- .../apache/zeppelin/cluster/Constants.java | 52 +++++++++ .../cluster/ClusterManagerFactoryTest.java | 2 +- .../apache/zeppelin/cluster/yarn/Client.java | 14 ++- .../yarn/RemoteInterpreterYarnProcess.java | 20 ++-- .../zeppelin/cluster/yarn/YarnClientTest.java | 2 +- .../zeppelin/interpreter/Constants.java | 28 ----- .../zeppelin/server/ZeppelinServer.java | 3 +- zeppelin-zengine/pom.xml | 6 + .../zeppelin/conf/ZeppelinConfiguration.java | 10 +- .../interpreter/remote/RemoteInterpreter.java | 4 +- 13 files changed, 122 insertions(+), 135 deletions(-) create mode 100644 zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/Constants.java diff --git a/bin/zeppelin-daemon.sh b/bin/zeppelin-daemon.sh index 33f3a090b52..54739845c3e 100755 --- a/bin/zeppelin-daemon.sh +++ b/bin/zeppelin-daemon.sh @@ -74,12 +74,15 @@ addJarInDir "${ZEPPELIN_HOME}" addJarInDir "${ZEPPELIN_HOME}/lib" addJarInDir "${ZEPPELIN_HOME}/lib/interpreter" addJarInDir "${ZEPPELIN_HOME}/lib/cluster/common" +addJarInDir "${ZEPPELIN_HOME}/lib/cluster/yarn" addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib" addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib" addJarInDir "${ZEPPELIN_HOME}/zeppelin-cluster/common/target" addJarInDir "${ZEPPELIN_HOME}/zeppelin-cluster/common/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-cluster/yarn/target" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-cluster/yarn/target/lib" CLASSPATH+=":${ZEPPELIN_CLASSPATH}" diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java index a57e6f8ead5..174992cd470 100644 --- a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManager.java @@ -29,11 +29,11 @@ * This is an abstraction class for implementing cluster managing service. */ public abstract class ClusterManager { - protected final ClassLoader classLoader; - public ClusterManager(ClassLoader classLoader) { - this.classLoader = classLoader; - } + /** + * @return name of ClusterManager to be used in setting + */ + public abstract String getClusterManagerName(); /** * Can throw `RuntimeException` diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java index 0210f4458ef..c598c968107 100644 --- a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java @@ -17,21 +17,10 @@ package org.apache.zeppelin.cluster; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; import java.lang.reflect.InvocationTargetException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.DirectoryStream.Filter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.List; import java.util.Map; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,13 +34,16 @@ public class ClusterManagerFactory { private static final String CLUSTER_COMMON_DIR_NAME = "common"; private static final String CLUSTER_CLASS_NAME_FILE = "clustermanager-class"; + private final List clusterManagerList; private final String zeppelinHome; private final String defaultClusterKey; private final Map clusterManagerMap; private boolean initialized; - public ClusterManagerFactory(String zeppelinHome, String defaultClusterKey) { + public ClusterManagerFactory(List clusterManagerList, String zeppelinHome, + String defaultClusterKey) { + this.clusterManagerList = clusterManagerList; this.zeppelinHome = zeppelinHome; this.defaultClusterKey = defaultClusterKey; this.clusterManagerMap = Maps.newHashMap(); @@ -83,86 +75,33 @@ synchronized void init() { if (initialized) { return; } - // Find clusters inside source tree - logger.info("ZeppelinHome: {}", Paths.get(zeppelinHome).toAbsolutePath().toString()); - try { - for (Path p : Files - .newDirectoryStream(Paths.get(zeppelinHome, "zeppelin-cluster"), new Filter() { - @Override - public boolean accept(Path entry) throws IOException { - return !entry.toString().endsWith(CLUSTER_COMMON_DIR_NAME); - } - })) { - String key = p.getFileName().toString(); - List jars = Lists.newArrayList(); - addJarsFromPath(Paths.get(p.toString(), "target"), jars); - addJarsFromPath(Paths.get(p.toString(), "target", "lib"), jars); - - registerClusterManager(key, jars); - } - for (Path p : Files - .newDirectoryStream(Paths.get(zeppelinHome, "lib", "cluster"), new Filter() { - @Override - public boolean accept(Path entry) throws IOException { - return !entry.toString().endsWith(CLUSTER_COMMON_DIR_NAME); - } - })) { - String key = p.getFileName().toString(); - List jars = Lists.newArrayList(); - addJarsFromPath(p, jars); - - registerClusterManager(key, jars); - } - } catch (IOException e) { - logger.debug("error while reading directory", e); - } + findAndRegisterClusterManager(); initialized = true; } - private void registerClusterManager(String key, List jars) { - ClassLoader parentClassLoader = ClassLoader.getSystemClassLoader(); - ClassLoader classLoader = new URLClassLoader(jars.toArray(new URL[0]), parentClassLoader); - String className = null; - - try (InputStream classNameInputstream = - classLoader.getResourceAsStream(CLUSTER_CLASS_NAME_FILE); - StringWriter classNameWriter = new StringWriter()) { - if (null != classNameInputstream) { - IOUtils.copy(classNameInputstream, classNameWriter); - className = classNameWriter.toString(); - Class clazz = classLoader.loadClass(className); - Object cm = clazz.getConstructor(ClassLoader.class).newInstance(classLoader); - logger.info("Class loaded. {}", clazz.getName()); - if (!clusterManagerMap.containsKey(key)) { - clusterManagerMap.put(key, (ClusterManager) cm); - } - } else { - logger.debug("No resources {} in {}", CLUSTER_COMMON_DIR_NAME, jars); - } - } catch (ClassNotFoundException | IllegalAccessException | InstantiationException - | NoSuchMethodException | InvocationTargetException | IOException e) { - if (null != className) { - logger.error("class name is defined {}, but witch doesn't exist", className); - } - } catch (NoClassDefFoundError e) { - logger.info("{} exists but insufficient dependencies. jars: {}", className, jars); + private void findAndRegisterClusterManager() { + if (null == clusterManagerList) { + return; } - } - public void addJarsFromPath(Path dir, List jars) { - try { - for (Path p : Files.newDirectoryStream(dir, new Filter() { - @Override - public boolean accept(Path entry) throws IOException { - return entry.toString().endsWith(".jar"); + for (String clusterManagerClassName : clusterManagerList) { + try { + Class clazz = Class.forName(clusterManagerClassName); + Object cm = clazz.getConstructor().newInstance(); + String name = ((ClusterManager) cm).getClusterManagerName(); + if (null == name || name.isEmpty()) { + throw new IllegalArgumentException("Cluster manager is null:" + clusterManagerClassName); + } + if (!clusterManagerMap.containsKey(name)) { + clusterManagerMap.put(name, (ClusterManager) cm); } - })) { - jars.add(p.toUri().toURL()); + logger.info("ClusterManager {} is loaded with class {}", name, clusterManagerClassName); + } catch (ClassNotFoundException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException | InstantiationException | IllegalArgumentException e) { + logger.error("Wrong cluster manager name: {}", clusterManagerClassName); } - } catch (IOException e) { - logger.error("Error occurs while reading {}", dir.toAbsolutePath().toString()); } } } diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/Constants.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/Constants.java new file mode 100644 index 00000000000..22bbeb81c6a --- /dev/null +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/Constants.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.cluster; + +/** + * TBD + */ +public class Constants { + + public static final String ZEPPELIN_CLUSTER_MANAGER_KEY = "zeppelin.cluster_manager"; + + public static final String ZEPPELIN_CLUSTER_MANAGER_YARN = "yarn"; + + public static final String ZEPPELIN_CLUSTER_MANAGER_LOCAL = "local"; + + public static final String ZEPPELIN_YARN_VCORES_KEY = "zeppelin.yarn.vcores"; + + public static final String ZEPPELIN_YARN_VCORES_DEFAULT = "1"; + + public static final String ZEPPELIN_YARN_MEMORY_KEY = "zeppelin.yarn.memory"; + + public static final String ZEPPELIN_YARN_MEMORY_DEFAULT = "1024"; + + public static final String ZEPPELIN_YARN_QUEUE_KEY = "zeppelin.yarn.queue"; + + public static final String ZEPPELIN_YARN_QUEUE_DEFAULT = "default"; + + public static final String ZEPPELIN_YARN_APPLICATION_TYPE_KEY = "zeppelin.yarn.application.type"; + + public static final String ZEPPELIN_YARN_APPLICATION_TYPE_DEFAULT = "ZEPPELIN INTERPRETER"; + + public static final String ZEPPELIN_YARN_PRIORITY_KEY = "zeppelin.yarn.priority"; + + public static final String ZEPPELIN_YARN_PRIORITY_DEFAULT = "0"; + + public static final String ZEPPELIN_YARN_PROPERTY_KEY_PREFIX = "zeppelin.yarn."; +} diff --git a/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java b/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java index 30d8abdb199..ca650ac4379 100644 --- a/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java +++ b/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java @@ -28,7 +28,7 @@ public class ClusterManagerFactoryTest { @Test public void initTest() { System.out.println(System.getenv("CLASSPATH")); - ClusterManagerFactory cmf = new ClusterManagerFactory("../../", "local"); + ClusterManagerFactory cmf = new ClusterManagerFactory(null, "../../", "local"); // TODO(jl): Fix order of building module for testing it //assertTrue("One of cluster manager will exists", null != cmf.getClusterManager("yarn") || null != cmf.getClusterManager("mesos")); } diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java index 6cff7f40f25..4ff3cafed20 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java @@ -38,6 +38,8 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_CLUSTER_MANAGER_YARN; + /** * This class is in charge of making yarn client and implementing cluster manager. */ @@ -56,24 +58,26 @@ public class Client extends ClusterManager { */ private Map idApplicationIdMap; - public Client(ClassLoader classLoader) { - super(classLoader); + public Client() { this.started = false; } // For Testing - Client(ClassLoader classLoader, Configuration configuration) { - super(classLoader); + Client(Configuration configuration) { this.configuration = configuration; } + @Override + public String getClusterManagerName() { + return ZEPPELIN_CLUSTER_MANAGER_YARN; + } + public synchronized void start() { if (!started) { // it will help when calling it multiple times from different threads logger.info("Start to initialize yarn client"); if (null == this.configuration) { this.configuration = new Configuration(); } - this.configuration.setClassLoader(classLoader); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(configuration); this.yarnClient.start(); diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index d54043a3d61..00b24bdd164 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -68,16 +68,16 @@ import static org.apache.hadoop.yarn.api.records.YarnApplicationState.RUNNING; import static org.apache.zeppelin.cluster.yarn.YarnUtils.addLocalResource; import static org.apache.zeppelin.cluster.yarn.YarnUtils.getPathsFromDirPath; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_APPLICATION_TYPE_DEFAULT; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_APPLICATION_TYPE_KEY; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_MEMORY_DEFAULT; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_MEMORY_KEY; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_PRIORITY_DEFAULT; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_PRIORITY_KEY; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_QUEUE_DEFAULT; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_QUEUE_KEY; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_VCORES_DEFAULT; -import static org.apache.zeppelin.interpreter.Constants.ZEPPELIN_YARN_VCORES_KEY; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_APPLICATION_TYPE_DEFAULT; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_APPLICATION_TYPE_KEY; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_MEMORY_DEFAULT; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_MEMORY_KEY; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_PRIORITY_DEFAULT; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_PRIORITY_KEY; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_QUEUE_DEFAULT; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_QUEUE_KEY; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_VCORES_DEFAULT; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_YARN_VCORES_KEY; /** * diff --git a/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java b/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java index 767a3c03c01..71945830252 100644 --- a/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java +++ b/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java @@ -90,7 +90,7 @@ public static void shutdownCluster() { @Test public void clientTest() throws Exception { System.out.println("RM_ADDRESS = " + miniYARNCluster.getConfig().get(YarnConfiguration.RM_ADDRESS)); - Client client = new Client(ClassLoader.getSystemClassLoader(), miniYARNCluster.getConfig()); + Client client = new Client(miniYARNCluster.getConfig()); client.start(); RemoteInterpreterProcess remoteInterpreterProcess = client.createInterpreter("id", "name", "fake", Maps.newHashMap(), new Properties(), 100000, null, null, "fakeHome", "fake"); String parentClasspath = Paths.get(YarnRemoteInterpreterServer.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getParent().getParent().getParent().toAbsolutePath().toString(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java index f1e9c8a9bf8..4c5b68b3384 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Constants.java @@ -36,34 +36,6 @@ public class Constants { public static final int ZEPPELIN_INTERPRETER_OUTPUT_LIMIT = 1024 * 100; - public static final String ZEPPELIN_CLUSTER_MANAGER_KEY = "zeppelin.cluster_manager"; - - public static final String ZEPPELIN_CLUSTER_MANAGER_YARN = "yarn"; - - public static final String ZEPPELIN_CLUSTER_MANAGER_LOCAL = "local"; - - public static final String ZEPPELIN_YARN_VCORES_KEY = "zeppelin.yarn.vcores"; - - public static final String ZEPPELIN_YARN_VCORES_DEFAULT = "1"; - - public static final String ZEPPELIN_YARN_MEMORY_KEY = "zeppelin.yarn.memory"; - - public static final String ZEPPELIN_YARN_MEMORY_DEFAULT = "1024"; - - public static final String ZEPPELIN_YARN_QUEUE_KEY = "zeppelin.yarn.queue"; - - public static final String ZEPPELIN_YARN_QUEUE_DEFAULT = "default"; - - public static final String ZEPPELIN_YARN_APPLICATION_TYPE_KEY = "zeppelin.yarn.application.type"; - - public static final String ZEPPELIN_YARN_APPLICATION_TYPE_DEFAULT = "ZEPPELIN INTERPRETER"; - - public static final String ZEPPELIN_YARN_PRIORITY_KEY = "zeppelin.yarn.priority"; - - public static final String ZEPPELIN_YARN_PRIORITY_DEFAULT = "0"; - - public static final String ZEPPELIN_YARN_PROPERTY_KEY_PREFIX = "zeppelin.yarn."; - public static final Map TIME_SUFFIXES; static { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 7179b900571..ae245211646 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -150,7 +150,8 @@ public ZeppelinServer() throws Exception { this.schedulerFactory = new SchedulerFactory(); this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); - this.clusterManagerFactory = new ClusterManagerFactory(conf.getHome(), "local"); + this.clusterManagerFactory = new ClusterManagerFactory(conf.getClusterManagerList(), + conf.getHome(), conf.getZeppelinDefaultCluster()); this.replFactory = new InterpreterFactory(conf, notebookWsServer, notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(), interpreterSettingManager, clusterManagerFactory); diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index d3879564d55..2eec26d3c58 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -65,6 +65,12 @@ ${project.version} + + org.apache.zeppelin + zeppelin-cluster-yarn + ${project.version} + + org.slf4j slf4j-api diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 793e244d257..2821b94391e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.conf; +import com.google.common.base.Splitter; import java.io.File; import java.net.URL; import java.util.Arrays; @@ -28,6 +29,8 @@ import org.apache.commons.configuration.XMLConfiguration; import org.apache.commons.configuration.tree.ConfigurationNode; import org.apache.commons.lang.StringUtils; + +import org.apache.zeppelin.cluster.yarn.Client; import org.apache.zeppelin.notebook.repo.GitNotebookRepo; import org.apache.zeppelin.util.Util; import org.slf4j.Logger; @@ -518,6 +521,10 @@ public String getZeppelinDefaultCluster() { return getString(ConfVars.ZEPPELIN_CLUSTER_DEFAULT); } + public List getClusterManagerList() { + return Splitter.on(",").splitToList(getString(ConfVars.ZEPPELIN_CLUSTER_MANAGER_LIST)); + } + public Map dumpConfigurations(ZeppelinConfiguration conf, ConfigurationKeyPredicate predicate) { Map configurations = new HashMap<>(); @@ -673,7 +680,8 @@ public static enum ConfVars { ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"), ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED("zeppelin.server.default.dir.allowed", false), ZEPPELIN_SERVER_JETTY_NAME("zeppelin.server.jetty.name", null), - ZEPPELIN_CLUSTER_DEFAULT("zeppelin.cluster.default", "local"); + ZEPPELIN_CLUSTER_DEFAULT("zeppelin.cluster.default", "local"), + ZEPPELIN_CLUSTER_MANAGER_LIST("zeppelin.cluster.manager.list", Client.class.getName()); private String varName; @SuppressWarnings("rawtypes") diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 5d85f32f474..392823bc866 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -41,6 +41,8 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import static org.apache.zeppelin.cluster.Constants.ZEPPELIN_CLUSTER_MANAGER_KEY; + /** * Proxy for Interpreter instance that runs on separate process */ @@ -185,7 +187,7 @@ public RemoteInterpreterProcess getInterpreterProcess() { host, port); } else { - String clusterManagerKey = getProperty(Constants.ZEPPELIN_CLUSTER_MANAGER_KEY); + String clusterManagerKey = getProperty(ZEPPELIN_CLUSTER_MANAGER_KEY); ClusterManager clusterManager; //TODO(jl): Fix the parameter list to unify all methods From f629ff8e1ad9152bd064e7b1393736f43a2f09ff Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Sun, 9 Jul 2017 23:49:04 +0900 Subject: [PATCH 49/52] Removed unused file --- zeppelin-cluster/yarn/src/main/resources/clustermanager-class | 1 - 1 file changed, 1 deletion(-) delete mode 100644 zeppelin-cluster/yarn/src/main/resources/clustermanager-class diff --git a/zeppelin-cluster/yarn/src/main/resources/clustermanager-class b/zeppelin-cluster/yarn/src/main/resources/clustermanager-class deleted file mode 100644 index 871c35ee9be..00000000000 --- a/zeppelin-cluster/yarn/src/main/resources/clustermanager-class +++ /dev/null @@ -1 +0,0 @@ -org.apache.zeppelin.cluster.yarn.Client \ No newline at end of file From 05532c2f81a1cb54d2b2329059b7220d94e8ec0a Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 10 Jul 2017 00:01:14 +0900 Subject: [PATCH 50/52] Fixed comments by review --- .../cluster/ClusterManagerFactory.java | 27 ++++++++----------- .../yarn/RemoteInterpreterYarnProcess.java | 2 +- .../{Client.java => YarnClusterManager.java} | 8 +++--- ...tTest.java => YarnClusterManagerTest.java} | 13 ++++----- .../zeppelin/server/ZeppelinServer.java | 2 +- .../zeppelin/conf/ZeppelinConfiguration.java | 4 +-- 6 files changed, 26 insertions(+), 30 deletions(-) rename zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/{Client.java => YarnClusterManager.java} (96%) rename zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/{YarnClientTest.java => YarnClusterManagerTest.java} (89%) diff --git a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java index c598c968107..0dffb40507b 100644 --- a/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java +++ b/zeppelin-cluster/common/src/main/java/org/apache/zeppelin/cluster/ClusterManagerFactory.java @@ -31,44 +31,39 @@ public class ClusterManagerFactory { private static final Logger logger = LoggerFactory.getLogger(ClusterManagerFactory.class); - private static final String CLUSTER_COMMON_DIR_NAME = "common"; - private static final String CLUSTER_CLASS_NAME_FILE = "clustermanager-class"; private final List clusterManagerList; - private final String zeppelinHome; - private final String defaultClusterKey; + private final String defaultClusterManagerName; private final Map clusterManagerMap; private boolean initialized; - public ClusterManagerFactory(List clusterManagerList, String zeppelinHome, - String defaultClusterKey) { + public ClusterManagerFactory(List clusterManagerList, String defaultClusterManagerName) { this.clusterManagerList = clusterManagerList; - this.zeppelinHome = zeppelinHome; - this.defaultClusterKey = defaultClusterKey; + this.defaultClusterManagerName = defaultClusterManagerName; this.clusterManagerMap = Maps.newHashMap(); this.initialized = false; } - public ClusterManager getClusterManager() { - return getClusterManager(defaultClusterKey); + public ClusterManager getDefaultClusterManager() { + return getClusterManager(defaultClusterManagerName); } - public ClusterManager getClusterManager(String key) { + public ClusterManager getClusterManager(String name) { if (!initialized) { init(); } - if (null == key) { - return getClusterManager(); + if (null == name) { + return getDefaultClusterManager(); } - if (!clusterManagerMap.containsKey(key)) { - logger.info("Not supported. {}", key); + if (!clusterManagerMap.containsKey(name)) { + logger.info("Not supported. {}", name); return null; } - return clusterManagerMap.get(key); + return clusterManagerMap.get(name); } synchronized void init() { diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java index 00b24bdd164..45e49fec866 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/RemoteInterpreterYarnProcess.java @@ -389,7 +389,7 @@ public void start(String userName, Boolean isUserImpersonate) { this.applicationId = appContext.getApplicationId(); yarnClient.submitApplication(appContext); - monitor = Client.scheduledExecutorService + monitor = YarnClusterManager.scheduledExecutorService .scheduleAtFixedRate(new ApplicationMonitor(), 1, 1, TimeUnit.SECONDS); waitingInitialized.await(5, TimeUnit.MINUTES); diff --git a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnClusterManager.java similarity index 96% rename from zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java rename to zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnClusterManager.java index 4ff3cafed20..d6cd1580eef 100644 --- a/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/Client.java +++ b/zeppelin-cluster/yarn/src/main/java/org/apache/zeppelin/cluster/yarn/YarnClusterManager.java @@ -43,9 +43,9 @@ /** * This class is in charge of making yarn client and implementing cluster manager. */ -public class Client extends ClusterManager { +public class YarnClusterManager extends ClusterManager { - private static final Logger logger = LoggerFactory.getLogger(Client.class); + private static final Logger logger = LoggerFactory.getLogger(YarnClusterManager.class); public static final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1); @@ -58,12 +58,12 @@ public class Client extends ClusterManager { */ private Map idApplicationIdMap; - public Client() { + public YarnClusterManager() { this.started = false; } // For Testing - Client(Configuration configuration) { + YarnClusterManager(Configuration configuration) { this.configuration = configuration; } diff --git a/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java b/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClusterManagerTest.java similarity index 89% rename from zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java rename to zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClusterManagerTest.java index 71945830252..4c0f0eca5b5 100644 --- a/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClientTest.java +++ b/zeppelin-cluster/yarn/src/test/java/org/apache/zeppelin/cluster/yarn/YarnClusterManagerTest.java @@ -37,7 +37,7 @@ /** * */ -public class YarnClientTest { +public class YarnClusterManagerTest { private static MiniYARNCluster miniYARNCluster; private static YarnConfiguration yarnConfiguration; @@ -49,7 +49,7 @@ public static void setupCluster() { yarnConfiguration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true"); yarnConfiguration.set(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, "true"); - miniYARNCluster = new MiniYARNCluster(YarnClientTest.class.getName(), 1, 1, 1); + miniYARNCluster = new MiniYARNCluster(YarnClusterManagerTest.class.getName(), 1, 1, 1); miniYARNCluster.init(yarnConfiguration); miniYARNCluster.start(); @@ -88,11 +88,12 @@ public static void shutdownCluster() { } @Test - public void clientTest() throws Exception { + public void yarnClusterManagerTest() throws Exception { System.out.println("RM_ADDRESS = " + miniYARNCluster.getConfig().get(YarnConfiguration.RM_ADDRESS)); - Client client = new Client(miniYARNCluster.getConfig()); - client.start(); - RemoteInterpreterProcess remoteInterpreterProcess = client.createInterpreter("id", "name", "fake", Maps.newHashMap(), new Properties(), 100000, null, null, "fakeHome", "fake"); + YarnClusterManager yarnClusterManager = new YarnClusterManager(miniYARNCluster.getConfig()); + yarnClusterManager.start(); + RemoteInterpreterProcess remoteInterpreterProcess = yarnClusterManager + .createInterpreter("id", "name", "fake", Maps.newHashMap(), new Properties(), 100000, null, null, "fakeHome", "fake"); String parentClasspath = Paths.get(YarnRemoteInterpreterServer.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getParent().getParent().getParent().toAbsolutePath().toString(); String commonClasspath = parentClasspath + "/common/target"; String yarnClasspath = parentClasspath + "/yarn/target"; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index ae245211646..85e5fb7bfb2 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -151,7 +151,7 @@ public ZeppelinServer() throws Exception { this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); this.clusterManagerFactory = new ClusterManagerFactory(conf.getClusterManagerList(), - conf.getHome(), conf.getZeppelinDefaultCluster()); + conf.getZeppelinDefaultCluster()); this.replFactory = new InterpreterFactory(conf, notebookWsServer, notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(), interpreterSettingManager, clusterManagerFactory); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 2821b94391e..863e070ccb9 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -30,7 +30,7 @@ import org.apache.commons.configuration.tree.ConfigurationNode; import org.apache.commons.lang.StringUtils; -import org.apache.zeppelin.cluster.yarn.Client; +import org.apache.zeppelin.cluster.yarn.YarnClusterManager; import org.apache.zeppelin.notebook.repo.GitNotebookRepo; import org.apache.zeppelin.util.Util; import org.slf4j.Logger; @@ -681,7 +681,7 @@ public static enum ConfVars { ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED("zeppelin.server.default.dir.allowed", false), ZEPPELIN_SERVER_JETTY_NAME("zeppelin.server.jetty.name", null), ZEPPELIN_CLUSTER_DEFAULT("zeppelin.cluster.default", "local"), - ZEPPELIN_CLUSTER_MANAGER_LIST("zeppelin.cluster.manager.list", Client.class.getName()); + ZEPPELIN_CLUSTER_MANAGER_LIST("zeppelin.cluster.manager.list", YarnClusterManager.class.getName()); private String varName; @SuppressWarnings("rawtypes") From 29c85caa78bb070cc9e4f042b164e3c2eb975656 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 10 Jul 2017 00:26:33 +0900 Subject: [PATCH 51/52] Changed parameters --- .../org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java b/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java index ca650ac4379..074ea44728e 100644 --- a/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java +++ b/zeppelin-cluster/common/src/test/java/org/apache/zeppelin/cluster/ClusterManagerFactoryTest.java @@ -28,7 +28,7 @@ public class ClusterManagerFactoryTest { @Test public void initTest() { System.out.println(System.getenv("CLASSPATH")); - ClusterManagerFactory cmf = new ClusterManagerFactory(null, "../../", "local"); + ClusterManagerFactory cmf = new ClusterManagerFactory(null, "local"); // TODO(jl): Fix order of building module for testing it //assertTrue("One of cluster manager will exists", null != cmf.getClusterManager("yarn") || null != cmf.getClusterManager("mesos")); } From 677fb59f555a8562bda9af3c4128ac71e6801e6b Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Mon, 10 Jul 2017 01:06:14 +0900 Subject: [PATCH 52/52] Fixed style --- .../java/org/apache/zeppelin/conf/ZeppelinConfiguration.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 863e070ccb9..91e7b8b5394 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -681,7 +681,8 @@ public static enum ConfVars { ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED("zeppelin.server.default.dir.allowed", false), ZEPPELIN_SERVER_JETTY_NAME("zeppelin.server.jetty.name", null), ZEPPELIN_CLUSTER_DEFAULT("zeppelin.cluster.default", "local"), - ZEPPELIN_CLUSTER_MANAGER_LIST("zeppelin.cluster.manager.list", YarnClusterManager.class.getName()); + ZEPPELIN_CLUSTER_MANAGER_LIST("zeppelin.cluster.manager.list", + YarnClusterManager.class.getName()); private String varName; @SuppressWarnings("rawtypes")