From 2745151cbb8c1f0f666a29997d4f48a2c45698e5 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 8 Nov 2017 10:56:30 +0800 Subject: [PATCH] ZEPPELIN-3039. Interpreter logs are mixed together --- bin/interpreter.sh | 10 +++---- .../launcher/InterpreterLaunchContext.java | 28 +++++++++++-------- .../interpreter/InterpreterSetting.java | 2 +- .../launcher/ShellScriptLauncher.java | 10 +++---- .../RemoteInterpreterManagedProcess.java | 14 ++++------ .../launcher/ShellScriptLauncherTest.java | 6 ++-- .../SparkInterpreterLauncherTest.java | 20 ++++++------- 7 files changed, 45 insertions(+), 45 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index d27b0767bf5..4e983ecb95f 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -54,7 +54,7 @@ while getopts "hc:p:d:l:v:u:g:" o; do fi ;; g) - INTERPRETER_GROUP_NAME=${OPTARG} + INTERPRETER_SETTING_NAME=${OPTARG} ;; esac done @@ -91,14 +91,12 @@ ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer INTERPRETER_ID=$(basename "${INTERPRETER_DIR}") ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-interpreter-${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid" -ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-" -if [[ ! -z "$INTERPRETER_GROUP_NAME" ]]; then - ZEPPELIN_LOGFILE+="${INTERPRETER_GROUP_NAME}-" -fi +ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-${INTERPRETER_SETTING_NAME}-" + if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then ZEPPELIN_LOGFILE+="${ZEPPELIN_IMPERSONATE_USER}-" fi -ZEPPELIN_LOGFILE+="${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log" +ZEPPELIN_LOGFILE+="${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log" JAVA_INTP_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java index db8f8dd2fa9..9e253555a90 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java @@ -17,7 +17,6 @@ package org.apache.zeppelin.interpreter.launcher; -import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterRunner; @@ -31,19 +30,22 @@ public class InterpreterLaunchContext { private Properties properties; private InterpreterOption option; private InterpreterRunner runner; - private String interpreterGroupId; - private String interpreterGroupName; + private String interpreterSettingId; + private String interpreterSettingGroup; + private String interpreterSettingName; public InterpreterLaunchContext(Properties properties, InterpreterOption option, InterpreterRunner runner, - String interpreterGroupId, - String interpreterGroupName) { + String interpreterSettingId, + String interpreterSettingGroup, + String interpreterSettingName) { this.properties = properties; this.option = option; this.runner = runner; - this.interpreterGroupId = interpreterGroupId; - this.interpreterGroupName = interpreterGroupName; + this.interpreterSettingId = interpreterSettingId; + this.interpreterSettingGroup = interpreterSettingGroup; + this.interpreterSettingName = interpreterSettingName; } public Properties getProperties() { @@ -58,11 +60,15 @@ public InterpreterRunner getRunner() { return runner; } - public String getInterpreterGroupId() { - return interpreterGroupId; + public String getInterpreterSettingId() { + return interpreterSettingId; } - public String getInterpreterGroupName() { - return interpreterGroupName; + public String getInterpreterSettingGroup() { + return interpreterSettingGroup; + } + + public String getInterpreterSettingName() { + return interpreterSettingName; } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 0596cc52551..4ce10a80d38 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -663,7 +663,7 @@ synchronized RemoteInterpreterProcess createInterpreterProcess() throws IOExcept createLauncher(); } InterpreterLaunchContext launchContext = new - InterpreterLaunchContext(getJavaProperties(), option, interpreterRunner, id, group); + InterpreterLaunchContext(getJavaProperties(), option, interpreterRunner, id, group, name); RemoteInterpreterProcess process = (RemoteInterpreterProcess) launcher.launch(launchContext); process.setRemoteInterpreterEventPoller( new RemoteInterpreterEventPoller(remoteInterpreterProcessListener, appEventListener)); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java index f419967dee5..0966ec543a2 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.Properties; /** * Interpreter Launcher which use shell script to launch the interpreter process. @@ -45,11 +44,12 @@ public ShellScriptLauncher(ZeppelinConfiguration zConf) { @Override public InterpreterClient launch(InterpreterLaunchContext context) { - LOGGER.info("Launching Interpreter: " + context.getInterpreterGroupName()); + LOGGER.info("Launching Interpreter: " + context.getInterpreterSettingGroup()); this.properties = context.getProperties(); InterpreterOption option = context.getOption(); InterpreterRunner runner = context.getRunner(); - String groupName = context.getInterpreterGroupName(); + String groupName = context.getInterpreterSettingGroup(); + String name = context.getInterpreterSettingName(); int connectTimeout = zConf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); @@ -61,12 +61,12 @@ public InterpreterClient launch(InterpreterLaunchContext context) { } else { // create new remote process String localRepoPath = zConf.getInterpreterLocalRepoPath() + "/" - + context.getInterpreterGroupId(); + + context.getInterpreterSettingId(); return new RemoteInterpreterManagedProcess( runner != null ? runner.getPath() : zConf.getInterpreterRemoteRunnerPath(), zConf.getCallbackPortRange(), zConf.getInterpreterDir() + "/" + groupName, localRepoPath, - buildEnvFromProperties(), connectTimeout, groupName); + buildEnvFromProperties(), connectTimeout, name); } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index 6e26e58f8d0..9f8f3462453 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -31,8 +31,6 @@ import org.apache.thrift.server.TThreadPoolServer; import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TTransportException; -import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.thrift.CallbackInfo; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterCallbackService; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; @@ -63,7 +61,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess private int port = -1; private final String interpreterDir; private final String localRepoDir; - private final String interpreterGroupName; + private final String interpreterSettingName; private Map env; @@ -74,14 +72,14 @@ public RemoteInterpreterManagedProcess( String localRepoDir, Map env, int connectTimeout, - String interpreterGroupName) { + String interpreterSettingName) { super(connectTimeout); this.interpreterRunner = intpRunner; this.portRange = portRange; this.env = env; this.interpreterDir = intpDir; this.localRepoDir = localRepoDir; - this.interpreterGroupName = interpreterGroupName; + this.interpreterSettingName = interpreterSettingName; } @Override @@ -167,7 +165,7 @@ public void run() { cmdLine.addArgument("-l", false); cmdLine.addArgument(localRepoDir, false); cmdLine.addArgument("-g", false); - cmdLine.addArgument(interpreterGroupName, false); + cmdLine.addArgument(interpreterSettingName, false); executor = new DefaultExecutor(); @@ -263,8 +261,8 @@ public String getInterpreterDir() { } @VisibleForTesting - public String getInterpreterGroupName() { - return interpreterGroupName; + public String getInterpreterSettingName() { + return interpreterSettingName; } @VisibleForTesting diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java index 9708ee7d563..0c7f4baacfb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java @@ -18,10 +18,8 @@ package org.apache.zeppelin.interpreter.launcher; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.junit.Test; import java.util.Properties; @@ -39,11 +37,11 @@ public void testLauncher() { properties.setProperty("ENV_1", "VALUE_1"); properties.setProperty("property_1", "value_1"); InterpreterOption option = new InterpreterOption(); - InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "groupName"); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "groupName", "name"); InterpreterClient client = launcher.launch(context); assertTrue( client instanceof RemoteInterpreterManagedProcess); RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; - assertEquals("groupName", interpreterProcess.getInterpreterGroupName()); + assertEquals("name", interpreterProcess.getInterpreterSettingName()); assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir()); assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java index 45e009f7194..b788ebdeeea 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java @@ -41,11 +41,11 @@ public void testLocalMode() { properties.setProperty("spark.jars", "jar_1"); InterpreterOption option = new InterpreterOption(); - InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark"); InterpreterClient client = launcher.launch(context); assertTrue( client instanceof RemoteInterpreterManagedProcess); RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; - assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals("spark", interpreterProcess.getInterpreterSettingName()); assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); @@ -66,11 +66,11 @@ public void testYarnClientMode_1() { properties.setProperty("spark.jars", "jar_1"); InterpreterOption option = new InterpreterOption(); - InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark"); InterpreterClient client = launcher.launch(context); assertTrue( client instanceof RemoteInterpreterManagedProcess); RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; - assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals("spark", interpreterProcess.getInterpreterSettingName()); assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); @@ -92,11 +92,11 @@ public void testYarnClientMode_2() { properties.setProperty("spark.jars", "jar_1"); InterpreterOption option = new InterpreterOption(); - InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark"); InterpreterClient client = launcher.launch(context); assertTrue( client instanceof RemoteInterpreterManagedProcess); RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; - assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals("spark", interpreterProcess.getInterpreterSettingName()); assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); @@ -117,11 +117,11 @@ public void testYarnClusterMode_1() { properties.setProperty("spark.jars", "jar_1"); InterpreterOption option = new InterpreterOption(); - InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark"); InterpreterClient client = launcher.launch(context); assertTrue( client instanceof RemoteInterpreterManagedProcess); RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; - assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals("spark", interpreterProcess.getInterpreterSettingName()); assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); @@ -144,11 +144,11 @@ public void testYarnClusterMode_2() { properties.setProperty("spark.jars", "jar_1"); InterpreterOption option = new InterpreterOption(); - InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark"); InterpreterClient client = launcher.launch(context); assertTrue( client instanceof RemoteInterpreterManagedProcess); RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; - assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals("spark", interpreterProcess.getInterpreterSettingName()); assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());