From 6aafac445db4cd8944caa07280fcdbebb2faea0f Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Thu, 11 Aug 2016 20:38:12 +0530 Subject: [PATCH 01/10] user should have option to run these interpreters as different user. --- bin/interpreter.sh | 8 ++++++-- conf/zeppelin-env.sh.template | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index a81c8f21067..7ac35e2554a 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -155,10 +155,14 @@ addJarInDirForIntp "${LOCAL_INTERPRETER_REPO}" CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}" +if [ -n "${ZEPPELIN_INTERPRETER_USER}" ]; then + ZEPPELIN_SSH_COMMAND="ssh $ZEPPELIN_INTERPRETER_USER@localhost " +fi + if [[ -n "${SPARK_SUBMIT}" ]]; then - ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & + ${ZEPPELIN_SSH_COMMAND} ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & else - ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & + ${ZEPPELIN_SSH_COMMAND} ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & fi pid=$! diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 14fdd54eeb6..9553dee19cc 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -36,6 +36,7 @@ # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). +# export ZEPPELIN_INTERPRETER_USER= # If user wants to run interpreter as different user #### Spark interpreter configuration #### From 0ab2fe0a05d5c574d16344d1dcc0cde39a577ed0 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Fri, 12 Aug 2016 15:46:43 +0530 Subject: [PATCH 02/10] add userName as parameter in createOrGetIntepreter --- bin/interpreter.sh | 8 +-- conf/shiro.ini | 12 ++-- conf/zeppelin-env.sh.template | 4 +- .../RemoteInterpreterManagedProcess.java | 4 ++ .../zeppelin/conf/ZeppelinConfiguration.java | 24 +++++-- .../interpreter/InterpreterFactory.java | 70 +++++++------------ .../org/apache/zeppelin/notebook/Note.java | 8 ++- .../apache/zeppelin/notebook/Paragraph.java | 3 +- .../interpreter/InterpreterFactoryTest.java | 2 +- .../notebook/NoteInterpreterLoaderTest.java | 34 ++++----- .../apache/zeppelin/notebook/NoteTest.java | 12 ++-- .../zeppelin/notebook/NotebookTest.java | 4 +- .../zeppelin/notebook/ParagraphTest.java | 2 +- 13 files changed, 93 insertions(+), 94 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 7ac35e2554a..2aa9988caa0 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -23,7 +23,7 @@ function usage() { echo "usage) $0 -p -d -l " } -while getopts "hp:d:l:v" o; do +while getopts "hp:d:l:v:u" o; do case ${o} in h) usage @@ -42,6 +42,9 @@ while getopts "hp:d:l:v" o; do . "${bin}/common.sh" getZeppelinVersion ;; + u) + ZEPPELIN_SSH_COMMAND="ssh ${OPTARG}@localhost " + ;; esac done @@ -155,9 +158,6 @@ addJarInDirForIntp "${LOCAL_INTERPRETER_REPO}" CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}" -if [ -n "${ZEPPELIN_INTERPRETER_USER}" ]; then - ZEPPELIN_SSH_COMMAND="ssh $ZEPPELIN_INTERPRETER_USER@localhost " -fi if [[ -n "${SPARK_SUBMIT}" ]]; then ${ZEPPELIN_SSH_COMMAND} ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & diff --git a/conf/shiro.ini b/conf/shiro.ini index 75a3fb9bd91..ce8102a2bb8 100644 --- a/conf/shiro.ini +++ b/conf/shiro.ini @@ -18,10 +18,10 @@ [users] # List of users with their password allowed to access Zeppelin. # To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections -admin = password1, admin -user1 = password2, role1, role2 -user2 = password3, role3 -user3 = password4, role2 +admin = admin, admin +user1 = user1, role1, role2 +user2 = user2, role3 +user3 = user3, role2 # Sample LDAP configuration, for user Authentication, currently tested for single Realm [main] @@ -75,5 +75,5 @@ admin = * #/api/interpreter/** = authc, roles[admin] #/api/configurations/** = authc, roles[admin] #/api/credential/** = authc, roles[admin] -/** = anon -#/** = authc +#/** = anon +/** = authc diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 9553dee19cc..ee24458acde 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -36,7 +36,9 @@ # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). -# export ZEPPELIN_INTERPRETER_USER= # If user wants to run interpreter as different user + +# export ZEPPELIN_INTERPRETER_RUNAS_USER= # set this to true if user wants to run interpreter as different user, and if shiro authentication is enabled then it will run as end web user. +# export ZEPPELIN_INTERPRETER_USER= # user to be run as if shiro authentication is not enbaled and you want to run it as different user than the server. #### Spark interpreter configuration #### diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index eb345338569..05a43023cc9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -102,6 +102,10 @@ public void start() { cmdLine.addArgument("-p", false); cmdLine.addArgument(Integer.toString(port), false); cmdLine.addArgument("-l", false); + + + cmdLine.addArgument("-u", false); + cmdLine.addArgument(localRepoDir, false); executor = new DefaultExecutor(); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 43fb6be02c0..098c8d28d93 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -17,12 +17,6 @@ package org.apache.zeppelin.conf; -import java.net.URL; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.XMLConfiguration; import org.apache.commons.configuration.tree.ConfigurationNode; @@ -30,6 +24,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.URL; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * Zeppelin configuration. * @@ -431,6 +431,14 @@ public boolean getUseJdbcAlias() { return getBoolean(ConfVars.ZEPPELIN_USE_JDBC_ALIAS); } + public boolean getInterpreterRunasUser() { + return getBoolean(ConfVars.ZEPPELIN_INTERPRETER_RUNAS_USER); + } + + public String getZeppelinInterpreterUser() { + return getString(ConfVars.ZEPPELIN_INTERPRETER_USER); + } + public Map dumpConfigurations(ZeppelinConfiguration conf, ConfigurationKeyPredicate predicate) { Map configurations = new HashMap<>(); @@ -557,7 +565,9 @@ public static enum ConfVars { ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true), ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true), ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"), - ZEPPELIN_USE_JDBC_ALIAS("zeppelin.use.jdbc.alias", true); + ZEPPELIN_USE_JDBC_ALIAS("zeppelin.use.jdbc.alias", true), + ZEPPELIN_INTERPRETER_RUNAS_USER("zeppelin.interpreter.runas.user", null), + ZEPPELIN_INTERPRETER_USER("zeppelin.interpreter.user", null); private String varName; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 1c7b9d9d420..2a900ed3f30 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -17,37 +17,6 @@ package org.apache.zeppelin.interpreter; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - import com.google.common.base.Preconditions; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -55,12 +24,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.NullArgumentException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.RepositoryException; -import org.sonatype.aether.repository.Authentication; -import org.sonatype.aether.repository.RemoteRepository; - import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.Dependency; @@ -76,6 +39,24 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.RepositoryException; +import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.RemoteRepository; + +import java.io.*; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; /** * Manage interpreters. @@ -1094,14 +1075,15 @@ private String getInterpreterClassFromInterpreterSetting(InterpreterSetting sett return null; } - private Interpreter getInterpreter(String noteId, InterpreterSetting setting, String name) { + private Interpreter getInterpreter(String noteId, InterpreterSetting setting, String name, + String userName) { Preconditions.checkNotNull(noteId, "noteId should be not null"); Preconditions.checkNotNull(setting, "setting should be not null"); Preconditions.checkNotNull(name, "name should be not null"); String className; if (null != (className = getInterpreterClassFromInterpreterSetting(setting, name))) { - List interpreterGroup = createOrGetInterpreterList(noteId, setting); + List interpreterGroup = createOrGetInterpreterList(noteId + userName, setting); for (Interpreter interpreter : interpreterGroup) { if (className.equals(interpreter.getClassName())) { return interpreter; @@ -1111,7 +1093,7 @@ private Interpreter getInterpreter(String noteId, InterpreterSetting setting, St return null; } - public Interpreter getInterpreter(String noteId, String replName) { + public Interpreter getInterpreter(String noteId, String replName, String userName) { List settings = getInterpreterSettings(noteId); InterpreterSetting setting; Interpreter interpreter; @@ -1124,7 +1106,7 @@ public Interpreter getInterpreter(String noteId, String replName) { // get default settings (first available) // TODO(jl): Fix it in case of returning null InterpreterSetting defaultSettings = getDefaultInterpreterSetting(settings); - return createOrGetInterpreterList(noteId, defaultSettings).get(0); + return createOrGetInterpreterList(noteId + userName, defaultSettings).get(0); } String[] replNameSplit = replName.split("\\."); @@ -1137,7 +1119,7 @@ public Interpreter getInterpreter(String noteId, String replName) { setting = getInterpreterSettingByGroup(settings, group); if (null != setting) { - interpreter = getInterpreter(noteId, setting, name); + interpreter = getInterpreter(noteId, setting, name, userName); if (null != interpreter) { return interpreter; @@ -1152,7 +1134,7 @@ public Interpreter getInterpreter(String noteId, String replName) { // TODO(jl): Handle with noteId to support defaultInterpreter per note. setting = getDefaultInterpreterSetting(settings); - interpreter = getInterpreter(noteId, setting, replName); + interpreter = getInterpreter(noteId, setting, replName, userName); if (null != interpreter) { return interpreter; @@ -1163,7 +1145,7 @@ public Interpreter getInterpreter(String noteId, String replName) { setting = getInterpreterSettingByGroup(settings, replName); if (null != setting) { - List interpreters = createOrGetInterpreterList(noteId, setting); + List interpreters = createOrGetInterpreterList(noteId + userName, setting); if (null != interpreters) { return interpreters.get(0); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 9106cf5f17c..d6b17d10868 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -447,7 +447,7 @@ public void runAll() { p.setAuthenticationInfo(authenticationInfo); p.setListener(jobListenerFactory.getParagraphJobListener(this)); - Interpreter intp = factory.getInterpreter(getId(), p.getRequiredReplName()); + Interpreter intp = factory.getInterpreter(getId(), p.getRequiredReplName(), null); intp.getScheduler().submit(p); } @@ -463,11 +463,13 @@ public void run(String paragraphId) { Paragraph p = getParagraph(paragraphId); p.setListener(jobListenerFactory.getParagraphJobListener(this)); String requiredReplName = p.getRequiredReplName(); - Interpreter intp = factory.getInterpreter(getId(), requiredReplName); + Interpreter intp = factory.getInterpreter(getId(), requiredReplName, + getParagraph(paragraphId).getAuthenticationInfo().getUser()); if (intp == null) { // TODO(jongyoul): Make "%jdbc" configurable from JdbcInterpreter - if (conf.getUseJdbcAlias() && null != (intp = factory.getInterpreter(getId(), "jdbc"))) { + if (conf.getUseJdbcAlias() && null != (intp = factory.getInterpreter(getId(), "jdbc", + getParagraph(paragraphId).getAuthenticationInfo().getUser()))) { String pText = p.getText().replaceFirst(requiredReplName, "jdbc(" + requiredReplName + ")"); logger.debug("New paragraph: {}", pText); p.setEffectiveText(pText); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 308cbb05047..b92b6360550 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -201,7 +201,8 @@ public static String getScriptBody(String text) { } public Interpreter getRepl(String name) { - return factory.getInterpreter(note.getId(), name); + return factory.getInterpreter(note.getId(), name, + getAuthenticationInfo().getUser()); } public Interpreter getCurrentRepl() { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index e28416491ee..c19e5344254 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -154,6 +154,6 @@ public void testInterpreterAliases() throws IOException, RepositoryException { add(setting2.getId()); }}); - assertEquals("className1", factory.getInterpreter("note", "test-group1").getClassName()); + assertEquals("className1", factory.getInterpreter("note", "test-group1", null).getClassName()); } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 245089963ea..c2b497a704b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -74,21 +74,21 @@ public void testGetInterpreter() throws IOException { factory.setInterpreters("note", factory.getDefaultInterpreterSettingList()); // when there're no interpreter selection directive - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", null).getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", " ").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", null, null).getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "", null).getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", " ", null).getClassName()); // when group name is omitted - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "mock11").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "mock11", null).getClassName()); // when 'name' is ommitted - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1", null).getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2", null).getClassName()); // when nothing is ommitted - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1.mock1").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "group1.mock11").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2.mock2").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1.mock1", null).getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "group1.mock11", null).getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2.mock2", null).getClassName()); factory.closeNote("note"); } @@ -105,12 +105,12 @@ public void testNoteSession() throws IOException { assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB")); - factory.getInterpreter("noteA", null).open(); - factory.getInterpreter("noteB", null).open(); + factory.getInterpreter("noteA", null, null).open(); + factory.getInterpreter("noteB", null, null).open(); assertTrue( - factory.getInterpreter("noteA", null).getInterpreterGroup().getId().equals( - factory.getInterpreter("noteB", null).getInterpreterGroup().getId())); + factory.getInterpreter("noteA", null, null).getInterpreterGroup().getId().equals( + factory.getInterpreter("noteB", null, null).getInterpreterGroup().getId())); // interpreters are created after accessing it assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); @@ -138,13 +138,13 @@ public void testNotePerInterpreterProcess() throws IOException { assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB")); - factory.getInterpreter("noteA", null).open(); - factory.getInterpreter("noteB", null).open(); + factory.getInterpreter("noteA", null, null).open(); + factory.getInterpreter("noteB", null, null).open(); // per note interpreter process assertFalse( - factory.getInterpreter("noteA", null).getInterpreterGroup().getId().equals( - factory.getInterpreter("noteB", null).getInterpreterGroup().getId())); + factory.getInterpreter("noteA", null, null).getInterpreterGroup().getId().equals( + factory.getInterpreter("noteB", null, null).getInterpreterGroup().getId())); // interpreters are created after accessing it assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java index cff66adc78d..eb63be707eb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java @@ -17,8 +17,6 @@ package org.apache.zeppelin.notebook; -import com.google.common.base.Optional; - import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterFactory; @@ -65,7 +63,7 @@ public class NoteTest { @Test public void runNormalTest() { - when(interpreterFactory.getInterpreter(anyString(), eq("spark"))).thenReturn(interpreter); + when(interpreterFactory.getInterpreter(anyString(), eq("spark"), null)).thenReturn(interpreter); when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%spark sc.version"; @@ -77,15 +75,15 @@ public void runNormalTest() { ArgumentCaptor pCaptor = ArgumentCaptor.forClass(Paragraph.class); verify(scheduler, only()).submit(pCaptor.capture()); - verify(interpreterFactory, only()).getInterpreter(anyString(), eq("spark")); + verify(interpreterFactory, only()).getInterpreter(anyString(), eq("spark"), null); assertEquals("Paragraph text", pText, pCaptor.getValue().getText()); } @Test public void runJdbcTest() { - when(interpreterFactory.getInterpreter(anyString(), eq("mysql"))).thenReturn(null); - when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"))).thenReturn(interpreter); + when(interpreterFactory.getInterpreter(anyString(), eq("mysql"), null)).thenReturn(null); + when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"), null)).thenReturn(interpreter); when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%mysql show databases"; @@ -97,7 +95,7 @@ public void runJdbcTest() { ArgumentCaptor pCaptor = ArgumentCaptor.forClass(Paragraph.class); verify(scheduler, only()).submit(pCaptor.capture()); - verify(interpreterFactory, times(2)).getInterpreter(anyString(), anyString()); + verify(interpreterFactory, times(2)).getInterpreter(anyString(), anyString(), null); assertEquals("Change paragraph text", "%jdbc(mysql) show databases", pCaptor.getValue().getEffectiveText()); assertEquals("Change paragraph text", pText, pCaptor.getValue().getText()); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index bfa97e05de3..8abb108f2db 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -322,11 +322,11 @@ public void testAutoRestartInterpreterAfterSchedule() throws InterruptedExceptio MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock1")).getInnerInterpreter()) + ((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock1", null)).getInnerInterpreter()) .getInnerInterpreter())); MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock2")).getInnerInterpreter()) + ((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock2", null)).getInnerInterpreter()) .getInnerInterpreter())); // wait until interpreters are started diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java index 1f8519cb399..a074cfb4ddb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java @@ -85,7 +85,7 @@ public void effectiveTextTest() { assertEquals("Get right replName", "jdbc", p.getRequiredReplName()); assertEquals("Get right scriptBody", "(h2) show databases", p.getScriptBody()); - when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"))).thenReturn(interpreter); + when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"), null)).thenReturn(interpreter); when(interpreter.getFormType()).thenReturn(Interpreter.FormType.NATIVE); when(note.getId()).thenReturn("noteId"); From 2758b7f8ca3b7aab9f281833ca760f75f98b8e0d Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Tue, 16 Aug 2016 13:08:14 +0530 Subject: [PATCH 03/10] passing user and isUserImpersonate to RemoteInterpreterManagedProcess --- bin/interpreter.sh | 2 +- conf/shiro.ini | 14 ++-- conf/zeppelin-env.sh.template | 3 - .../interpreter/InterpreterGroup.java | 20 ++--- .../interpreter/remote/RemoteInterpreter.java | 47 +++++++----- .../RemoteInterpreterManagedProcess.java | 11 ++- .../remote/RemoteInterpreterProcess.java | 14 ++-- .../RemoteInterpreterRunningProcess.java | 2 +- .../remote/RemoteAngularObjectTest.java | 5 +- .../RemoteInterpreterOutputTestStream.java | 3 +- .../remote/RemoteInterpreterProcessTest.java | 9 +-- .../remote/RemoteInterpreterTest.java | 19 +++-- .../resource/DistributedResourcePoolTest.java | 10 ++- .../scheduler/RemoteSchedulerTest.java | 6 +- .../zeppelin/socket/NotebookServer.java | 20 ++--- .../zeppelin/socket/NotebookServerTest.java | 2 +- .../interpreter-create.html | 12 +++ .../app/interpreter/interpreter.controller.js | 3 + .../src/app/interpreter/interpreter.html | 14 ++++ .../zeppelin/conf/ZeppelinConfiguration.java | 12 +-- .../interpreter/InterpreterFactory.java | 73 +++++++++++-------- .../interpreter/InterpreterOption.java | 9 +++ .../interpreter/InterpreterSetting.java | 38 +++++----- .../org/apache/zeppelin/notebook/Note.java | 52 ++++++------- .../apache/zeppelin/notebook/Notebook.java | 7 +- .../apache/zeppelin/notebook/Paragraph.java | 22 +++--- .../interpreter/InterpreterFactoryTest.java | 8 +- .../notebook/NoteInterpreterLoaderTest.java | 58 +++++++-------- .../apache/zeppelin/notebook/NoteTest.java | 10 +-- .../zeppelin/notebook/NotebookTest.java | 14 ++-- .../zeppelin/notebook/ParagraphTest.java | 2 +- 31 files changed, 293 insertions(+), 228 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 2aa9988caa0..9b5f51ef569 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -23,7 +23,7 @@ function usage() { echo "usage) $0 -p -d -l " } -while getopts "hp:d:l:v:u" o; do +while getopts "hp:d:l:v:u:" o; do case ${o} in h) usage diff --git a/conf/shiro.ini b/conf/shiro.ini index ce8102a2bb8..05cb85569cf 100644 --- a/conf/shiro.ini +++ b/conf/shiro.ini @@ -18,10 +18,10 @@ [users] # List of users with their password allowed to access Zeppelin. # To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections -admin = admin, admin -user1 = user1, role1, role2 -user2 = user2, role3 -user3 = user3, role2 +admin = password1, admin +user1 = password2, role1, role2 +user2 = password3, role3 +user3 = password4, role2 # Sample LDAP configuration, for user Authentication, currently tested for single Realm [main] @@ -67,7 +67,7 @@ admin = * [urls] # This section is used for url-based security. -# You can secure interpreter, configuration and credential information by urls. Comment or uncomment the below urls that you want to hide. +# You can secure interpreter, configuration and credential information by urls. Comment or uncomment the below urls that you want to hide. # anon means the access is anonymous. # authc means Form based Auth Security # To enfore security, comment the line below and uncomment the next one @@ -75,5 +75,5 @@ admin = * #/api/interpreter/** = authc, roles[admin] #/api/configurations/** = authc, roles[admin] #/api/credential/** = authc, roles[admin] -#/** = anon -/** = authc +/** = anon +#/** = authc diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index ee24458acde..14fdd54eeb6 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -37,9 +37,6 @@ # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). -# export ZEPPELIN_INTERPRETER_RUNAS_USER= # set this to true if user wants to run interpreter as different user, and if shiro authentication is enabled then it will run as end web user. -# export ZEPPELIN_INTERPRETER_USER= # user to be run as if shiro authentication is not enbaled and you want to run it as different user than the server. - #### Spark interpreter configuration #### ## Use provided spark installation ## diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java index bc56784b15b..0e6d7fd69c7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java @@ -17,9 +17,6 @@ package org.apache.zeppelin.interpreter; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; - import org.apache.log4j.Logger; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; @@ -27,15 +24,18 @@ import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + /** * InterpreterGroup is list of interpreters in the same interpreter group. * For example spark, pyspark, sql interpreters are in the same 'spark' group * and InterpreterGroup will have reference to these all interpreters. - * + *

* Remember, list of interpreters are dedicated to a note. * (when InterpreterOption.perNoteSession==true) * So InterpreterGroup internally manages map of [noteId, list of interpreters] - * + *

* A InterpreterGroup runs on interpreter process. * And unit of interpreter instantiate, restart, bind, unbind. */ @@ -66,6 +66,7 @@ public static Collection getAll() { /** * Create InterpreterGroup with given id + * * @param id */ public InterpreterGroup(String id) { @@ -83,7 +84,7 @@ public InterpreterGroup() { private static String generateId() { return "InterpreterGroup_" + System.currentTimeMillis() + "_" - + new Random().nextInt(); + + new Random().nextInt(); } public String getId() { @@ -97,6 +98,7 @@ public String getId() { /** * Get combined property of all interpreters in this group + * * @return */ public Properties getProperty() { @@ -131,8 +133,6 @@ public void setRemoteInterpreterProcess(RemoteInterpreterProcess remoteInterpret this.remoteInterpreterProcess = remoteInterpreterProcess; } - - /** * Close all interpreter instances in this group */ @@ -147,6 +147,7 @@ public void close() { /** * Close all interpreter instances in this group for the note + * * @param noteId */ public void close(String noteId) { @@ -188,6 +189,7 @@ public void run() { /** * Destroy all interpreter instances in this group for the note + * * @param noteId */ public void destroy(String noteId) { @@ -215,6 +217,7 @@ public void destroy() { } } + allInterpreterGroups.remove(id); } @@ -246,7 +249,6 @@ public void run() { } - public void setResourcePool(ResourcePool resourcePool) { this.resourcePool = resourcePool; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index db740f4873a..eacbf200b06 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -17,14 +17,14 @@ package org.apache.zeppelin.interpreter.remote; -import java.util.*; - +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import org.apache.thrift.TException; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; -import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.display.Input; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.InterpreterResult.Type; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -36,8 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; +import java.util.*; /** * Proxy for Interpreter instance that runs on separate process @@ -59,20 +58,24 @@ public class RemoteInterpreter extends Interpreter { private int maxPoolSize; private String host; private int port; + private String userName; + private Boolean isUserImpersonate; /** * Remote interpreter and manage interpreter process */ public RemoteInterpreter(Properties property, - String noteId, - String className, - String interpreterRunner, - String interpreterPath, - String localRepoPath, - int connectTimeout, - int maxPoolSize, - RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appListener) { + String noteId, + String className, + String interpreterRunner, + String interpreterPath, + String localRepoPath, + int connectTimeout, + int maxPoolSize, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appListener, + String userName, + Boolean isUserImpersonate) { super(property); this.noteId = noteId; this.className = className; @@ -85,6 +88,8 @@ public RemoteInterpreter(Properties property, this.maxPoolSize = maxPoolSize; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; this.applicationEventListener = appListener; + this.userName = userName; + this.isUserImpersonate = isUserImpersonate; } @@ -100,7 +105,9 @@ public RemoteInterpreter( int connectTimeout, int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appListener) { + ApplicationEventListener appListener, + String userName, + Boolean isUserImpersonate) { super(property); this.noteId = noteId; this.className = className; @@ -111,6 +118,8 @@ public RemoteInterpreter( this.maxPoolSize = maxPoolSize; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; this.applicationEventListener = appListener; + this.userName = userName; + this.isUserImpersonate = isUserImpersonate; } @@ -125,7 +134,9 @@ public RemoteInterpreter( Map env, int connectTimeout, RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appListener) { + ApplicationEventListener appListener, + String userName, + Boolean isUserImpersonate) { super(property); this.className = className; this.noteId = noteId; @@ -138,6 +149,8 @@ public RemoteInterpreter( this.maxPoolSize = 10; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; this.applicationEventListener = appListener; + this.userName = userName; + this.isUserImpersonate = isUserImpersonate; } private Map getEnvFromInterpreterProperty(Properties property) { @@ -205,7 +218,7 @@ public synchronized void init() { RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); final InterpreterGroup interpreterGroup = getInterpreterGroup(); - interpreterProcess.reference(interpreterGroup); + interpreterProcess.reference(interpreterGroup, userName, isUserImpersonate); interpreterProcess.setMaxPoolSize( Math.max(this.maxPoolSize, interpreterProcess.getMaxPoolSize())); String groupId = interpreterGroup.getId(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index 05a43023cc9..e843e73f116 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -25,7 +25,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.io.OutputStream; import java.util.Map; /** @@ -88,7 +87,7 @@ public int getPort() { } @Override - public void start() { + public void start(String userName, Boolean isUserImpersonate) { // start server process try { port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); @@ -101,11 +100,11 @@ public void start() { cmdLine.addArgument(interpreterDir, false); cmdLine.addArgument("-p", false); cmdLine.addArgument(Integer.toString(port), false); + if (isUserImpersonate && !userName.equals("anonymous") ) { + cmdLine.addArgument("-u", false); + cmdLine.addArgument(userName, false); + } cmdLine.addArgument("-l", false); - - - cmdLine.addArgument("-u", false); - cmdLine.addArgument(localRepoDir, false); executor = new DefaultExecutor(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index aef6c2bd809..be1dc3f9006 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -17,18 +17,16 @@ package org.apache.zeppelin.interpreter.remote; import com.google.gson.Gson; -import org.apache.commons.exec.*; +import org.apache.commons.exec.ExecuteWatchdog; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.thrift.TException; import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.Constants; -import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.util.concurrent.atomic.AtomicInteger; -import java.util.Properties; /** * Abstract class for interpreter process @@ -63,7 +61,8 @@ public RemoteInterpreterProcess( public abstract String getHost(); public abstract int getPort(); - public abstract void start(); + + public abstract void start(String userName, Boolean isUserImpersonate); public abstract void stop(); public abstract boolean isRunning(); @@ -71,10 +70,11 @@ public int getConnectTimeout() { return connectTimeout; } - public int reference(InterpreterGroup interpreterGroup) { + public int reference(InterpreterGroup interpreterGroup, String userName, + Boolean isUserImpersonate) { synchronized (referenceCount) { if (!isRunning()) { - start(); + start(userName, isUserImpersonate); } if (clientPool == null) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java index 42e6250e3ca..bb176bea590 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java @@ -51,7 +51,7 @@ public int getPort() { } @Override - public void start() { + public void start(String userName, Boolean isUserImpersonate) { // assume process is externally managed. nothing to do } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index 5def888b637..472fe2a8608 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -74,8 +74,9 @@ public void setUp() throws Exception { env, 10 * 1000, null, - null - ); + null, + null, + false); intpGroup.put("note", new LinkedList()); intpGroup.get("note").add(intp); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java index 74649b1e750..dc88f5e6d70 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java @@ -71,7 +71,8 @@ private RemoteInterpreter createMockInterpreter() { env, 10 * 1000, this, - null); + null, + null, false); intpGroup.get("note").add(intp); intp.setInterpreterGroup(intpGroup); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index 01582821308..b6b41db1315 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -25,7 +25,6 @@ import java.util.Properties; import org.apache.thrift.TException; -import org.apache.thrift.transport.TTransportException; import org.apache.zeppelin.interpreter.Constants; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; @@ -46,8 +45,8 @@ public void testStartStop() { 10 * 1000, null, null); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); - assertEquals(1, rip.reference(intpGroup)); - assertEquals(2, rip.reference(intpGroup)); + assertEquals(1, rip.reference(intpGroup, null, false)); + assertEquals(2, rip.reference(intpGroup, null, false)); assertEquals(true, rip.isRunning()); assertEquals(1, rip.dereference()); assertEquals(true, rip.isRunning()); @@ -61,7 +60,7 @@ public void testClientFactory() throws Exception { RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess( INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap(), mock(RemoteInterpreterEventPoller.class), 10 * 1000); - rip.reference(intpGroup); + rip.reference(intpGroup, null, false); assertEquals(0, rip.getNumActiveClient()); assertEquals(0, rip.getNumIdleClient()); @@ -106,7 +105,7 @@ public void testStartStopRemoteInterpreter() throws TException, InterruptedExcep , 10 * 1000); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); - assertEquals(1, rip.reference(intpGroup)); + assertEquals(1, rip.reference(intpGroup, null, false)); assertEquals(true, rip.isRunning()); } } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index af1c4471a05..b4b559e3517 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -31,7 +31,6 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterEnv; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; @@ -90,7 +89,8 @@ private RemoteInterpreter createMockInterpreterA(Properties p, String noteId) { env, 10 * 1000, null, - null); + null, + null, false); } private RemoteInterpreter createMockInterpreterB(Properties p) { @@ -108,7 +108,8 @@ private RemoteInterpreter createMockInterpreterB(Properties p, String noteId) { env, 10 * 1000, null, - null); + null, + null, false); } @Test @@ -207,7 +208,8 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException env, 10 * 1000, null, - null); + null, + null, false); intpGroup.get("note").add(intpA); @@ -223,7 +225,8 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException env, 10 * 1000, null, - null); + null, + null, false); intpGroup.get("note").add(intpB); intpB.setInterpreterGroup(intpGroup); @@ -687,7 +690,8 @@ public void should_push_local_angular_repo_to_remote() throws Exception { //Given final Client client = Mockito.mock(Client.class); final RemoteInterpreter intr = new RemoteInterpreter(new Properties(), "noteId", - MockInterpreterA.class.getName(), "runner", "path","localRepo", env, 10 * 1000, null, null); + MockInterpreterA.class.getName(), "runner", "path","localRepo", env, 10 * 1000, null, null, + null, false); final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null); registry.add("name", "DuyHai DOAN", "nodeId", "paragraphId"); final InterpreterGroup interpreterGroup = new InterpreterGroup("groupId"); @@ -733,7 +737,8 @@ public void testEnvronmentAndPropertySet() { env, 10 * 1000, null, - null); + null, + null, false); intpGroup.put("note", new LinkedList()); intpGroup.get("note").add(intp); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java index 02dba20c05d..2a69a5e91c6 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java @@ -70,8 +70,9 @@ public void setUp() throws Exception { env, 10 * 1000, null, - null - ); + null, + null, + false); intpGroup1 = new InterpreterGroup("intpGroup1"); intpGroup1.put("note", new LinkedList()); @@ -88,8 +89,9 @@ public void setUp() throws Exception { env, 10 * 1000, null, - null - ); + null, + null, + false); intpGroup2 = new InterpreterGroup("intpGroup2"); intpGroup2.put("note", new LinkedList()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index f17d88d50af..de5561daac4 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -81,7 +81,8 @@ public void test() throws Exception { env, 10 * 1000, this, - null); + null, + null, false); intpGroup.put("note", new LinkedList()); intpGroup.get("note").add(intpA); @@ -170,7 +171,8 @@ public void testAbortOnPending() throws Exception { env, 10 * 1000, this, - null); + null, + null, false); intpGroup.put("note", new LinkedList()); intpGroup.get("note").add(intpA); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 49aafa48d1c..9e5e2d29b6f 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -20,7 +20,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; - import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; @@ -802,6 +801,7 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo String paragraphId = (String) fromMessage.get("paragraphId"); String interpreterGroupId = (String) fromMessage.get("interpreterGroupId"); String varName = (String) fromMessage.get("name"); + AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal); Object varValue = fromMessage.get("value"); AngularObject ao = null; boolean global = false; @@ -811,12 +811,13 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo List settings = notebook.getInterpreterFactory() .getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { - if (setting.getInterpreterGroup(note.id()) == null) { + if (setting.getInterpreterGroup(note.id(), subject.getUser()) == null) { continue; } - if (interpreterGroupId.equals(setting.getInterpreterGroup(note.id()).getId())) { + if (interpreterGroupId.equals(setting.getInterpreterGroup(note.id(), + subject.getUser()).getId())) { AngularObjectRegistry angularObjectRegistry = setting - .getInterpreterGroup(note.id()).getAngularObjectRegistry(); + .getInterpreterGroup(note.id(), subject.getUser()).getAngularObjectRegistry(); // first trying to get local registry ao = angularObjectRegistry.get(varName, noteId, paragraphId); @@ -853,12 +854,13 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo List settings = notebook.getInterpreterFactory() .getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { - if (setting.getInterpreterGroup(n.id()) == null) { + if (setting.getInterpreterGroup(n.id(), subject.getUser()) == null) { continue; } - if (interpreterGroupId.equals(setting.getInterpreterGroup(n.id()).getId())) { + if (interpreterGroupId.equals(setting.getInterpreterGroup(n.id(), subject.getUser()) + .getId())) { AngularObjectRegistry angularObjectRegistry = setting - .getInterpreterGroup(n.id()).getAngularObjectRegistry(); + .getInterpreterGroup(n.id(), subject.getUser()).getAngularObjectRegistry(); this.broadcastExcept( n.id(), new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) @@ -1385,14 +1387,14 @@ private void sendAllAngularObjects(Note note, NotebookSocket conn) throws IOExce } for (InterpreterSetting intpSetting : settings) { - AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.id()) + AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.id(), "anonymous") .getAngularObjectRegistry(); List objects = registry.getAllWithGlobal(note.id()); for (AngularObject object : objects) { conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) .put("angularObject", object) .put("interpreterGroupId", - intpSetting.getInterpreterGroup(note.id()).getId()) + intpSetting.getInterpreterGroup(note.id(), "anonymous").getId()) .put("noteId", note.id()) .put("paragraphId", object.getParagraphId()) )); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java index 01a24e2e1de..6e4cd4783ee 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java @@ -101,7 +101,7 @@ public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() thr List settings = notebook.getInterpreterFactory().getInterpreterSettings(note1.getId()); for (InterpreterSetting setting : settings) { if (setting.getName().equals("md")) { - interpreterGroup = setting.getInterpreterGroup("sharedProcess"); + interpreterGroup = setting.getInterpreterGroup("sharedProcess", "anonymous"); break; } } diff --git a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html index ea5d98fb951..9f8fb0b8bd1 100644 --- a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html +++ b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html @@ -36,6 +36,18 @@

Create new interpreter

Option + +
+
+ + + +
+
+