Merge branch 'master' into doc-formatting-fixes

bin/common.cmd

@@ -71,6 +71,14 @@ if not defined ZEPPELIN_JAVA_OPTS (
     set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM%
 )
 
+if defined ZEPPELIN_JMX_ENABLE (
+  if not defined ZEPPELIN_JMX_PORT (
+    set ZEPPELIN_JMX_PORT="9996"
+  }
+  set JMX_JAVA_OPTS=" -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${ZEPPELIN_JMX_PORT} -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+  set ZEPPELIN_JAVA_OPTS=%JMX_JAVA_OPTS% %ZEPPELIN_JAVA_OPTS
+)
+
 if not defined JAVA_OPTS (
     set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
 ) else (

bin/common.sh

@@ -121,6 +121,18 @@ JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPEL
 JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
 export JAVA_OPTS
 
+if [[ x"${ZEPPELIN_JMX_ENABLE}" == x"true" ]]; then
+  if [[ -z "${ZEPPELIN_JMX_PORT}" ]]; then
+    ZEPPELIN_JMX_PORT="9996"
+  fi
+  JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote"
+  JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.port=${ZEPPELIN_JMX_PORT}"
+  JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.authenticate=false"
+  JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.ssl=false"
+  JAVA_OPTS="${JMX_JAVA_OPTS} ${JAVA_OPTS}"
+fi
+export JAVA_OPTS
+
 JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
 if [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
     JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"

bin/interpreter.sh

@@ -24,7 +24,7 @@ function usage() {
     echo "usage) $0 -p <port> -r <intp_port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
 }
 
-while getopts "hc:p:r:d:l:v:u:g:" o; do
+while getopts "hc:p:r:i:d:l:v:u:g:" o; do
     case ${o} in
         h)
             usage
@@ -42,6 +42,9 @@ while getopts "hc:p:r:d:l:v:u:g:" o; do
         r)
             INTP_PORT=${OPTARG} # This will be used for interpreter process port
             ;;
+        i)
+            INTP_GROUP_ID=${OPTARG} # This will be used for interpreter group id
+            ;;
         l)
             LOCAL_INTERPRETER_REPO=${OPTARG}
             ;;
@@ -218,9 +221,9 @@ if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
 fi
 
 if [[ -n "${SPARK_SUBMIT}" ]]; then
-    INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_PORT}`
+    INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}`
 else
-    INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_PORT}`
+    INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}`
 fi
 
 

cassandra/src/main/resources/interpreter-setting.json

@@ -224,7 +224,7 @@
       "cassandra.ssl.enabled": {
         "envName": null,
         "propertyName": "cassandra.ssl.enabled",
-        "defaultValue": "false",
+        "defaultValue": false,
         "description": "Cassandra SSL",
         "type": "checkbox"
       },

conf/zeppelin-env.cmd.template

@@ -22,6 +22,8 @@ REM set ZEPPELIN_JAVA_OPTS      		REM Additional jvm options. for example, set Z
 REM set ZEPPELIN_MEM            		REM Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
 REM set ZEPPELIN_INTP_MEM       		REM zeppelin interpreter process jvm mem options. Default -Xmx1024m -Xms1024m -XX:MaxPermSize=512m
 REM set ZEPPELIN_INTP_JAVA_OPTS 		REM zeppelin interpreter process jvm options.
+REM set ZEPPELIN_JMX_ENABLE     		REM Enable JMX feature by defining it like "true"
+REM set ZEPPELIN_JMX_PORT       		REM Port number which JMX uses. Default: "9996"
 
 REM set ZEPPELIN_LOG_DIR        		REM Where log files are stored.  PWD by default.
 REM set ZEPPELIN_PID_DIR        		REM The pid files are stored. /tmp by default.

conf/zeppelin-env.sh.template

@@ -23,6 +23,8 @@
 # export ZEPPELIN_INTP_MEM       		# zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
 # export ZEPPELIN_INTP_JAVA_OPTS 		# zeppelin interpreter process jvm options.
 # export ZEPPELIN_SSL_PORT       		# ssl port (used when ssl environment variable is set to true)
+# export ZEPPELIN_JMX_ENABLE    		# Enable JMX feature by defining "true"
+# export ZEPPELIN_JMX_PORT      		# Port number which JMX uses. Default: "9996"
 
 # export ZEPPELIN_LOG_DIR        		# Where log files are stored.  PWD by default.
 # export ZEPPELIN_PID_DIR        		# The pid files are stored. ${ZEPPELIN_HOME}/run by default.

conf/zeppelin-site.xml.template

@@ -67,6 +67,12 @@
   <description>hide homescreen notebook from list when this value set to true</description>
 </property>
 
+<property>
+  <name>zeppelin.notebook.collaborative.mode.enable</name>
+  <value>true</value>
+  <description>Enable collaborative mode</description>
+</property>
+
 <!-- Google Cloud Storage notebook storage -->
 <!--
 <property>

docs/interpreter/spark.md

@@ -205,8 +205,7 @@ You can either specify them in `zeppelin-env.sh`, or in interpreter setting page
 in interpreter setting page means you can use multiple versions of `spark` & `hadoop` in one zeppelin instance.
 
 ### 4. New Version of SparkInterpreter
-There's one new version of SparkInterpreter starting with better spark support and code completion from Zeppelin 0.8.0, by default we still use the old version of SparkInterpreter.
-If you want to use the new one, you can configure `zeppelin.spark.useNew` as `true` in its interpreter setting.
+There's one new version of SparkInterpreter with better spark support and code completion starting from Zeppelin 0.8.0. We enable it by default, but user can still use the old version of SparkInterpreter by setting `zeppelin.spark.useNew` as `false` in its interpreter setting.
 
 ## SparkContext, SQLContext, SparkSession, ZeppelinContext
 SparkContext, SQLContext and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext` and `z`, respectively, in Scala, Python and R environments.

docs/setup/operation/configuration.md

@@ -101,6 +101,12 @@ If both are defined, then the **environment variables** will take priority.
     <td>/</td>
     <td>Context path of the web application</td>
   </tr>
+  <tr>
+    <td><h6 class="properties">ZEPPELIN_NOTEBOOK_COLLABORATIVE_MODE_ENABLE</h6></td>
+    <td><h6 class="properties">zeppelin.notebook.collaborative.mode.enable</h6></td>
+    <td>true</td>
+    <td>Enable basic opportunity for collaborative editing. Does not change the logic of operation if the note is used by one person.</td>
+  </tr>
   <tr>
     <td><h6 class="properties">ZEPPELIN_SSL</h6></td>
     <td><h6 class="properties">zeppelin.ssl</h6></td>

docs/setup/security/shiro_authentication.md

@@ -104,6 +104,9 @@ To learn more about Apache Shiro Realm, please check [this documentation](http:/
 
 We also provide community custom Realms.
 
+**Note**: When using any of the below realms the default 
+      password-based (IniRealm) authentication needs to be disabled.
+
 ### Active Directory
 
 ```
@@ -182,6 +185,17 @@ securityManager.sessionManager = $sessionManager
 securityManager.realms = $ldapRealm
 ```
 
+Also instead of specifying systemPassword in clear text in `shiro.ini` administrator can choose to specify the same in "hadoop credential". 
+Create a keystore file using the hadoop credential command line:
+``` 
+hadoop credential create ldapRealm.systemPassword -provider jceks://file/user/zeppelin/conf/zeppelin.jceks
+```
+
+Add the following line in the `shiro.ini` file:
+``` 
+ldapRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/conf/zeppelin.jceks
+```
+
 ### PAM
 [PAM](https://en.wikipedia.org/wiki/Pluggable_authentication_module) authentication support allows the reuse of existing authentication
 moduls on the host where Zeppelin is running. On a typical system modules are configured per service for example sshd, passwd, etc. under `/etc/pam.d/`. You can
@@ -274,15 +288,16 @@ By default, Shiro will allow access to a URL if only user is part of "**all the
 /api/interpreter/** = authc, roles[admin, role1]
 ```
 
-If there is a need that user with "**any of the defined roles**" should be allowed, then following Shiro configuration can be used:
+### Apply multiple roles or user in Shiro configuration
+If there is a need that user with "**any of the defined roles or user itself**" should be allowed, then following Shiro configuration can be used:
 
 ```
 [main]
-anyofroles = org.apache.zeppelin.utils.AnyOfRolesAuthorizationFilter
+anyofrolesuser = org.apache.zeppelin.utils.AnyOfRolesUserAuthorizationFilter
 
 [urls]
 
-/api/interpreter/** = authc, anyofroles[admin, role1]
+/api/interpreter/** = authc, anyofrolesuser[admin, user1]
 /api/configurations/** = authc, roles[admin]
 /api/credential/** = authc, roles[admin]
 ```

docs/usage/other_features/cron_scheduler.md

@@ -45,7 +45,7 @@ You can set the cron schedule by filling in this form. Please see [Cron Trigger
 
 You can set the cron executing user by filling in this form and press the enter key.
 
-### auto-restart interpreter on cron execution
+### After execution stop the interpreter
 
 When this checkbox is set to "on", the interpreters which are binded to the notebook are stopped automatically after the cron execution. This feature is useful if you want to release the interpreter resources after the cron execution.
 

elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java

@@ -161,9 +161,11 @@ public static void clean() {
   }
 
   private InterpreterContext buildContext(String noteAndParagraphId) {
-    final AngularObjectRegistry angularObjReg = new AngularObjectRegistry("elasticsearch", null);
-    return new InterpreterContext(noteAndParagraphId, noteAndParagraphId, null, null, null, null,
-        null, null, null, angularObjReg , null, null, null);
+    return InterpreterContext.builder()
+        .setNoteId(noteAndParagraphId)
+        .setParagraphId(noteAndParagraphId)
+        .setAngularObjectRegistry(new AngularObjectRegistry("elasticsearch", null))
+        .build();
   }
 
   @Theory

flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java

@@ -17,19 +17,18 @@
  */
 package org.apache.zeppelin.flink;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.util.Arrays;
 import java.util.Properties;
 
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
 
 public class FlinkInterpreterTest {
 
@@ -41,8 +40,7 @@ public static void setUp() {
     Properties p = new Properties();
     flink = new FlinkInterpreter(p);
     flink.open();
-    context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null,
-            null, null, null);
+    context = InterpreterContext.builder().build();
   }
 
   @AfterClass