Merge branch 'master' into py4jPythonInterpreter

.travis.yml

@@ -25,6 +25,7 @@ cache:
     - ${HOME}/R
     -  zeppelin-web/node
     -  zeppelin-web/node_modules
+    -  zeppelin-web/bower_components
 
 addons:
   apt:
@@ -70,6 +71,13 @@ matrix:
       env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
 
 before_install:
+  # check files included in commit range, clear bower_components if a bower.json file has changed.
+  # bower cache clearing can also be forced by putting "bower clear" or "clear bower" in a commit message
+  - changedfiles=$(git diff --name-only $TRAVIS_COMMIT_RANGE)
+  - echo $changedfiles
+  - hasbowerchanged=$(echo $changedfiles | grep -c "bower.json" || true);
+  - clearcache=$(git log $TRAVIS_COMMIT_RANGE | grep -c -E "clear bower|bower clear" || true)
+  - if [ "$hasbowerchanged" -gt 0 ] || [ "$clearcache" -gt 0 ]; then echo "Clearing bower_components cache"; rm -r zeppelin-web/bower_components; npm cache clear; else echo "Using cached bower_components."; fi
   - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
   - ./testing/install_external_dependencies.sh
   - ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true
@@ -110,4 +118,4 @@ after_failure:
   - cat livy/target/tmp/livy-int-test/*/output.log
   - ls -R livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
   - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
-  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr
+  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr

bin/interpreter.sh

@@ -20,10 +20,10 @@ bin=$(dirname "${BASH_SOURCE-$0}")
 bin=$(cd "${bin}">/dev/null; pwd)
 
 function usage() {
-    echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load>"
+    echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
 }
 
-while getopts "hp:d:l:v:u:" o; do
+while getopts "hp:d:l:v:u:n:" o; do
     case ${o} in
         h)
             usage
@@ -50,6 +50,9 @@ while getopts "hp:d:l:v:u:" o; do
               ZEPPELIN_IMPERSONATE_RUN_CMD=$(eval "echo ${ZEPPELIN_IMPERSONATE_CMD} ")
             fi
             ;;
+        g)
+            INTERPRETER_GROUP_NAME=${OPTARG}
+            ;;
         esac
 done
 
@@ -86,6 +89,9 @@ ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
 INTERPRETER_ID=$(basename "${INTERPRETER_DIR}")
 ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-interpreter-${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid"
 ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-"
+if [[ ! -z "$INTERPRETER_GROUP_NAME" ]]; then
+    ZEPPELIN_LOGFILE+="${INTERPRETER_GROUP_NAME}-"
+fi
 if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
     ZEPPELIN_LOGFILE+="${ZEPPELIN_IMPERSONATE_USER}-"
 fi

docs/install/build.md

@@ -210,7 +210,7 @@ mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests
 Ignite Interpreter
 
 ```bash
-mvn clean package -Dignite.version=1.8.0 -DskipTests
+mvn clean package -Dignite.version=1.9.0 -DskipTests
 ```
 
 Scalding Interpreter

docs/interpreter/jdbc.md

@@ -118,6 +118,11 @@ The JDBC interpreter properties are defined by default like below.
     <td>gpadmin</td>
     <td>The JDBC user name</td>
   </tr>
+  <tr>
+    <td>default.precode</td>
+    <td></td>
+    <td>Some SQL which executes while opening connection</td>
+  </tr>
 </table>
 
 If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.
@@ -167,10 +172,6 @@ There are more JDBC interpreter properties you can specify like below.
     <td>default.jceks.credentialKey</td>
     <td>jceks credential key</td>
   </tr>
-  <tr>
-    <td>zeppelin.jdbc.precode</td>
-    <td>Some SQL which executes while opening connection</td>
-  </tr>
 </table>
 
 You can also add more properties by using this [method](http://docs.oracle.com/javase/7/docs/api/java/sql/DriverManager.html#getConnection%28java.lang.String,%20java.util.Properties%29).
@@ -221,6 +222,75 @@ SELECT name, country, performer
 FROM demo.performers
 WHERE name='{{"{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia"}}}}'
 ```
+### Usage *precode*
+You can set *precode* for each data source. Code runs once while opening the connection.
+
+##### Properties
+An example settings of interpreter for the two data sources, each of which has its *precode* parameter.
+
+<table class="table-configuration">
+  <tr>
+    <th>Property Name</th>
+    <th>Value</th>
+  </tr>
+  <tr>
+    <td>default.driver</td>
+    <td>org.postgresql.Driver</td>
+  </tr>
+  <tr>
+    <td>default.password</td>
+    <td>1</td>
+  </tr>
+  <tr>
+    <td>default.url</td>
+    <td>jdbc:postgresql://localhost:5432/</td>
+  </tr>
+  <tr>
+    <td>default.user</td>
+    <td>postgres</td>
+  </tr>
+  <tr>
+    <td>default.precode</td>
+    <td>set search_path='test_path'</td>
+  </tr>
+  <tr>
+    <td>mysql.driver</td>
+    <td>com.mysql.jdbc.Driver</td>
+  </tr>
+  <tr>
+    <td>mysql.password</td>
+    <td>1</td>
+  </tr>
+  <tr>
+    <td>mysql.url</td>
+    <td>jdbc:mysql://localhost:3306/</td>
+  </tr>
+  <tr>
+    <td>mysql.user</td>
+    <td>root</td>
+  </tr>
+  <tr>
+    <td>mysql.precode</td>
+    <td>set @v=12</td>
+  </tr>
+</table>
+
+##### Usage
+Test of execution *precode* for each data source. 
+
+```sql
+%jdbc
+show search_path
+```
+Returns value of `search_path` which is set in the *default.precode*.
+
+
+```sql
+%jdbc(mysql)
+select @v
+```
+Returns value of `v` which is set in the *mysql.precode*.
+
 
 ## Examples
 Here are some examples you can refer to. Including the below connectors, you can connect every databases as long as it can be configured with it's JDBC driver.

ignite/pom.xml

@@ -32,7 +32,7 @@
   <name>Zeppelin: Apache Ignite interpreter</name>
 
   <properties>
-    <ignite.version>1.8.0</ignite.version>
+    <ignite.version>1.9.0</ignite.version>
   </properties>
 
   <dependencies>

jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java

@@ -101,9 +101,10 @@ public class JDBCInterpreter extends Interpreter {
   static final String URL_KEY = "url";
   static final String USER_KEY = "user";
   static final String PASSWORD_KEY = "password";
+  static final String PRECODE_KEY = "precode";
   static final String JDBC_JCEKS_FILE = "jceks.file";
   static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey";
-  static final String ZEPPELIN_JDBC_PRECODE_KEY = "zeppelin.jdbc.precode";
+  static final String PRECODE_KEY_TEMPLATE = "%s.precode";
   static final String DOT = ".";
 
   private static final char WHITESPACE = ' ';
@@ -118,6 +119,7 @@ public class JDBCInterpreter extends Interpreter {
   static final String DEFAULT_URL = DEFAULT_KEY + DOT + URL_KEY;
   static final String DEFAULT_USER = DEFAULT_KEY + DOT + USER_KEY;
   static final String DEFAULT_PASSWORD = DEFAULT_KEY + DOT + PASSWORD_KEY;
+  static final String DEFAULT_PRECODE = DEFAULT_KEY + DOT + PRECODE_KEY;
 
   static final String EMPTY_COLUMN_VALUE = "";
 
@@ -342,7 +344,7 @@ private Connection getConnectionFromPool(String url, String user, String propert
     if (!getJDBCConfiguration(user).isConnectionInDBDriverPool(propertyKey)) {
       createConnectionPool(url, user, propertyKey, properties);
       try (Connection connection = DriverManager.getConnection(jdbcDriver)) {
-        executePrecode(connection);
+        executePrecode(connection, propertyKey);
       }
     }
     return DriverManager.getConnection(jdbcDriver);
@@ -548,8 +550,8 @@ protected ArrayList<String> splitSqlQueries(String sql) {
     return queries;
   }
 
-  private void executePrecode(Connection connection) throws SQLException {
-    String precode = getProperty(ZEPPELIN_JDBC_PRECODE_KEY);
+  private void executePrecode(Connection connection, String propertyKey) throws SQLException {
+    String precode = getProperty(String.format(PRECODE_KEY_TEMPLATE, propertyKey));
     if (StringUtils.isNotBlank(precode)) {
       precode = StringUtils.trim(precode);
       logger.info("Run SQL precode '{}'", precode);

jdbc/src/main/resources/interpreter-setting.json

@@ -28,6 +28,12 @@
         "defaultValue": "org.postgresql.Driver",
         "description": "JDBC Driver Name"
       },
+      "default.precode": {
+        "envName": null,
+        "propertyName": "zeppelin.jdbc.precode",
+        "defaultValue": "",
+        "description": "SQL which executes while opening connection"
+      },
       "common.max_count": {
         "envName": null,
         "propertyName": "common.max_count",
@@ -63,12 +69,6 @@
         "propertyName": "zeppelin.jdbc.principal",
         "defaultValue": "",
         "description": "Kerberos principal"
-      },
-      "zeppelin.jdbc.precode": {
-        "envName": null,
-        "propertyName": "zeppelin.jdbc.precode",
-        "defaultValue": "",
-        "description": "SQL which executes while opening connection"
       }
     },
     "editor": {

jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java

@@ -19,6 +19,8 @@
 import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
 import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
 import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
+import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PRECODE;
+import static org.apache.zeppelin.jdbc.JDBCInterpreter.PRECODE_KEY_TEMPLATE;
 import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
 import static org.junit.Assert.*;
 
@@ -44,8 +46,6 @@
 
 import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
 
-import static org.apache.zeppelin.jdbc.JDBCInterpreter.ZEPPELIN_JDBC_PRECODE_KEY;
-
 /**
  * JDBC interpreter unit tests
  */
@@ -397,7 +397,7 @@ public void testPrecode() throws SQLException, IOException {
     properties.setProperty("default.url", getJdbcConnection());
     properties.setProperty("default.user", "");
     properties.setProperty("default.password", "");
-    properties.setProperty(ZEPPELIN_JDBC_PRECODE_KEY, "SET @testVariable=1");
+    properties.setProperty(DEFAULT_PRECODE, "SET @testVariable=1");
     JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
     jdbcInterpreter.open();
 
@@ -417,7 +417,7 @@ public void testIncorrectPrecode() throws SQLException, IOException {
     properties.setProperty("default.url", getJdbcConnection());
     properties.setProperty("default.user", "");
     properties.setProperty("default.password", "");
-    properties.setProperty(ZEPPELIN_JDBC_PRECODE_KEY, "incorrect command");
+    properties.setProperty(DEFAULT_PRECODE, "incorrect command");
     JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
     jdbcInterpreter.open();
 
@@ -428,4 +428,24 @@ public void testIncorrectPrecode() throws SQLException, IOException {
     assertEquals(InterpreterResult.Code.ERROR, interpreterResult.code());
     assertEquals(InterpreterResult.Type.TEXT, interpreterResult.message().get(0).getType());
   }
- }
+
+  @Test
+  public void testPrecodeWithAnotherPrefix() throws SQLException, IOException {
+    Properties properties = new Properties();
+    properties.setProperty("anotherPrefix.driver", "org.h2.Driver");
+    properties.setProperty("anotherPrefix.url", getJdbcConnection());
+    properties.setProperty("anotherPrefix.user", "");
+    properties.setProperty("anotherPrefix.password", "");
+    properties.setProperty(String.format(PRECODE_KEY_TEMPLATE, "anotherPrefix"), "SET @testVariable=2");
+    JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
+    jdbcInterpreter.open();
+
+    String sqlQuery = "(anotherPrefix) select @testVariable";
+
+    InterpreterResult interpreterResult = jdbcInterpreter.interpret(sqlQuery, interpreterContext);
+
+    assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
+    assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
+    assertEquals("@TESTVARIABLE\n2\n", interpreterResult.message().get(0).getData());
+  }
+}

livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java

@@ -55,7 +55,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
   private int pullStatusInterval;
   protected boolean displayAppInfo;
   private AtomicBoolean sessionExpired = new AtomicBoolean(false);
-  private LivyVersion livyVersion;
+  protected LivyVersion livyVersion;
 
   // keep tracking the mapping between paragraphId and statementId, so that we can cancel the
   // statement after we execute it.