From e4054c117380317a33d153117c6e30e3a6e1aaf7 Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Sun, 31 May 2015 09:37:53 +0900 Subject: [PATCH 1/2] Get useHiveContext property from environment variable --- .../java/org/apache/zeppelin/spark/SparkInterpreter.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index c875e8557b8..845d5721b0c 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -105,8 +105,10 @@ public class SparkInterpreter extends Interpreter { getSystemDefault("SPARK_YARN_JAR", "spark.yarn.jar", ""), "The location of the Spark jar file. If you use yarn as a cluster, " + "we should set this value") - .add("zeppelin.spark.useHiveContext", "true", - "Use HiveContext instead of SQLContext if it is true.") + .add("zeppelin.spark.useHiveContext", + getSystemDefault("ZEPPELIN_SPARK_USEHIVECONTEXT", + "zeppelin.spark.useHiveContext", "true"), + "Use HiveContext instead of SQLContext if it is true.") .add("zeppelin.spark.maxResult", "1000", "Max number of SparkSQL result to display.") .add("args", "", "spark commandline args").build()); From 8bc0698882957f056db26f88d1bc2c75f9663ff8 Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Tue, 2 Jun 2015 18:20:17 +0900 Subject: [PATCH 2/2] Get interpreter properties from environment variable --- conf/zeppelin-env.sh.template | 10 +++++++--- .../org/apache/zeppelin/spark/SparkInterpreter.java | 4 +++- .../org/apache/zeppelin/spark/SparkSqlInterpreter.java | 9 +++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 62aa95a54a6..43d302bbb3d 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -17,18 +17,22 @@ # # export JAVA_HOME= -# export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode +# export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode. # export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16" # export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m -# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Defualt = ZEPPELIN_MEM +# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM # export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS # export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default. # export ZEPPELIN_PID_DIR # The pid files are stored. /tmp by default. # export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved -# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default +# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default. # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. +# export ZEPPELIN_SPARK_USEHIVECONTEXT # Use HiveContext instead of SQLContext if set true. true by default. +# export ZEPPELIN_SPARK_CONCURRENTSQL # Execute multiple SQL concurrently if set true. false by default. +# export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default. + # Options read in YARN client mode # export HADOOP_CONF_DIR # yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR. diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 845d5721b0c..b68fd2f2125 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -109,7 +109,9 @@ public class SparkInterpreter extends Interpreter { getSystemDefault("ZEPPELIN_SPARK_USEHIVECONTEXT", "zeppelin.spark.useHiveContext", "true"), "Use HiveContext instead of SQLContext if it is true.") - .add("zeppelin.spark.maxResult", "1000", "Max number of SparkSQL result to display.") + .add("zeppelin.spark.maxResult", + getSystemDefault("ZEPPELIN_SPARK_MAXRESULT", "zeppelin.spark.maxResult", "1000"), + "Max number of SparkSQL result to display.") .add("args", "", "spark commandline args").build()); } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index 618579d0eaa..4c6499c05bd 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -66,8 +66,13 @@ public class SparkSqlInterpreter extends Interpreter { "spark", SparkSqlInterpreter.class.getName(), new InterpreterPropertyBuilder() - .add("zeppelin.spark.maxResult", "10000", "Max number of SparkSQL result to display.") - .add("zeppelin.spark.concurrentSQL", "false", + .add("zeppelin.spark.maxResult", + SparkInterpreter.getSystemDefault("ZEPPELIN_SPARK_MAXRESULT", + "zeppelin.spark.maxResult", "1000"), + "Max number of SparkSQL result to display.") + .add("zeppelin.spark.concurrentSQL", + SparkInterpreter.getSystemDefault("ZEPPELIN_SPARK_CONCURRENTSQL", + "zeppelin.spark.concurrentSQL", "false"), "Execute multiple SQL concurrently if set true.") .build()); }