From 08251226203107303785ffde458b074d47b13a6f Mon Sep 17 00:00:00 2001 From: mebelousov Date: Tue, 23 Jan 2018 15:39:00 +0300 Subject: [PATCH 001/386] [ZEPPELIN-3187] Remove doubles of settings in zeppelin-site.xml.template ### What is this PR for? Remove doubles in zeppelin-site.xml.template ### What type of PR is it? [Bug Fix ] ### What is the Jira issue? * [ZEPPELIN-3187] ### How should this be tested? * No need ### Questions: * Does the licenses files need update? N/A * Is there breaking changes for older versions? N/A * Does this needs documentation? N/A Author: mebelousov Closes #2741 from mebelousov/ZEPPELIN-3187 and squashes the following commits: 44d6d08 [mebelousov] Remove doubles of settings --- conf/zeppelin-site.xml.template | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index d566a717884..33aa8acf6da 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -427,13 +427,13 @@ zeppelin.interpreter.lifecyclemanager.timeout.checkinterval 60000 - milliseconds of the interval to checking whether interpreter is time out + Milliseconds of the interval to checking whether interpreter is time out zeppelin.interpreter.lifecyclemanager.timeout.threshold 3600000 - milliseconds of the interpreter timeout threshold, by default it is 1 hour + Milliseconds of the interpreter timeout threshold, by default it is 1 hour + - - - - - - - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-groovy jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Groovy interpreter diff --git a/hbase/pom.xml b/hbase/pom.xml index 8f27631ea2d..46886fd424b 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -22,14 +22,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-hbase jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: HBase interpreter diff --git a/helium-dev/pom.xml b/helium-dev/pom.xml index 8514946d76b..77c59791649 100644 --- a/helium-dev/pom.xml +++ b/helium-dev/pom.xml @@ -24,13 +24,13 @@ org.apache.zeppelin interpreter-parent - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin helium-dev - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Helium development interpreter diff --git a/ignite/pom.xml b/ignite/pom.xml index 5ce6e37296c..cd778b8c11a 100644 --- a/ignite/pom.xml +++ b/ignite/pom.xml @@ -22,13 +22,13 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent zeppelin-ignite_2.10 jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Apache Ignite interpreter diff --git a/interpreter-parent/pom.xml b/interpreter-parent/pom.xml index fc924c19a55..cbfedd5d474 100644 --- a/interpreter-parent/pom.xml +++ b/interpreter-parent/pom.xml @@ -22,14 +22,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin interpreter-parent pom - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Interpreter Parent diff --git a/jdbc/pom.xml b/jdbc/pom.xml index beec50c0ad6..2f8976a9d07 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -23,14 +23,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-jdbc jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: JDBC interpreter diff --git a/kylin/pom.xml b/kylin/pom.xml index b70facbcc12..6d78fdfc52c 100644 --- a/kylin/pom.xml +++ b/kylin/pom.xml @@ -23,7 +23,7 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent 4.0.0 @@ -31,7 +31,7 @@ org.apache.zeppelin zeppelin-kylin jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Kylin interpreter diff --git a/lens/pom.xml b/lens/pom.xml index ecc0a12ed6f..828e8f3e9b4 100644 --- a/lens/pom.xml +++ b/lens/pom.xml @@ -22,14 +22,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-lens jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Lens interpreter diff --git a/livy/pom.xml b/livy/pom.xml index 0ec174a9c4f..1c9d8fb8efb 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -24,14 +24,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-livy jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Livy interpreter diff --git a/markdown/pom.xml b/markdown/pom.xml index ca954498075..4feb3bca63d 100644 --- a/markdown/pom.xml +++ b/markdown/pom.xml @@ -22,14 +22,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-markdown jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Markdown interpreter diff --git a/neo4j/pom.xml b/neo4j/pom.xml index 298726fb82b..2bf9e9a36bb 100644 --- a/neo4j/pom.xml +++ b/neo4j/pom.xml @@ -22,14 +22,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-neo4j jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Neo4j interpreter diff --git a/pig/pom.xml b/pig/pom.xml index 4a56a342909..4553b5cf5cd 100644 --- a/pig/pom.xml +++ b/pig/pom.xml @@ -24,14 +24,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-pig jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Apache Pig Interpreter Zeppelin interpreter for Apache Pig http://zeppelin.apache.org diff --git a/pom.xml b/pom.xml index bb1a1e2d68b..880bea6365f 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ org.apache.zeppelin zeppelin pom - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin Zeppelin project http://zeppelin.apache.org diff --git a/python/pom.xml b/python/pom.xml index a906b5d5f50..3ce47b06374 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -22,14 +22,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-python jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Python interpreter diff --git a/r/pom.xml b/r/pom.xml index 4c1b218335d..8c80b3420e8 100644 --- a/r/pom.xml +++ b/r/pom.xml @@ -23,7 +23,7 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. diff --git a/scalding/pom.xml b/scalding/pom.xml index 763afe0bb41..ec7fe40ba5b 100644 --- a/scalding/pom.xml +++ b/scalding/pom.xml @@ -22,14 +22,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-scalding_2.10 jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Scalding interpreter diff --git a/scio/pom.xml b/scio/pom.xml index 27ccb0cc7b8..a62ff2b83e4 100644 --- a/scio/pom.xml +++ b/scio/pom.xml @@ -22,14 +22,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-scio_2.10 jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Scio Zeppelin Scio support diff --git a/scripts/docker/zeppelin/bin/Dockerfile b/scripts/docker/zeppelin/bin/Dockerfile index da5a67764d6..4f6166a2688 100644 --- a/scripts/docker/zeppelin/bin/Dockerfile +++ b/scripts/docker/zeppelin/bin/Dockerfile @@ -17,7 +17,7 @@ FROM ubuntu:16.04 MAINTAINER Apache Software Foundation # `Z_VERSION` will be updated by `dev/change_zeppelin_version.sh` -ENV Z_VERSION="0.8.0-SNAPSHOT" +ENV Z_VERSION="0.9.0-SNAPSHOT" ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \ Z_HOME="/zeppelin" \ LANG=en_US.UTF-8 \ diff --git a/shell/pom.xml b/shell/pom.xml index 56714f5b159..6a7fda9ed01 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -22,14 +22,14 @@ interpreter-parent org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT ../interpreter-parent org.apache.zeppelin zeppelin-shell jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Shell interpreter diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml index b7904c091fa..15138cd593c 100644 --- a/spark-dependencies/pom.xml +++ b/spark-dependencies/pom.xml @@ -23,14 +23,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-spark-dependencies_2.10 jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Spark dependencies Zeppelin spark support diff --git a/spark/pom.xml b/spark/pom.xml index 71110e30ace..1972f26d383 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -22,14 +22,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-spark_2.10 jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Spark Zeppelin spark support diff --git a/zeppelin-display/pom.xml b/zeppelin-display/pom.xml index 4058aefbf11..c6edd95e42f 100644 --- a/zeppelin-display/pom.xml +++ b/zeppelin-display/pom.xml @@ -22,14 +22,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-display_2.10 jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Display system apis diff --git a/zeppelin-distribution/pom.xml b/zeppelin-distribution/pom.xml index ed05c9383be..84a19b54fd4 100644 --- a/zeppelin-distribution/pom.xml +++ b/zeppelin-distribution/pom.xml @@ -23,7 +23,7 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. diff --git a/zeppelin-examples/pom.xml b/zeppelin-examples/pom.xml index e9f04731b80..bfbffa08693 100644 --- a/zeppelin-examples/pom.xml +++ b/zeppelin-examples/pom.xml @@ -22,14 +22,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-examples pom - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Examples Zeppelin examples diff --git a/zeppelin-examples/zeppelin-example-clock/pom.xml b/zeppelin-examples/zeppelin-example-clock/pom.xml index d4fed2171fa..6b24987cbc8 100644 --- a/zeppelin-examples/zeppelin-example-clock/pom.xml +++ b/zeppelin-examples/zeppelin-example-clock/pom.xml @@ -22,14 +22,14 @@ zeppelin-examples org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-example-clock jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Example application - Clock diff --git a/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json index 3db70e93960..34701ac0ec7 100644 --- a/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json +++ b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json @@ -18,7 +18,7 @@ "type" : "APPLICATION", "name" : "zeppelin.clock", "description" : "Clock (example)", - "artifact" : "zeppelin-examples/zeppelin-example-clock/target/zeppelin-example-clock-0.8.0-SNAPSHOT.jar", + "artifact" : "zeppelin-examples/zeppelin-example-clock/target/zeppelin-example-clock-0.9.0-SNAPSHOT.jar", "className" : "org.apache.zeppelin.example.app.clock.Clock", "resources" : [[":java.util.Date"]], "license" : "Apache-2.0", diff --git a/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml b/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml index 8e08c4a3af2..51e87043e16 100644 --- a/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml +++ b/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml @@ -22,14 +22,14 @@ zeppelin-examples org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-example-horizontalbar jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Example application - Horizontal Bar chart diff --git a/zeppelin-examples/zeppelin-example-spell-echo/pom.xml b/zeppelin-examples/zeppelin-example-spell-echo/pom.xml index 348abd20354..d3828494868 100644 --- a/zeppelin-examples/zeppelin-example-spell-echo/pom.xml +++ b/zeppelin-examples/zeppelin-example-spell-echo/pom.xml @@ -22,14 +22,14 @@ zeppelin-examples org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-example-spell-echo jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Example Spell - Echo diff --git a/zeppelin-examples/zeppelin-example-spell-flowchart/pom.xml b/zeppelin-examples/zeppelin-example-spell-flowchart/pom.xml index b3575c99c6a..3f22d315507 100644 --- a/zeppelin-examples/zeppelin-example-spell-flowchart/pom.xml +++ b/zeppelin-examples/zeppelin-example-spell-flowchart/pom.xml @@ -22,14 +22,14 @@ zeppelin-examples org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-example-spell-flowchart jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Example Spell - Flowchart diff --git a/zeppelin-examples/zeppelin-example-spell-markdown/pom.xml b/zeppelin-examples/zeppelin-example-spell-markdown/pom.xml index b615eadc8a7..777b3d49fef 100644 --- a/zeppelin-examples/zeppelin-example-spell-markdown/pom.xml +++ b/zeppelin-examples/zeppelin-example-spell-markdown/pom.xml @@ -22,14 +22,14 @@ zeppelin-examples org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-example-spell-markdown jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Example Spell - Markdown diff --git a/zeppelin-examples/zeppelin-example-spell-translator/pom.xml b/zeppelin-examples/zeppelin-example-spell-translator/pom.xml index 09e6daaad38..51e1e48cb6a 100644 --- a/zeppelin-examples/zeppelin-example-spell-translator/pom.xml +++ b/zeppelin-examples/zeppelin-example-spell-translator/pom.xml @@ -22,14 +22,14 @@ zeppelin-examples org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-example-spell-translator jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Example Spell - Translator diff --git a/zeppelin-integration/pom.xml b/zeppelin-integration/pom.xml index e939a6351b9..eafa48540cf 100644 --- a/zeppelin-integration/pom.xml +++ b/zeppelin-integration/pom.xml @@ -24,14 +24,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-integration jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Integration Test diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index b0fd9920908..da31f787246 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -24,14 +24,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-interpreter jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Interpreter Zeppelin Interpreter diff --git a/zeppelin-jupyter/pom.xml b/zeppelin-jupyter/pom.xml index eef0e367b95..eab1d6280c6 100644 --- a/zeppelin-jupyter/pom.xml +++ b/zeppelin-jupyter/pom.xml @@ -24,13 +24,13 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. zeppelin-jupyter jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Jupyter Support Jupyter support for Apache Zeppelin diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 296d58f01fd..a73cd9643f0 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -22,14 +22,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-server jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Server diff --git a/zeppelin-web/pom.xml b/zeppelin-web/pom.xml index efec03941e4..444f8b29a4e 100644 --- a/zeppelin-web/pom.xml +++ b/zeppelin-web/pom.xml @@ -22,14 +22,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-web war - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: web Application diff --git a/zeppelin-web/src/app/spell/package.json b/zeppelin-web/src/app/spell/package.json index 7003e062b5b..3962887f552 100644 --- a/zeppelin-web/src/app/spell/package.json +++ b/zeppelin-web/src/app/spell/package.json @@ -1,7 +1,7 @@ { "name": "zeppelin-spell", "description": "Zeppelin Spell Framework", - "version": "0.8.0-SNAPSHOT", + "version": "0.9.0-SNAPSHOT", "main": "index", "dependencies": { }, diff --git a/zeppelin-web/src/app/tabledata/package.json b/zeppelin-web/src/app/tabledata/package.json index 2eec9090f91..837e29d4961 100644 --- a/zeppelin-web/src/app/tabledata/package.json +++ b/zeppelin-web/src/app/tabledata/package.json @@ -1,7 +1,7 @@ { "name": "zeppelin-tabledata", "description": "tabledata api", - "version": "0.8.0-SNAPSHOT", + "version": "0.9.0-SNAPSHOT", "main": "tabledata", "dependencies": { "json3": "~3.3.1", diff --git a/zeppelin-web/src/app/visualization/package.json b/zeppelin-web/src/app/visualization/package.json index 51a18149082..cd5a9eb6c5b 100644 --- a/zeppelin-web/src/app/visualization/package.json +++ b/zeppelin-web/src/app/visualization/package.json @@ -1,7 +1,7 @@ { "name": "zeppelin-vis", "description": "Visualization API", - "version": "0.8.0-SNAPSHOT", + "version": "0.9.0-SNAPSHOT", "main": "visualization", "dependencies": { "json3": "~3.3.1", diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 08de7ad50bc..ac7536018b1 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -23,14 +23,14 @@ zeppelin org.apache.zeppelin - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT .. org.apache.zeppelin zeppelin-zengine jar - 0.8.0-SNAPSHOT + 0.9.0-SNAPSHOT Zeppelin: Zengine Zeppelin Zengine From 49786a7717bec0345aaac895d09736b6c6605610 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 16 Jan 2018 16:16:40 +0800 Subject: [PATCH 003/386] ZEPPELIN-3137. Improve code style check ### What is this PR for? Although currently zeppelin has code check style, it is very weak. Such as unused import is not detected. The is the first PR for adding code style check. It just improve the code style check file , but just disable it for now. I have created subtask under ZEPPELIN-3137 for each module. This `checkstyle.xml` is from spark project. I think code style of spark is also fit for zeppelin project, just remove some spark related content in this `checkstyle.xml` ### What type of PR is it? [ Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3137 ### How should this be tested? * Travis build pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2727 from zjffdu/ZEPPELIN-3137 and squashes the following commits: a41bd32 [Jeff Zhang] address comment 9935281 [Jeff Zhang] ZEPPELIN-3137. Add code style check --- _tools/checkstyle.xml | 16 +++++++++++++--- pom.xml | 10 +++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/_tools/checkstyle.xml b/_tools/checkstyle.xml index cdb8fbf3096..d7eaaf94360 100644 --- a/_tools/checkstyle.xml +++ b/_tools/checkstyle.xml @@ -58,10 +58,11 @@ limitations under the License. - + + @@ -279,5 +280,14 @@ limitations under the License. + + + + + + + + + diff --git a/pom.xml b/pom.xml index 880bea6365f..725db415c41 100644 --- a/pom.xml +++ b/pom.xml @@ -385,9 +385,17 @@ org.apache.maven.plugins maven-checkstyle-plugin + 2.17 + true + false + true + ${basedir}/src/main/java,${basedir}/src/main/scala + ${basedir}/src/test/java _tools/checkstyle.xml - false + ${basedir}/target/checkstyle-output.xml + ${project.build.sourceEncoding} + ${project.reporting.outputEncoding} From ed1947bb062472b999a722238a9c250f11894e54 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Tue, 23 Jan 2018 18:13:23 +0530 Subject: [PATCH 004/386] [ZEPPELIN-3185] Add profiles for adding dependencies for Hive and Phoenix ### What is this PR for? This PR adds profiles for adding dependencies for Hive and Phoenix ### What type of PR is it? [Improvement] ### What is the Jira issue? * [ZEPPELIN-3185](https://issues.apache.org/jira/browse/ZEPPELIN-3185) ### Questions: * Does the licenses files need update? * Is there breaking changes for older versions? * Does this needs documentation? Author: Prabhjyot Singh Closes #2740 from prabhjyotsingh/ZEPPELIN-3185 and squashes the following commits: 47dd05e [Prabhjyot Singh] append "jdbc-" to all the profiles 99eafde [Prabhjyot Singh] add profile for adding dependencies for hive and phoenix --- jdbc/pom.xml | 195 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 2f8976a9d07..ba745dd632d 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -32,6 +32,201 @@ jar 0.9.0-SNAPSHOT Zeppelin: JDBC interpreter + + + + jdbc-hive + + 1.2.1 + 2.1.0 + + + + + org.apache.hive + hive-jdbc + ${hive.version} + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-auth + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpclient + + + + + + org.apache.httpcomponents + httpcore + 4.4.1 + + + org.apache.httpcomponents + httpclient + 4.4.1 + + + + org.apache.hive.shims + hive-shims-0.23 + ${hive2.version} + + + + + + jdbc-phoenix + + 4.7.0 + + + + org.apache.phoenix + phoenix-core + ${phoenix.version} + + + + xerces + xercesImpl + 2.11.0 + + + + + + jdbc-hadoop2 + + 2.7.3 + + + + org.apache.hadoop + hadoop-common + ${hadoop-common.version} + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-server + + + + javax.servlet + servlet-api + + + org.apache.avro + avro + + + org.apache.jackrabbit + jackrabbit-webdav + + + io.netty + netty + + + commons-httpclient + commons-httpclient + + + org.apache.zookeeper + zookeeper + + + org.eclipse.jgit + org.eclipse.jgit + + + com.jcraft + jsch + + + + + + + + jdbc-hadoop3 + + 3.0.0 + + + + org.apache.hadoop + hadoop-common + ${hadoop-common.version} + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-server + + + + javax.servlet + servlet-api + + + org.apache.avro + avro + + + org.apache.jackrabbit + jackrabbit-webdav + + + io.netty + netty + + + commons-httpclient + commons-httpclient + + + org.apache.zookeeper + zookeeper + + + org.eclipse.jgit + org.eclipse.jgit + + + com.jcraft + jsch + + + + + + From 7fbb5985ffd13dbdb76e708fcf8f5705b2df0a54 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 30 Jan 2018 10:19:41 +0800 Subject: [PATCH 005/386] ZEPPELIN-3138. checkstyle for zeppelin-interpreter ### What is this PR for? Code Style improvement ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3138 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jeff Zhang Closes #2753 from zjffdu/ZEPPELIN-3138 and squashes the following commits: c777161 [Jeff Zhang] ZEPPELIN-3138. checkstyle for zeppelin-interpreter --- _tools/checkstyle.xml | 12 +- zeppelin-interpreter/pom.xml | 9 + .../zeppelin/conf/ZeppelinConfiguration.java | 2 +- .../zeppelin/dep/DependencyResolver.java | 3 +- .../display/AngularObjectListener.java | 2 +- .../display/AngularObjectRegistry.java | 1 - .../AngularObjectRegistryListener.java | 6 +- .../org/apache/zeppelin/display/Input.java | 14 +- .../display/RuntimeTypeAdapterFactory.java | 8 +- .../apache/zeppelin/display/ui/CheckBox.java | 3 - .../apache/zeppelin/helium/Application.java | 1 - .../zeppelin/helium/ApplicationContext.java | 1 - .../helium/ApplicationEventListener.java | 8 +- .../zeppelin/helium/ApplicationLoader.java | 8 +- .../apache/zeppelin/helium/HeliumPackage.java | 6 +- .../zeppelin/interpreter/Interpreter.java | 23 +- .../interpreter/InterpreterHookListener.java | 4 +- .../InterpreterOutputChangeListener.java | 2 +- .../InterpreterOutputListener.java | 6 +- .../interpreter/InterpreterResult.java | 12 +- .../InterpreterResultMessageOutput.java | 7 +- ...nterpreterResultMessageOutputListener.java | 4 +- .../interpreter/WrappedInterpreter.java | 2 +- .../remote/RemoteEventClientWrapper.java | 4 +- .../RemoteInterpreterContextRunner.java | 1 - .../remote/RemoteInterpreterEventClient.java | 7 +- .../remote/RemoteInterpreterServer.java | 63 +++-- .../zeppelin/resource/LocalResourcePool.java | 7 +- .../apache/zeppelin/resource/Resource.java | 23 +- .../zeppelin/resource/ResourcePool.java | 16 +- .../resource/ResourcePoolConnector.java | 8 +- .../zeppelin/scheduler/JobListener.java | 6 +- .../zeppelin/scheduler/JobProgressPoller.java | 4 +- .../zeppelin/scheduler/ParallelScheduler.java | 2 +- .../apache/zeppelin/scheduler/Scheduler.java | 12 +- .../zeppelin/scheduler/SchedulerListener.java | 4 +- .../apache/zeppelin/tabledata/ColumnDef.java | 2 +- .../apache/zeppelin/tabledata/TableData.java | 4 +- .../org/apache/zeppelin/util/IdHashes.java | 4 +- .../zeppelin/dep/DependencyResolverTest.java | 22 +- .../display/AngularObjectRegistryTest.java | 18 +- .../zeppelin/display/AngularObjectTest.java | 71 +++--- .../org/apache/zeppelin/display/GUITest.java | 6 - .../apache/zeppelin/display/InputTest.java | 216 +++++++++--------- .../helium/ApplicationLoaderTest.java | 7 +- .../zeppelin/helium/HeliumPackageTest.java | 6 +- .../zeppelin/helium/MockApplication1.java | 1 - .../interpreter/InterpreterContextTest.java | 8 +- .../InterpreterHookRegistryTest.java | 16 +- .../InterpreterOutputChangeWatcherTest.java | 32 +-- .../interpreter/InterpreterOutputTest.java | 11 +- .../interpreter/InterpreterResultTest.java | 87 ++++--- .../zeppelin/interpreter/InterpreterTest.java | 10 +- .../interpreter/LazyOpenInterpreterTest.java | 11 +- .../remote/RemoteInterpreterServerTest.java | 34 +-- .../remote/RemoteInterpreterUtilsTest.java | 12 +- .../resource/LocalResourcePoolTest.java | 5 +- .../zeppelin/scheduler/FIFOSchedulerTest.java | 132 ++++++----- .../scheduler/ParallelSchedulerTest.java | 67 +++--- .../zeppelin/scheduler/SleepingJob.java | 114 ++++----- .../src/test/resources/log4j.properties | 2 +- 61 files changed, 658 insertions(+), 541 deletions(-) diff --git a/_tools/checkstyle.xml b/_tools/checkstyle.xml index d7eaaf94360..aa53d3078ad 100644 --- a/_tools/checkstyle.xml +++ b/_tools/checkstyle.xml @@ -58,9 +58,10 @@ limitations under the License. - Checks for out of order import statements - - This ensures that static imports go first + + + + @@ -78,11 +79,6 @@ limitations under the License. - - - - - diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index da31f787246..80864f9285d 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -239,6 +239,15 @@ + + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 17e3e5bfd0f..a10732023fd 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -603,7 +603,7 @@ public interface ConfigurationKeyPredicate { /** * Wrapper class. */ - public static enum ConfVars { + public enum ConfVars { ZEPPELIN_HOME("zeppelin.home", "./"), ZEPPELIN_ADDR("zeppelin.server.addr", "0.0.0.0"), ZEPPELIN_PORT("zeppelin.server.port", 8080), diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index c3ecdeedc13..495c69bf5d9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -156,7 +156,8 @@ private List loadFromMvn(String artifact, Collection excludes) */ @Override public List getArtifactsWithDep(String dependency, - Collection excludes) throws RepositoryException { + Collection excludes) + throws RepositoryException { Artifact artifact = new DefaultArtifact(dependency); DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE); PatternExclusionsDependencyFilter exclusionFilter = diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectListener.java index 880e48732c8..20f34af0e37 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectListener.java @@ -21,5 +21,5 @@ * */ public interface AngularObjectListener { - public void updated(AngularObject updatedObject); + void updated(AngularObject updatedObject); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java index a993992ae73..930ed7c8da0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java @@ -17,7 +17,6 @@ package org.apache.zeppelin.display; -import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; import java.util.List; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java index 103336dbfaa..081bb43d814 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java @@ -22,7 +22,7 @@ * */ public interface AngularObjectRegistryListener { - public void onAdd(String interpreterGroupId, AngularObject object); - public void onUpdate(String interpreterGroupId, AngularObject object); - public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId); + void onAdd(String interpreterGroupId, AngularObject object); + void onUpdate(String interpreterGroupId, AngularObject object); + void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java index a6860dea5e4..51e27d29151 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java @@ -18,12 +18,20 @@ package org.apache.zeppelin.display; import org.apache.commons.lang.StringUtils; -import org.apache.zeppelin.common.JsonSerializable; -import org.apache.zeppelin.display.ui.*; +import org.apache.zeppelin.display.ui.CheckBox; +import org.apache.zeppelin.display.ui.OptionInput; import org.apache.zeppelin.display.ui.OptionInput.ParamOption; +import org.apache.zeppelin.display.ui.Select; +import org.apache.zeppelin.display.ui.TextBox; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/RuntimeTypeAdapterFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/RuntimeTypeAdapterFactory.java index da05caa6338..65b4f6b04b3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/RuntimeTypeAdapterFactory.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/RuntimeTypeAdapterFactory.java @@ -17,7 +17,13 @@ package org.apache.zeppelin.display; -import com.google.gson.*; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; import com.google.gson.internal.Streams; import com.google.gson.reflect.TypeToken; import com.google.gson.stream.JsonReader; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/ui/CheckBox.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/ui/CheckBox.java index f9b4650f4da..02a0ff444cd 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/ui/CheckBox.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/ui/CheckBox.java @@ -18,10 +18,7 @@ package org.apache.zeppelin.display.ui; -import java.awt.*; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; /** * Html Checkbox diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java index 291899877db..d138595fe7d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java @@ -17,7 +17,6 @@ package org.apache.zeppelin.helium; import org.apache.zeppelin.annotation.Experimental; -import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.resource.ResourceSet; import java.io.IOException; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java index e0ea94cb01f..8d3f67e9912 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java @@ -16,7 +16,6 @@ */ package org.apache.zeppelin.helium; -import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.InterpreterOutput; /** diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java index eda907ae840..ca971f5c32f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java @@ -22,11 +22,11 @@ * Event from HeliumApplication running on remote interpreter process */ public interface ApplicationEventListener { - public void onOutputAppend( + void onOutputAppend( String noteId, String paragraphId, int index, String appId, String output); - public void onOutputUpdated( + void onOutputUpdated( String noteId, String paragraphId, int index, String appId, InterpreterResult.Type type, String output); - public void onLoad(String noteId, String paragraphId, String appId, HeliumPackage pkg); - public void onStatusChange(String noteId, String paragraphId, String appId, String status); + void onLoad(String noteId, String paragraphId, String appId, HeliumPackage pkg); + void onStatusChange(String noteId, String paragraphId, String appId, String status); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java index ddd061cc621..241273a9a8b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java @@ -28,7 +28,11 @@ import java.lang.reflect.Constructor; import java.net.URL; import java.net.URLClassLoader; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; /** * Load application @@ -55,7 +59,7 @@ private static class RunningApplication { String noteId; String paragraphId; - public RunningApplication(HeliumPackage packageInfo, String noteId, String paragraphId) { + RunningApplication(HeliumPackage packageInfo, String noteId, String paragraphId) { this.packageInfo = packageInfo; this.noteId = noteId; this.paragraphId = paragraphId; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java index 759b991645c..e9995c10664 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java @@ -19,7 +19,6 @@ import com.google.gson.Gson; import org.apache.zeppelin.annotation.Experimental; import org.apache.zeppelin.common.JsonSerializable; -import org.apache.zeppelin.dep.Repository; import java.util.Map; @@ -35,8 +34,9 @@ public class HeliumPackage implements JsonSerializable { private String description; // description private String artifact; // artifact name e.g) groupId:artifactId:versionId private String className; // entry point - private String [][] resources; // resource classnames that requires - // [[ .. and .. and .. ] or [ .. and .. and ..] ..] + // resource classnames that requires [[ .. and .. and .. ] or [ .. and .. and ..] ..] + private String [][] resources; + private String license; private String icon; private String published; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java index 386de4178c3..52cc161ccf7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java @@ -18,26 +18,25 @@ package org.apache.zeppelin.interpreter; -import java.lang.reflect.Field; -import java.net.URL; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.zeppelin.annotation.Experimental; import org.apache.zeppelin.annotation.ZeppelinApi; -import org.apache.zeppelin.interpreter.launcher.InterpreterLauncher; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.reflect.Field; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + /** * Interface for interpreters. * If you want to implement new Zeppelin interpreter, extend this class @@ -371,7 +370,7 @@ private void replaceContextParameters(Properties properties) { /** * Type of interpreter. */ - public static enum FormType { + public enum FormType { NATIVE, SIMPLE, NONE } @@ -458,7 +457,7 @@ public InterpreterRunner getRunner() { /** * Type of Scheduling. */ - public static enum SchedulingMode { + public enum SchedulingMode { FIFO, PARALLEL } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookListener.java index c70212c7b7e..d0dbad17f75 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookListener.java @@ -24,10 +24,10 @@ public interface InterpreterHookListener { /** * Prepends pre-execute hook code to the script that will be interpreted */ - public void onPreExecute(String script); + void onPreExecute(String script); /** * Appends post-execute hook code to the script that will be interpreted */ - public void onPostExecute(String script); + void onPostExecute(String script); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeListener.java index a639e0c6418..44bcd7cd089 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeListener.java @@ -22,6 +22,6 @@ * InterpreterOutputChangeListener */ public interface InterpreterOutputChangeListener { - public void fileChanged(File file); + void fileChanged(File file); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputListener.java index 42f6cfaef6a..a176ef299a6 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputListener.java @@ -23,7 +23,7 @@ public interface InterpreterOutputListener { /** * update all message outputs */ - public void onUpdateAll(InterpreterOutput out); + void onUpdateAll(InterpreterOutput out); /** * called when newline is detected @@ -31,12 +31,12 @@ public interface InterpreterOutputListener { * @param out * @param line */ - public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line); + void onAppend(int index, InterpreterResultMessageOutput out, byte[] line); /** * when entire output is updated. eg) after detecting new display system * @param index * @param out */ - public void onUpdate(int index, InterpreterResultMessageOutput out); + void onUpdate(int index, InterpreterResultMessageOutput out); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java index b08a97e1660..255b21ec78b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java @@ -17,15 +17,15 @@ package org.apache.zeppelin.interpreter; -import java.io.IOException; -import java.io.Serializable; - import com.google.gson.Gson; import org.apache.zeppelin.common.JsonSerializable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.io.IOException; +import java.io.Serializable; +import java.util.LinkedList; +import java.util.List; /** * Interpreter result template. @@ -37,7 +37,7 @@ public class InterpreterResult implements Serializable, JsonSerializable { /** * Type of result after code execution. */ - public static enum Code { + public enum Code { SUCCESS, INCOMPLETE, ERROR, @@ -47,7 +47,7 @@ public static enum Code { /** * Type of Data. */ - public static enum Type { + public enum Type { TEXT, HTML, ANGULAR, diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutput.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutput.java index 41e1fd0e184..da31364522a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutput.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutput.java @@ -19,7 +19,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.URL; import java.util.LinkedList; import java.util.List; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutputListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutputListener.java index ba5acf9414a..7f14a3edb98 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutputListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutputListener.java @@ -24,10 +24,10 @@ public interface InterpreterResultMessageOutputListener { * called when newline is detected * @param line */ - public void onAppend(InterpreterResultMessageOutput out, byte[] line); + void onAppend(InterpreterResultMessageOutput out, byte[] line); /** * when entire output is updated. eg) after detecting new display system */ - public void onUpdate(InterpreterResultMessageOutput out); + void onUpdate(InterpreterResultMessageOutput out); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/WrappedInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/WrappedInterpreter.java index a12a9aa109e..040b5469a51 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/WrappedInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/WrappedInterpreter.java @@ -21,5 +21,5 @@ * WrappedInterpreter */ public interface WrappedInterpreter { - public Interpreter getInnerInterpreter(); + Interpreter getInnerInterpreter(); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteEventClientWrapper.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteEventClientWrapper.java index bf36cd6354f..e43365ceb36 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteEventClientWrapper.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteEventClientWrapper.java @@ -10,9 +10,9 @@ */ public interface RemoteEventClientWrapper { - public void onMetaInfosReceived(Map infos); + void onMetaInfosReceived(Map infos); - public void onParaInfosReceived(String noteId, String paragraphId, + void onParaInfosReceived(String noteId, String paragraphId, Map infos); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java index 74b8db6d9d1..c0b1251d84e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java @@ -18,7 +18,6 @@ package org.apache.zeppelin.interpreter.remote; import org.apache.zeppelin.interpreter.InterpreterContextRunner; -import org.apache.zeppelin.interpreter.InterpreterException; /** * diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java index 5f8ccb7ab1c..9ca8a32f5f8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java @@ -25,7 +25,11 @@ import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType; import org.apache.zeppelin.interpreter.thrift.ZeppelinServerResourceParagraphRunner; -import org.apache.zeppelin.resource.*; +import org.apache.zeppelin.resource.RemoteResource; +import org.apache.zeppelin.resource.Resource; +import org.apache.zeppelin.resource.ResourceId; +import org.apache.zeppelin.resource.ResourcePoolConnector; +import org.apache.zeppelin.resource.ResourceSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -356,6 +360,7 @@ public RemoteInterpreterEvent pollEvent() { try { eventQueue.wait(1000); } catch (InterruptedException e) { + // ignore exception } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index c2a578c565e..fca84498aa4 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -97,8 +97,8 @@ * Accepting thrift connections from ZeppelinServer. */ public class RemoteInterpreterServer - extends Thread - implements RemoteInterpreterService.Iface, AngularObjectRegistryListener { + extends Thread + implements RemoteInterpreterService.Iface, AngularObjectRegistryListener { Logger logger = LoggerFactory.getLogger(RemoteInterpreterServer.class); InterpreterGroup interpreterGroup; @@ -171,6 +171,7 @@ public void run() { if (null != callbackHost && !isTest) { new Thread(new Runnable() { boolean interrupted = false; + @Override public void run() { while (!interrupted && !server.isServing()) { @@ -300,7 +301,7 @@ public void createInterpreter(String interpreterGroupId, String sessionId, Strin setSystemProperty(p); Constructor constructor = - replClass.getConstructor(new Class[] {Properties.class}); + replClass.getConstructor(new Class[]{Properties.class}); Interpreter repl = constructor.newInstance(p); repl.setClassloaderUrls(new URL[]{}); logger.info("Instantiate interpreter {}", className); @@ -417,7 +418,8 @@ public void close(String sessionId, String className) throws TException { @Override public RemoteInterpreterResult interpret(String noteId, String className, String st, - RemoteInterpreterContext interpreterContext) throws TException { + RemoteInterpreterContext interpreterContext) + throws TException { if (logger.isDebugEnabled()) { logger.debug("st:\n{}", st); } @@ -527,7 +529,7 @@ class InterpretJob extends Job { private Map infos; private Object results; - public InterpretJob( + InterpretJob( String jobId, String jobName, JobListener listener, @@ -716,7 +718,10 @@ public String getFormType(String sessionId, String className) throws TException @Override public List completion(String sessionId, - String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext) + String className, + String buf, + int cursor, + RemoteInterpreterContext remoteInterpreterContext) throws TException { Interpreter intp = getInterpreter(sessionId, className); try { @@ -733,7 +738,7 @@ private InterpreterContext convert(RemoteInterpreterContext ric) { private InterpreterContext convert(RemoteInterpreterContext ric, InterpreterOutput output) { List contextRunners = new LinkedList<>(); List runners = gson.fromJson(ric.getRunners(), - new TypeToken>() { + new TypeToken>() { }.getType()); for (InterpreterContextRunner r : runners) { @@ -748,7 +753,8 @@ private InterpreterContext convert(RemoteInterpreterContext ric, InterpreterOutp ric.getParagraphText(), AuthenticationInfo.fromJson(ric.getAuthenticationInfo()), (Map) gson.fromJson(ric.getConfig(), - new TypeToken>() {}.getType()), + new TypeToken>() { + }.getType()), GUI.fromJson(ric.getGui()), GUI.fromJson(ric.getNoteGui()), interpreterGroup.getAngularObjectRegistry(), @@ -798,7 +804,7 @@ static class ParagraphRunner extends InterpreterContextRunner { Logger logger = LoggerFactory.getLogger(ParagraphRunner.class); private transient RemoteInterpreterServer server; - public ParagraphRunner(RemoteInterpreterServer server, String noteId, String paragraphId) { + ParagraphRunner(RemoteInterpreterServer server, String noteId, String paragraphId) { super(noteId, paragraphId); this.server = server; } @@ -809,13 +815,14 @@ public void run() { } } - static class ZeppelinRemoteWorksController implements RemoteWorksController{ + static class ZeppelinRemoteWorksController implements RemoteWorksController { Logger logger = LoggerFactory.getLogger(ZeppelinRemoteWorksController.class); private final long DEFAULT_TIMEOUT_VALUE = 300000; private final Map remoteWorksResponsePool; private RemoteInterpreterServer server; - public ZeppelinRemoteWorksController( + + ZeppelinRemoteWorksController( RemoteInterpreterServer server, Map remoteWorksResponsePool) { this.remoteWorksResponsePool = remoteWorksResponsePool; this.server = server; @@ -882,7 +889,7 @@ public List getRemoteContextRunner( } private RemoteInterpreterResult convert(InterpreterResult result, - Map config, GUI gui, GUI noteGui) { + Map config, GUI gui, GUI noteGui) { List msg = new LinkedList<>(); for (InterpreterResultMessage m : result.message()) { @@ -930,7 +937,6 @@ public String getStatus(String sessionId, String jobId) } - @Override public void onAdd(String interpreterGroupId, AngularObject object) { eventClient.angularObjectAdd(object); @@ -949,6 +955,7 @@ public void onRemove(String interpreterGroupId, String name, String noteId, Stri /** * Poll event from RemoteInterpreterEventPoller + * * @return * @throws TException */ @@ -959,8 +966,9 @@ public RemoteInterpreterEvent getEvent() throws TException { /** * called when object is updated in client (web) side. + * * @param name - * @param noteId noteId where the update issues + * @param noteId noteId where the update issues * @param paragraphId paragraphId where the update issues * @param object * @throws TException @@ -998,8 +1006,8 @@ public void angularObjectUpdate(String name, String noteId, String paragraphId, if (value == null) { try { value = gson.fromJson(object, - new TypeToken>() { - }.getType()); + new TypeToken>() { + }.getType()); } catch (Exception e) { // it's not a generic json object, too. okay, proceed to threat as a string type logger.debug(e.getMessage(), e); @@ -1050,7 +1058,7 @@ public void angularObjectAdd(String name, String noteId, String paragraphId, Str @Override public void angularObjectRemove(String name, String noteId, String paragraphId) throws - TException { + TException { AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry(); registry.remove(name, noteId, paragraphId, false); } @@ -1062,8 +1070,9 @@ public void resourcePoolResponseGetAll(List resources) throws TException /** * Get payload of resource from remote + * * @param resourceId json serialized ResourceId - * @param object java serialized of the object + * @param object java serialized of the object * @throws TException */ @Override @@ -1158,8 +1167,9 @@ public ByteBuffer resourceInvokeMethod( /** * Get payload of resource from remote + * * @param invokeResourceMethodEventMessage json serialized InvokeResourcemethodEventMessage - * @param object java serialized of the object + * @param object java serialized of the object * @throws TException */ @Override @@ -1184,8 +1194,9 @@ public void resourceResponseInvokeMethod( public void angularRegistryPush(String registryAsString) throws TException { try { Map> deserializedRegistry = gson - .fromJson(registryAsString, - new TypeToken>>() { }.getType()); + .fromJson(registryAsString, + new TypeToken>>() { + }.getType()); interpreterGroup.getAngularObjectRegistry().setRegistry(deserializedRegistry); } catch (Exception e) { logger.info("Exception in RemoteInterpreterServer while angularRegistryPush, nolock", e); @@ -1193,8 +1204,8 @@ public void angularRegistryPush(String registryAsString) throws TException { } protected InterpreterOutput createAppOutput(final String noteId, - final String paragraphId, - final String appId) { + final String paragraphId, + final String appId) { return new InterpreterOutput(new InterpreterOutputListener() { @Override public void onUpdateAll(InterpreterOutput out) { @@ -1322,7 +1333,7 @@ private static class RunningApplication { public final String noteId; public final String paragraphId; - public RunningApplication(HeliumPackage pkg, + RunningApplication(HeliumPackage pkg, Application app, String noteId, String paragraphId) { @@ -1331,5 +1342,7 @@ public RunningApplication(HeliumPackage pkg, this.noteId = noteId; this.paragraphId = paragraphId; } - }; + } + + ; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/LocalResourcePool.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/LocalResourcePool.java index 5a0193c01cd..7ae22735eff 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/LocalResourcePool.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/LocalResourcePool.java @@ -16,7 +16,9 @@ */ package org.apache.zeppelin.resource; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; /** * ResourcePool @@ -35,6 +37,7 @@ public LocalResourcePool(String id) { /** * Get unique id of this resource pool + * * @return */ @Override @@ -44,6 +47,7 @@ public String id() { /** * Get resource + * * @return null if resource not found */ @Override @@ -65,6 +69,7 @@ public ResourceSet getAll() { /** * Put resource into the pull + * * @param * @param object object to put into the resource */ diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java index b57d46b9139..ec95ffbfa44 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java @@ -21,7 +21,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; import java.lang.reflect.Method; import java.nio.ByteBuffer; @@ -109,13 +114,14 @@ public boolean isLocal() { /** * Call a method of the object that this resource holds + * * @param methodName name of method to call * @param paramTypes method parameter types - * @param params method parameter values + * @param params method parameter values * @return return value of the method */ public Object invokeMethod( - String methodName, Class [] paramTypes, Object [] params) { + String methodName, Class[] paramTypes, Object[] params) { if (r != null) { try { Method method = r.getClass().getMethod( @@ -124,7 +130,7 @@ public Object invokeMethod( method.setAccessible(true); Object ret = method.invoke(r, params); return ret; - } catch (Exception e) { + } catch (Exception e) { logException(e); return null; } @@ -135,14 +141,15 @@ public Object invokeMethod( /** * Call a method of the object that this resource holds and save return value as a resource - * @param methodName name of method to call - * @param paramTypes method parameter types - * @param params method parameter values + * + * @param methodName name of method to call + * @param paramTypes method parameter types + * @param params method parameter values * @param returnResourceName name of resource that return value will be saved * @return Resource that holds return value */ public Resource invokeMethod( - String methodName, Class [] paramTypes, Object [] params, String returnResourceName) { + String methodName, Class[] paramTypes, Object[] params, String returnResourceName) { if (r != null) { try { Method method = r.getClass().getMethod( diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePool.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePool.java index a1c28de9222..12b4d7ae283 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePool.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePool.java @@ -24,14 +24,14 @@ public interface ResourcePool { * Get unique id of the resource pool * @return */ - public String id(); + String id(); /** * Get resource from name * @param name Resource name * @return null if resource not found */ - public Resource get(String name); + Resource get(String name); /** * Get resource from name @@ -40,20 +40,20 @@ public interface ResourcePool { * @param name Resource name * @return null if resource not found */ - public Resource get(String noteId, String paragraphId, String name); + Resource get(String noteId, String paragraphId, String name); /** * Get all resources * @return */ - public ResourceSet getAll(); + ResourceSet getAll(); /** * Put an object into resource pool * @param name * @param object */ - public void put(String name, Object object); + void put(String name, Object object); /** * Put an object into resource pool @@ -65,14 +65,14 @@ public interface ResourcePool { * @param name * @param object */ - public void put(String noteId, String paragraphId, String name, Object object); + void put(String noteId, String paragraphId, String name, Object object); /** * Remove object * @param name Resource name to remove * @return removed Resource. null if resource not found */ - public Resource remove(String name); + Resource remove(String name); /** * Remove object @@ -81,5 +81,5 @@ public interface ResourcePool { * @param name Resource name to remove * @return removed Resource. null if resource not found */ - public Resource remove(String noteId, String paragraphId, String name); + Resource remove(String noteId, String paragraphId, String name); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolConnector.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolConnector.java index f270d920681..169229ba837 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolConnector.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolConnector.java @@ -24,19 +24,19 @@ public interface ResourcePoolConnector { * Get list of resources from all other resource pools in remote processes * @return */ - public ResourceSet getAllResources(); + ResourceSet getAllResources(); /** * Read remote object * @return */ - public Object readResource(ResourceId id); + Object readResource(ResourceId id); /** * Invoke method of Resource and get return * @return */ - public Object invokeMethod( + Object invokeMethod( ResourceId id, String methodName, Class[] paramTypes, @@ -45,7 +45,7 @@ public Object invokeMethod( /** * Invoke method, put result into resource pool and return */ - public Resource invokeMethod( + Resource invokeMethod( ResourceId id, String methodName, Class[] paramTypes, diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobListener.java index 9ee274654ff..3042941d774 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobListener.java @@ -21,9 +21,9 @@ * TODO(moon) : add description. */ public interface JobListener { - public void onProgressUpdate(Job job, int progress); + void onProgressUpdate(Job job, int progress); - public void beforeStatusChange(Job job, Job.Status before, Job.Status after); + void beforeStatusChange(Job job, Job.Status before, Job.Status after); - public void afterStatusChange(Job job, Job.Status before, Job.Status after); + void afterStatusChange(Job job, Job.Status before, Job.Status after); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java index 8b8cda0af84..3d6ce12f48b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java @@ -60,6 +60,8 @@ public void run() { } Thread.sleep(intervalMs); } - } catch (InterruptedException ignored) {} + } catch (InterruptedException ignored) { + // just ignore this exception + } } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ParallelScheduler.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ParallelScheduler.java index 36a67e3f9c1..6f67cd7c5d4 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ParallelScheduler.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ParallelScheduler.java @@ -136,7 +136,7 @@ private class JobRunner implements Runnable { private Scheduler scheduler; private Job job; - public JobRunner(Scheduler scheduler, Job job) { + JobRunner(Scheduler scheduler, Job job) { this.scheduler = scheduler; this.job = job; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Scheduler.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Scheduler.java index 90d43979c4f..30557274bc6 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Scheduler.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Scheduler.java @@ -23,15 +23,15 @@ * Interface for scheduler */ public interface Scheduler extends Runnable { - public String getName(); + String getName(); - public Collection getJobsWaiting(); + Collection getJobsWaiting(); - public Collection getJobsRunning(); + Collection getJobsRunning(); - public void submit(Job job); + void submit(Job job); - public Job removeFromWaitingQueue(String jobId); + Job removeFromWaitingQueue(String jobId); - public void stop(); + void stop(); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerListener.java index 79232e61a5a..9a6b3eddfb3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerListener.java @@ -21,7 +21,7 @@ * TODO(moon) : add description. */ public interface SchedulerListener { - public void jobStarted(Scheduler scheduler, Job job); + void jobStarted(Scheduler scheduler, Job job); - public void jobFinished(Scheduler scheduler, Job job); + void jobFinished(Scheduler scheduler, Job job); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/ColumnDef.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/ColumnDef.java index 75afc18c70f..a2fac20a20e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/ColumnDef.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/ColumnDef.java @@ -25,7 +25,7 @@ public class ColumnDef implements Serializable { /** * Type */ - public static enum TYPE { + public enum TYPE { STRING, LONG, INT diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableData.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableData.java index 6e34f47732b..ed254c5b8e8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableData.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableData.java @@ -26,12 +26,12 @@ public interface TableData { * Get column definitions * @return */ - public ColumnDef [] columns(); + ColumnDef [] columns(); /** * Get row iterator * @param * @return */ - public Iterator rows(); + Iterator rows(); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java index 052aaefd219..9d4e10f35dd 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java @@ -27,8 +27,8 @@ */ public class IdHashes { private static final char[] DICTIONARY = new char[] {'1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z'}; + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z'}; /** * encodes the given string into the base of the dictionary provided in the constructor. diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java index 876e8e7c88c..7ccc7dfb069 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java @@ -17,12 +17,6 @@ package org.apache.zeppelin.dep; -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.FileNotFoundException; -import java.util.Collections; - import org.apache.commons.io.FileUtils; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -31,6 +25,12 @@ import org.junit.rules.ExpectedException; import org.sonatype.aether.RepositoryException; +import java.io.File; +import java.io.FileNotFoundException; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + public class DependencyResolverTest { private static DependencyResolver resolver; private static String testPath; @@ -42,12 +42,13 @@ public class DependencyResolverTest { @BeforeClass public static void setUp() throws Exception { - tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + + System.currentTimeMillis()); testPath = tmpDir.getAbsolutePath() + "/test-repo"; testCopyPath = new File(tmpDir, "test-copy-repo"); resolver = new DependencyResolver(testPath); } - + @AfterClass public static void tearDown() throws Exception { FileUtils.deleteDirectory(tmpDir); @@ -85,7 +86,8 @@ public void testLoad() throws Exception { FileUtils.cleanDirectory(testCopyPath); // load from added repository - resolver.addRepo("sonatype", "https://oss.sonatype.org/content/repositories/agimatec-releases/", false); + resolver.addRepo("sonatype", + "https://oss.sonatype.org/content/repositories/agimatec-releases/", false); resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath); assertEquals(testCopyPath.list().length, 8); @@ -103,4 +105,4 @@ public void should_throw_exception_if_dependency_not_found() throws Exception { resolver.load("one.two:1.0", testCopyPath); } -} \ No newline at end of file +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java index 2d0436f205b..aefc48471e1 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java @@ -17,13 +17,12 @@ package org.apache.zeppelin.display; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import org.junit.Test; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class AngularObjectRegistryTest { @@ -47,10 +46,13 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { } @Override - public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId) { + public void onRemove(String interpreterGroupId, + String name, + String noteId, + String paragraphId) { onRemove.incrementAndGet(); } - }); + }); registry.add("name1", "value1", "note1", null); assertEquals(1, registry.getAll("note1", null).size()); @@ -65,12 +67,12 @@ public void onRemove(String interpreterGroupId, String name, String noteId, Stri assertEquals(1, onRemove.get()); assertEquals(null, registry.get("name1", "note1", null)); - + // namespace registry.add("name1", "value11", "note2", null); assertEquals("value11", registry.get("name1", "note2", null).get()); assertEquals(null, registry.get("name1", "note1", null)); - + // null namespace registry.add("name1", "global1", null, null); assertEquals("global1", registry.get("name1", null, null).get()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java index 924c5d4126c..0196526e2f3 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java @@ -17,61 +17,61 @@ package org.apache.zeppelin.display; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.junit.Test; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; public class AngularObjectTest { @Test public void testEquals() { assertEquals( - new AngularObject("name", "value", "note1", null, null), - new AngularObject("name", "value", "note1", null, null) + new AngularObject("name", "value", "note1", null, null), + new AngularObject("name", "value", "note1", null, null) ); assertEquals( - new AngularObject("name", "value", "note1", "paragraph1", null), - new AngularObject("name", "value", "note1", "paragraph1", null) + new AngularObject("name", "value", "note1", "paragraph1", null), + new AngularObject("name", "value", "note1", "paragraph1", null) ); assertEquals( - new AngularObject("name", "value", null, null, null), - new AngularObject("name", "value", null, null, null) + new AngularObject("name", "value", null, null, null), + new AngularObject("name", "value", null, null, null) ); assertEquals( - new AngularObject("name", "value1", null, null, null), - new AngularObject("name", "value2", null, null, null) + new AngularObject("name", "value1", null, null, null), + new AngularObject("name", "value2", null, null, null) ); assertNotSame( - new AngularObject("name1", "value", null, null, null), - new AngularObject("name2", "value", null, null, null) + new AngularObject("name1", "value", null, null, null), + new AngularObject("name2", "value", null, null, null) ); assertNotSame( - new AngularObject("name1", "value", "note1", null, null), - new AngularObject("name2", "value", "note2", null, null) + new AngularObject("name1", "value", "note1", null, null), + new AngularObject("name2", "value", "note2", null, null) ); assertNotSame( - new AngularObject("name1", "value", "note", null, null), - new AngularObject("name2", "value", null, null, null) + new AngularObject("name1", "value", "note", null, null), + new AngularObject("name2", "value", null, null, null) ); assertNotSame( - new AngularObject("name", "value", "note", "paragraph1", null), - new AngularObject("name", "value", "note", "paragraph2", null) + new AngularObject("name", "value", "note", "paragraph1", null), + new AngularObject("name", "value", "note", "paragraph2", null) ); assertNotSame( - new AngularObject("name", "value", "note1", null, null), - new AngularObject("name", "value", "note1", "paragraph1", null) + new AngularObject("name", "value", "note1", null, null), + new AngularObject("name", "value", "note1", "paragraph1", null) ); @@ -80,14 +80,14 @@ public void testEquals() { @Test public void testListener() { final AtomicInteger updated = new AtomicInteger(0); - AngularObject ao = new AngularObject("name", "value", "note1", null, new AngularObjectListener() { + AngularObject ao = new AngularObject("name", "value", "note1", null, + new AngularObjectListener() { - @Override - public void updated(AngularObject updatedObject) { - updated.incrementAndGet(); - } - - }); + @Override + public void updated(AngularObject updatedObject) { + updated.incrementAndGet(); + } + }); assertEquals(0, updated.get()); ao.set("newValue"); @@ -106,12 +106,13 @@ public void updated(AngularObject updatedObject) { public void testWatcher() throws InterruptedException { final AtomicInteger updated = new AtomicInteger(0); final AtomicInteger onWatch = new AtomicInteger(0); - AngularObject ao = new AngularObject("name", "value", "note1", null, new AngularObjectListener() { - @Override - public void updated(AngularObject updatedObject) { - updated.incrementAndGet(); - } - }); + AngularObject ao = new AngularObject("name", "value", "note1", null, + new AngularObjectListener() { + @Override + public void updated(AngularObject updatedObject) { + updated.incrementAndGet(); + } + }); ao.addWatcher(new AngularObjectWatcher(null) { @Override diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/GUITest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/GUITest.java index 15af2e66ab0..7d369b83cef 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/GUITest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/GUITest.java @@ -17,22 +17,16 @@ package org.apache.zeppelin.display; -import org.apache.commons.io.IOUtils; import org.apache.zeppelin.display.ui.CheckBox; import org.apache.zeppelin.display.ui.OptionInput.ParamOption; import org.apache.zeppelin.display.ui.Select; import org.apache.zeppelin.display.ui.TextBox; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/InputTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/InputTest.java index d3d5a009c04..a9252b96443 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/InputTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/InputTest.java @@ -17,9 +17,6 @@ package org.apache.zeppelin.display; -import java.util.HashMap; -import java.util.Map; - import org.apache.zeppelin.display.ui.CheckBox; import org.apache.zeppelin.display.ui.OptionInput.ParamOption; import org.apache.zeppelin.display.ui.Select; @@ -28,112 +25,123 @@ import org.junit.Before; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; + import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class InputTest { - @Before - public void setUp() throws Exception { - } - - @After - public void tearDown() throws Exception { - } - - @Test - public void testFormExtraction() { - // textbox form - String script = "${input_form=}"; - Map forms = Input.extractSimpleQueryForm(script, false); - assertEquals(1, forms.size()); - Input form = forms.get("input_form"); - assertEquals("input_form", form.name); - assertNull(form.displayName); - assertEquals("", form.defaultValue); - assertTrue(form instanceof TextBox); - - // textbox form with display name & default value - script = "${input_form(Input Form)=xxx}"; - forms = Input.extractSimpleQueryForm(script, false); - form = forms.get("input_form"); - assertEquals("xxx", form.defaultValue); - assertTrue(form instanceof TextBox); - - // selection form - script = "${select_form(Selection Form)=op1,op1|op2(Option 2)|op3}"; - form = Input.extractSimpleQueryForm(script, false).get("select_form"); - assertEquals("select_form", form.name); - assertEquals("op1", form.defaultValue); - assertTrue(form instanceof Select); - assertArrayEquals(new ParamOption[]{new ParamOption("op1", null), - new ParamOption("op2", "Option 2"), new ParamOption("op3", null)}, - ((Select) form).getOptions()); - - // checkbox form - script = "${checkbox:checkbox_form=op1,op1|op2|op3}"; - form = Input.extractSimpleQueryForm(script, false).get("checkbox_form"); - assertEquals("checkbox_form", form.name); - assertTrue(form instanceof CheckBox); - - assertArrayEquals(new Object[]{"op1"}, (Object[]) form.defaultValue); - assertArrayEquals(new ParamOption[]{new ParamOption("op1", null), - new ParamOption("op2", null), new ParamOption("op3", null)}, - ((CheckBox) form).getOptions()); - - // checkbox form with multiple default checks - script = "${checkbox:checkbox_form(Checkbox Form)=op1|op3,op1(Option 1)|op2|op3}"; - form = Input.extractSimpleQueryForm(script, false).get("checkbox_form"); - assertEquals("checkbox_form", form.name); - assertEquals("Checkbox Form", form.displayName); - assertTrue(form instanceof CheckBox); - assertArrayEquals(new Object[]{"op1", "op3"}, (Object[]) form.defaultValue); - assertArrayEquals(new ParamOption[]{new ParamOption("op1", "Option 1"), - new ParamOption("op2", null), new ParamOption("op3", null)}, - ((CheckBox) form).getOptions()); - - // checkbox form with no default check - script = "${checkbox:checkbox_form(Checkbox Form)=,op1(Option 1)|op2(Option 2)|op3(Option 3)}"; - form = Input.extractSimpleQueryForm(script, false).get("checkbox_form"); - assertEquals("checkbox_form", form.name); - assertEquals("Checkbox Form", form.displayName); - assertTrue(form instanceof CheckBox); - assertArrayEquals(new Object[]{}, (Object[]) form.defaultValue); - assertArrayEquals(new ParamOption[]{new ParamOption("op1", "Option 1"), - new ParamOption("op2", "Option 2"), new ParamOption("op3", "Option 3")}, - ((CheckBox) form).getOptions()); - } - - - @Test - public void testFormSubstitution() { - // test form substitution without new forms - String script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=,s_op1|s_op2|s_op3}\n" + - "CHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3}"; - Map params = new HashMap<>(); - params.put("input_form", "some_input"); - params.put("select_form", "s_op2"); - params.put("checkbox_form", new String[]{"c_op1", "c_op3"}); - String replaced = Input.getSimpleQuery(params, script, false); - assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1,c_op3", replaced); - - // test form substitution with new forms - script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=,s_op1|s_op2|s_op3}\n" + - "CHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3}\n" + - "NEW_CHECKED=${checkbox( and ):new_check=nc_a|nc_c,nc_a|nc_b|nc_c}"; - replaced = Input.getSimpleQuery(params, script, false); - assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1,c_op3\n" + - "NEW_CHECKED=nc_a and nc_c", replaced); - - // test form substitution with obsoleted values - script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=,s_op1|s_op2|s_op3}\n" + - "CHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3_new}\n" + - "NEW_CHECKED=${checkbox( and ):new_check=nc_a|nc_c,nc_a|nc_b|nc_c}"; - replaced = Input.getSimpleQuery(params, script, false); - assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1\n" + - "NEW_CHECKED=nc_a and nc_c", replaced); - } + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testFormExtraction() { + // textbox form + String script = "${input_form=}"; + Map forms = Input.extractSimpleQueryForm(script, false); + assertEquals(1, forms.size()); + Input form = forms.get("input_form"); + assertEquals("input_form", form.name); + assertNull(form.displayName); + assertEquals("", form.defaultValue); + assertTrue(form instanceof TextBox); + + // textbox form with display name & default value + script = "${input_form(Input Form)=xxx}"; + forms = Input.extractSimpleQueryForm(script, false); + form = forms.get("input_form"); + assertEquals("xxx", form.defaultValue); + assertTrue(form instanceof TextBox); + + // selection form + script = "${select_form(Selection Form)=op1,op1|op2(Option 2)|op3}"; + form = Input.extractSimpleQueryForm(script, false).get("select_form"); + assertEquals("select_form", form.name); + assertEquals("op1", form.defaultValue); + assertTrue(form instanceof Select); + assertArrayEquals(new ParamOption[]{ + new ParamOption("op1", null), + new ParamOption("op2", "Option 2"), + new ParamOption("op3", null)}, + ((Select) form).getOptions()); + + // checkbox form + script = "${checkbox:checkbox_form=op1,op1|op2|op3}"; + form = Input.extractSimpleQueryForm(script, false).get("checkbox_form"); + assertEquals("checkbox_form", form.name); + assertTrue(form instanceof CheckBox); + + assertArrayEquals(new Object[]{"op1"}, (Object[]) form.defaultValue); + assertArrayEquals(new ParamOption[]{ + new ParamOption("op1", null), + new ParamOption("op2", null), + new ParamOption("op3", null)}, + ((CheckBox) form).getOptions()); + + // checkbox form with multiple default checks + script = "${checkbox:checkbox_form(Checkbox Form)=op1|op3,op1(Option 1)|op2|op3}"; + form = Input.extractSimpleQueryForm(script, false).get("checkbox_form"); + assertEquals("checkbox_form", form.name); + assertEquals("Checkbox Form", form.displayName); + assertTrue(form instanceof CheckBox); + assertArrayEquals(new Object[]{"op1", "op3"}, (Object[]) form.defaultValue); + assertArrayEquals(new ParamOption[]{ + new ParamOption("op1", "Option 1"), + new ParamOption("op2", null), + new ParamOption("op3", null)}, + ((CheckBox) form).getOptions()); + + // checkbox form with no default check + script = "${checkbox:checkbox_form(Checkbox Form)=,op1(Option 1)|op2(Option 2)|op3(Option 3)}"; + form = Input.extractSimpleQueryForm(script, false).get("checkbox_form"); + assertEquals("checkbox_form", form.name); + assertEquals("Checkbox Form", form.displayName); + assertTrue(form instanceof CheckBox); + assertArrayEquals(new Object[]{}, (Object[]) form.defaultValue); + assertArrayEquals(new ParamOption[]{ + new ParamOption("op1", "Option 1"), + new ParamOption("op2", "Option 2"), + new ParamOption("op3", "Option 3")}, + ((CheckBox) form).getOptions()); + } + + + @Test + public void testFormSubstitution() { + // test form substitution without new forms + String script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=" + + ",s_op1|s_op2|s_op3}\nCHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3}"; + Map params = new HashMap<>(); + params.put("input_form", "some_input"); + params.put("select_form", "s_op2"); + params.put("checkbox_form", new String[]{"c_op1", "c_op3"}); + String replaced = Input.getSimpleQuery(params, script, false); + assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1,c_op3", replaced); + + // test form substitution with new forms + script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=,s_op1|s_op2|s_op3}\n" + + "CHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3}\n" + + "NEW_CHECKED=${checkbox( and ):new_check=nc_a|nc_c,nc_a|nc_b|nc_c}"; + replaced = Input.getSimpleQuery(params, script, false); + assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1,c_op3\n" + + "NEW_CHECKED=nc_a and nc_c", replaced); + + // test form substitution with obsoleted values + script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=,s_op1|s_op2|s_op3}\n" + + "CHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3_new}\n" + + "NEW_CHECKED=${checkbox( and ):new_check=nc_a|nc_c,nc_a|nc_b|nc_c}"; + replaced = Input.getSimpleQuery(params, script, false); + assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1\n" + + "NEW_CHECKED=nc_a and nc_c", replaced); + } } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java index acb4d7fbea1..490c9119ecf 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java @@ -28,14 +28,17 @@ import java.io.File; import java.io.IOException; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class ApplicationLoaderTest { private File tmpDir; @Before public void setUp() { - tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + + System.currentTimeMillis()); tmpDir.mkdirs(); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/HeliumPackageTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/HeliumPackageTest.java index ad427e93885..e8107424612 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/HeliumPackageTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/HeliumPackageTest.java @@ -21,7 +21,7 @@ import java.util.Map; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; public class HeliumPackageTest { @@ -71,10 +71,10 @@ public void parseConfig() { Map config = p.getConfig(); Map accessToken = (Map) config.get("access-token"); - assertEquals((String) accessToken.get("type"),"string"); + assertEquals((String) accessToken.get("type"), "string"); assertEquals((String) accessToken.get("description"), "access token for Google Translation API"); assertEquals((String) accessToken.get("defaultValue"), "EXAMPLE-TOKEN"); } -} \ No newline at end of file +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java index df3afeffc10..c962d8447a6 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java @@ -16,7 +16,6 @@ */ package org.apache.zeppelin.helium; -import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.resource.ResourceSet; /** diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterContextTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterContextTest.java index 70e2cbabbd0..8ad4841bb3b 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterContextTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterContextTest.java @@ -17,17 +17,19 @@ package org.apache.zeppelin.interpreter; -import static org.junit.Assert.*; - import org.junit.Test; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + public class InterpreterContextTest { @Test public void testThreadLocal() { assertNull(InterpreterContext.get()); - InterpreterContext.set(new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null, null, null)); + InterpreterContext.set(new InterpreterContext(null, null, null, null, null, null, null, null, + null, null, null, null, null)); assertNotNull(InterpreterContext.get()); InterpreterContext.remove(); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterHookRegistryTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterHookRegistryTest.java index 7614e9eb204..eab8a2885b4 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterHookRegistryTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterHookRegistryTest.java @@ -17,13 +17,11 @@ package org.apache.zeppelin.interpreter; +import org.junit.Test; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import java.util.concurrent.atomic.AtomicInteger; - -import org.junit.Test; - public class InterpreterHookRegistryTest { @Test @@ -38,7 +36,7 @@ public void testBasic() { final String preExecHook = "pre"; final String postExecHook = "post"; InterpreterHookRegistry registry = new InterpreterHookRegistry("intpId"); - + // Test register() registry.register(noteId, className, PRE_EXEC, preExecHook); registry.register(noteId, className, POST_EXEC, postExecHook); @@ -50,7 +48,7 @@ public void testBasic() { assertEquals(registry.get(noteId, className, POST_EXEC), postExecHook); assertEquals(registry.get(noteId, className, PRE_EXEC_DEV), preExecHook); assertEquals(registry.get(noteId, className, POST_EXEC_DEV), postExecHook); - + // Test Unregister registry.unregister(noteId, className, PRE_EXEC); registry.unregister(noteId, className, POST_EXEC); @@ -60,16 +58,16 @@ public void testBasic() { assertNull(registry.get(noteId, className, POST_EXEC)); assertNull(registry.get(noteId, className, PRE_EXEC_DEV)); assertNull(registry.get(noteId, className, POST_EXEC_DEV)); - + // Test Global Scope registry.register(null, className, PRE_EXEC, preExecHook); assertEquals(registry.get(GLOBAL_KEY, className, PRE_EXEC), preExecHook); } - + @Test(expected = IllegalArgumentException.class) public void testValidEventCode() { InterpreterHookRegistry registry = new InterpreterHookRegistry("intpId"); - + // Test that only valid event codes ("pre_exec", "post_exec") are accepted registry.register("foo", "bar", "baz", "whatever"); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java index f3a30fbd274..683200beecf 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java @@ -16,16 +16,18 @@ */ package org.apache.zeppelin.interpreter; -import static org.junit.Assert.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChangeListener { private File tmpDir; @@ -38,7 +40,8 @@ public void setUp() throws Exception { watcher = new InterpreterOutputChangeWatcher(this); watcher.start(); - tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + + System.currentTimeMillis()); tmpDir.mkdirs(); fileChanged = null; numChanged = new AtomicInteger(0); @@ -50,12 +53,13 @@ public void tearDown() throws Exception { delete(tmpDir); } - private void delete(File file){ - if(file.isFile()) file.delete(); - else if(file.isDirectory()){ - File [] files = file.listFiles(); - if(files!=null && files.length>0){ - for(File f : files){ + private void delete(File file) { + if (file.isFile()) { + file.delete(); + } else if (file.isDirectory()) { + File[] files = file.listFiles(); + if (files != null && files.length > 0) { + for (File f : files) { delete(f); } } @@ -89,7 +93,7 @@ public void test() throws IOException, InterruptedException { out2.close(); synchronized (this) { - wait(30*1000); + wait(30 * 1000); } assertNotNull(fileChanged); @@ -102,9 +106,9 @@ public void fileChanged(File file) { fileChanged = file; numChanged.incrementAndGet(); - synchronized(this) { + synchronized (this) { notify(); } } -} \ No newline at end of file +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputTest.java index 82d8c3fbc55..8158151a896 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputTest.java @@ -16,14 +16,15 @@ */ package org.apache.zeppelin.interpreter; -import static org.junit.Assert.*; - -import java.io.IOException; - import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class InterpreterOutputTest implements InterpreterOutputListener { private InterpreterOutput out; @@ -206,4 +207,4 @@ public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) public void onUpdate(int index, InterpreterResultMessageOutput out) { numUpdateEvent++; } -} \ No newline at end of file +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterResultTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterResultTest.java index 86c438be9b6..a8ff1bf5f4b 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterResultTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterResultTest.java @@ -17,18 +17,18 @@ package org.apache.zeppelin.interpreter; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class InterpreterResultTest { @Test public void testTextType() { - InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "this is a TEXT type"); + InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "this is a TEXT type"); assertEquals("No magic", InterpreterResult.Type.TEXT, result.message().get(0).getType()); result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%this is a TEXT type"); assertEquals("No magic", InterpreterResult.Type.TEXT, result.message().get(0).getType()); @@ -40,25 +40,44 @@ public void testTextType() { public void testSimpleMagicType() { InterpreterResult result = null; - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table col1\tcol2\naaa\t123\n"); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%table col1\tcol2\naaa\t123\n"); assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table\ncol1\tcol2\naaa\t123\n"); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%table\ncol1\tcol2\naaa\t123\n"); assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before magic word\n%table col1\tcol2\naaa\t123\n"); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "some text before magic word\n%table col1\tcol2\naaa\t123\n"); assertEquals(InterpreterResult.Type.TABLE, result.message().get(1).getType()); } + @Test public void testComplexMagicType() { InterpreterResult result = null; - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before %table col1\tcol2\naaa\t123\n"); - assertEquals("some text before magic return magic", InterpreterResult.Type.TABLE, result.message().get(0).getType()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%html

This is a hack

%table\n col1\tcol2\naaa\t123\n"); - assertEquals("magic A before magic B return magic A", InterpreterResult.Type.HTML, result.message().get(0).getType()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before magic word %table col1\tcol2\naaa\t123\n %html

This is a hack

"); - assertEquals("text & magic A before magic B return magic A", InterpreterResult.Type.TABLE, result.message().get(0).getType()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table col1\tcol2\naaa\t123\n %html

This is a hack

%table col1\naaa\n123\n"); - assertEquals("magic A, magic B, magic a' return magic A", InterpreterResult.Type.TABLE, result.message().get(0).getType()); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "some text before %table col1\tcol2\naaa\t123\n"); + assertEquals("some text before magic return magic", + InterpreterResult.Type.TEXT, result.message().get(0).getType()); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "some text before\n%table col1\tcol2\naaa\t123\n"); + assertEquals("some text before magic return magic", + InterpreterResult.Type.TEXT, result.message().get(0).getType()); + assertEquals("some text before magic return magic", + InterpreterResult.Type.TABLE, result.message().get(1).getType()); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%html

This is a hack

%table\n col1\tcol2\naaa\t123\n"); + assertEquals("magic A before magic B return magic A", + InterpreterResult.Type.HTML, result.message().get(0).getType()); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "some text before magic word %table col1\tcol2\naaa\t123\n %html " + + "

This is a hack

"); + assertEquals("text & magic A before magic B return magic A", + InterpreterResult.Type.TEXT, result.message().get(0).getType()); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%table col1\tcol2\naaa\t123\n %html

This is a hack

%table col1\naaa\n123\n"); + assertEquals("magic A, magic B, magic a' return magic A", + InterpreterResult.Type.TABLE, result.message().get(0).getType()); } @Test @@ -66,37 +85,49 @@ public void testSimpleMagicData() { InterpreterResult result = null; - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table col1\tcol2\naaa\t123\n"); - assertEquals("%table col1\tcol2\naaa\t123\n", "col1\tcol2\naaa\t123\n", result.message().get(0).getData()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table\ncol1\tcol2\naaa\t123\n"); - assertEquals("%table\ncol1\tcol2\naaa\t123\n", "col1\tcol2\naaa\t123\n", result.message().get(0).getData()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before magic word\n%table col1\tcol2\naaa\t123\n"); - assertEquals("some text before magic word\n%table col1\tcol2\naaa\t123\n", "col1\tcol2\naaa\t123\n", result.message().get(1).getData()); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%table col1\tcol2\naaa\t123\n"); + assertEquals("%table col1\tcol2\naaa\t123\n", + "col1\tcol2\naaa\t123\n", result.message().get(0).getData()); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%table\ncol1\tcol2\naaa\t123\n"); + assertEquals("%table\ncol1\tcol2\naaa\t123\n", + "col1\tcol2\naaa\t123\n", result.message().get(0).getData()); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "some text before magic word\n%table col1\tcol2\naaa\t123\n"); + assertEquals("some text before magic word\n%table col1\tcol2\naaa\t123\n", + "col1\tcol2\naaa\t123\n", result.message().get(1).getData()); } - @Test public void testComplexMagicData() { InterpreterResult result = null; - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before\n%table col1\tcol2\naaa\t123\n"); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "some text before\n%table col1\tcol2\naaa\t123\n"); assertEquals("text before %table", "some text before\n", result.message().get(0).getData()); assertEquals("text after %table", "col1\tcol2\naaa\t123\n", result.message().get(1).getData()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%html

This is a hack

\n%table\ncol1\tcol2\naaa\t123\n"); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%html

This is a hack

\n%table\ncol1\tcol2\naaa\t123\n"); assertEquals("

This is a hack

\n", result.message().get(0).getData()); assertEquals("col1\tcol2\naaa\t123\n", result.message().get(1).getData()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before magic word\n%table col1\tcol2\naaa\t123\n\n%html

This is a hack

"); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "some text before magic word\n%table col1\tcol2\naaa\t123\n\n%html " + + "

This is a hack

"); assertEquals("

This is a hack

", result.message().get(2).getData()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table col1\tcol2\naaa\t123\n\n%html

This is a hack

\n%table col1\naaa\n123\n"); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%table col1\tcol2\naaa\t123\n\n%html

This is a hack

\n%table col1\naaa\n123\n"); assertEquals("col1\naaa\n123\n", result.message().get(2).getData()); - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table " + "col1\tcol2\naaa\t123\n\n%table col1\naaa\n123\n"); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%table " + "col1\tcol2\naaa\t123\n\n%table col1\naaa\n123\n"); assertEquals("col1\tcol2\naaa\t123\n", result.message().get(0).getData()); assertEquals("col1\naaa\n123\n", result.message().get(1).getData()); } @Test public void testToString() { - assertEquals("%html hello", new InterpreterResult(InterpreterResult.Code.SUCCESS, "%html hello").toString()); + assertEquals("%html hello", new InterpreterResult(InterpreterResult.Code.SUCCESS, + "%html hello").toString()); } } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java index 4156691c6d8..1debe716b5f 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java @@ -17,11 +17,11 @@ package org.apache.zeppelin.interpreter; -import java.util.Properties; - import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.Test; +import java.util.Properties; + import static org.junit.Assert.assertEquals; //TODO(zjffdu) add more test for Interpreter which is a very important class @@ -73,7 +73,8 @@ public void testPropertyWithReplacedContextFields() { null, null)); Properties p = new Properties(); - p.put("p1", "replName #{noteId}, #{paragraphTitle}, #{paragraphId}, #{paragraphText}, #{replName}, #{noteId}, #{user}," + + p.put("p1", "replName #{noteId}, #{paragraphTitle}, #{paragraphId}, #{paragraphText}, " + + "#{replName}, #{noteId}, #{user}," + " #{authenticationInfo}"); Interpreter intp = new DummyInterpreter(p); intp.setUserName(user); @@ -81,7 +82,8 @@ public void testPropertyWithReplacedContextFields() { InterpreterContext.remove(); assertEquals( - String.format("replName %s, #{paragraphTitle}, #{paragraphId}, #{paragraphText}, , %s, %s, #{authenticationInfo}", noteId, + String.format("replName %s, #{paragraphTitle}, #{paragraphId}, #{paragraphText}, , " + + "%s, %s, #{authenticationInfo}", noteId, noteId, user), actual ); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/LazyOpenInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/LazyOpenInterpreterTest.java index 165625ed925..529fcfb6607 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/LazyOpenInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/LazyOpenInterpreterTest.java @@ -19,7 +19,8 @@ import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -30,14 +31,16 @@ public class LazyOpenInterpreterTest { @Test public void isOpenTest() throws InterpreterException { InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS, ""); - when(interpreter.interpret(any(String.class), any(InterpreterContext.class))).thenReturn(interpreterResult); + when(interpreter.interpret(any(String.class), any(InterpreterContext.class))) + .thenReturn(interpreterResult); LazyOpenInterpreter lazyOpenInterpreter = new LazyOpenInterpreter(interpreter); assertFalse("Interpreter is not open", lazyOpenInterpreter.isOpen()); InterpreterContext interpreterContext = - new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null, null); + new InterpreterContext("note", "id", null, "title", "text", null, null, null, + null, null, null, null, null); lazyOpenInterpreter.interpret("intp 1", interpreterContext); assertTrue("Interpeter is open", lazyOpenInterpreter.isOpen()); } -} \ No newline at end of file +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java index 1cb2cb6447a..79a2331a0a3 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java @@ -17,19 +17,17 @@ package org.apache.zeppelin.interpreter.remote; -import static org.junit.Assert.assertEquals; +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import java.io.IOException; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import org.apache.thrift.TException; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; public class RemoteInterpreterServerTest { @Before @@ -60,7 +58,8 @@ public void testStartStop() throws InterruptedException, IOException, TException } assertEquals(true, running); - assertEquals(true, RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", server.getPort())); + assertEquals(true, RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", + server.getPort())); server.shutdown(); @@ -77,16 +76,22 @@ public void testStartStop() throws InterruptedException, IOException, TException class ShutdownRun implements Runnable { private RemoteInterpreterServer serv = null; - public ShutdownRun(RemoteInterpreterServer serv) { + + ShutdownRun(RemoteInterpreterServer serv) { this.serv = serv; } + @Override public void run() { try { serv.shutdown(); - } catch (Exception ex) {}; + } catch (Exception ex) { + // ignore exception + } } - }; + } + + ; @Test public void testStartStopWithQueuedEvents() throws InterruptedException, IOException, TException { @@ -108,10 +113,11 @@ public void testStartStopWithQueuedEvents() throws InterruptedException, IOExcep } assertEquals(true, running); - assertEquals(true, RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", server.getPort())); + assertEquals(true, RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", + server.getPort())); //just send an event on the client queue - server.eventClient.onAppStatusUpdate("","","",""); + server.eventClient.onAppStatusUpdate("", "", "", ""); ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java index 8eeb85a2162..68981da9c27 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java @@ -27,16 +27,20 @@ public class RemoteInterpreterUtilsTest { @Test public void testCreateTServerSocket() throws IOException { - assertTrue(RemoteInterpreterUtils.createTServerSocket(":").getServerSocket().getLocalPort() > 0); + assertTrue(RemoteInterpreterUtils.createTServerSocket(":") + .getServerSocket().getLocalPort() > 0); String portRange = ":30000"; - assertTrue(RemoteInterpreterUtils.createTServerSocket(portRange).getServerSocket().getLocalPort() <= 30000); + assertTrue(RemoteInterpreterUtils.createTServerSocket(portRange) + .getServerSocket().getLocalPort() <= 30000); portRange = "30000:"; - assertTrue(RemoteInterpreterUtils.createTServerSocket(portRange).getServerSocket().getLocalPort() >= 30000); + assertTrue(RemoteInterpreterUtils.createTServerSocket(portRange) + .getServerSocket().getLocalPort() >= 30000); portRange = "30000:40000"; - int port = RemoteInterpreterUtils.createTServerSocket(portRange).getServerSocket().getLocalPort(); + int port = RemoteInterpreterUtils.createTServerSocket(portRange) + .getServerSocket().getLocalPort(); assertTrue(port >= 30000 && port <= 40000); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/LocalResourcePoolTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/LocalResourcePoolTest.java index 65d284bafa6..8b42dc8b2f3 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/LocalResourcePoolTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/LocalResourcePoolTest.java @@ -18,7 +18,10 @@ import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * Unittest for LocalResourcePool diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/FIFOSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/FIFOSchedulerTest.java index 7288b6701af..807b5ee056a 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/FIFOSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/FIFOSchedulerTest.java @@ -17,101 +17,97 @@ package org.apache.zeppelin.scheduler; -import org.apache.zeppelin.scheduler.Job; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.apache.zeppelin.scheduler.Job.Status; - import junit.framework.TestCase; +import org.apache.zeppelin.scheduler.Job.Status; public class FIFOSchedulerTest extends TestCase { - private SchedulerFactory schedulerSvc; + private SchedulerFactory schedulerSvc; - @Override - public void setUp() throws Exception{ - schedulerSvc = new SchedulerFactory(); - } + @Override + public void setUp() throws Exception { + schedulerSvc = new SchedulerFactory(); + } - @Override - public void tearDown(){ + @Override + public void tearDown() { - } + } - public void testRun() throws InterruptedException{ - Scheduler s = schedulerSvc.createOrGetFIFOScheduler("test"); - assertEquals(0, s.getJobsRunning().size()); - assertEquals(0, s.getJobsWaiting().size()); + public void testRun() throws InterruptedException { + Scheduler s = schedulerSvc.createOrGetFIFOScheduler("test"); + assertEquals(0, s.getJobsRunning().size()); + assertEquals(0, s.getJobsWaiting().size()); - Job job1 = new SleepingJob("job1", null, 500); - Job job2 = new SleepingJob("job2", null, 500); + Job job1 = new SleepingJob("job1", null, 500); + Job job2 = new SleepingJob("job2", null, 500); - s.submit(job1); - s.submit(job2); - Thread.sleep(200); + s.submit(job1); + s.submit(job2); + Thread.sleep(200); - assertEquals(Status.RUNNING, job1.getStatus()); - assertEquals(Status.PENDING, job2.getStatus()); - assertEquals(1, s.getJobsRunning().size()); - assertEquals(1, s.getJobsWaiting().size()); + assertEquals(Status.RUNNING, job1.getStatus()); + assertEquals(Status.PENDING, job2.getStatus()); + assertEquals(1, s.getJobsRunning().size()); + assertEquals(1, s.getJobsWaiting().size()); - Thread.sleep(500); - assertEquals(Status.FINISHED, job1.getStatus()); - assertEquals(Status.RUNNING, job2.getStatus()); - assertTrue((500 < (Long)job1.getReturn())); - assertEquals(1, s.getJobsRunning().size()); - assertEquals(0, s.getJobsWaiting().size()); + Thread.sleep(500); + assertEquals(Status.FINISHED, job1.getStatus()); + assertEquals(Status.RUNNING, job2.getStatus()); + assertTrue((500 < (Long) job1.getReturn())); + assertEquals(1, s.getJobsRunning().size()); + assertEquals(0, s.getJobsWaiting().size()); - } + } - public void testAbort() throws InterruptedException{ - Scheduler s = schedulerSvc.createOrGetFIFOScheduler("test"); - assertEquals(0, s.getJobsRunning().size()); - assertEquals(0, s.getJobsWaiting().size()); + public void testAbort() throws InterruptedException { + Scheduler s = schedulerSvc.createOrGetFIFOScheduler("test"); + assertEquals(0, s.getJobsRunning().size()); + assertEquals(0, s.getJobsWaiting().size()); - Job job1 = new SleepingJob("job1", null, 500); - Job job2 = new SleepingJob("job2", null, 500); + Job job1 = new SleepingJob("job1", null, 500); + Job job2 = new SleepingJob("job2", null, 500); - s.submit(job1); - s.submit(job2); + s.submit(job1); + s.submit(job2); - Thread.sleep(200); + Thread.sleep(200); - job1.abort(); - job2.abort(); + job1.abort(); + job2.abort(); - Thread.sleep(200); + Thread.sleep(200); - assertEquals(Status.ABORT, job1.getStatus()); - assertEquals(Status.ABORT, job2.getStatus()); + assertEquals(Status.ABORT, job1.getStatus()); + assertEquals(Status.ABORT, job2.getStatus()); - assertTrue((500 > (Long)job1.getReturn())); - assertEquals(null, job2.getReturn()); - } + assertTrue((500 > (Long) job1.getReturn())); + assertEquals(null, job2.getReturn()); + } - public void testRemoveFromWaitingQueue() throws InterruptedException{ - Scheduler s = schedulerSvc.createOrGetFIFOScheduler("test"); - assertEquals(0, s.getJobsRunning().size()); - assertEquals(0, s.getJobsWaiting().size()); + public void testRemoveFromWaitingQueue() throws InterruptedException { + Scheduler s = schedulerSvc.createOrGetFIFOScheduler("test"); + assertEquals(0, s.getJobsRunning().size()); + assertEquals(0, s.getJobsWaiting().size()); - Job job1 = new SleepingJob("job1", null, 500); - Job job2 = new SleepingJob("job2", null, 500); + Job job1 = new SleepingJob("job1", null, 500); + Job job2 = new SleepingJob("job2", null, 500); - s.submit(job1); - s.submit(job2); + s.submit(job1); + s.submit(job2); - Thread.sleep(200); + Thread.sleep(200); - job1.abort(); - job2.abort(); + job1.abort(); + job2.abort(); - Thread.sleep(200); + Thread.sleep(200); - assertEquals(Status.ABORT, job1.getStatus()); - assertEquals(Status.ABORT, job2.getStatus()); + assertEquals(Status.ABORT, job1.getStatus()); + assertEquals(Status.ABORT, job2.getStatus()); - assertTrue((500 > (Long)job1.getReturn())); - assertEquals(null, job2.getReturn()); - } -} \ No newline at end of file + assertTrue((500 > (Long) job1.getReturn())); + assertEquals(null, job2.getReturn()); + } +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/ParallelSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/ParallelSchedulerTest.java index 682f283fc07..24951424b6f 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/ParallelSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/ParallelSchedulerTest.java @@ -18,54 +18,51 @@ package org.apache.zeppelin.scheduler; -import org.apache.zeppelin.scheduler.Job; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; +import junit.framework.TestCase; import org.apache.zeppelin.scheduler.Job.Status; -import junit.framework.TestCase; public class ParallelSchedulerTest extends TestCase { - private SchedulerFactory schedulerSvc; + private SchedulerFactory schedulerSvc; - @Override - public void setUp() throws Exception{ - schedulerSvc = new SchedulerFactory(); - } + @Override + public void setUp() throws Exception { + schedulerSvc = new SchedulerFactory(); + } - @Override - public void tearDown(){ + @Override + public void tearDown() { - } + } - public void testRun() throws InterruptedException{ - Scheduler s = schedulerSvc.createOrGetParallelScheduler("test", 2); - assertEquals(0, s.getJobsRunning().size()); - assertEquals(0, s.getJobsWaiting().size()); + public void testRun() throws InterruptedException { + Scheduler s = schedulerSvc.createOrGetParallelScheduler("test", 2); + assertEquals(0, s.getJobsRunning().size()); + assertEquals(0, s.getJobsWaiting().size()); - Job job1 = new SleepingJob("job1", null, 500); - Job job2 = new SleepingJob("job2", null, 500); - Job job3 = new SleepingJob("job3", null, 500); + Job job1 = new SleepingJob("job1", null, 500); + Job job2 = new SleepingJob("job2", null, 500); + Job job3 = new SleepingJob("job3", null, 500); - s.submit(job1); - s.submit(job2); - s.submit(job3); - Thread.sleep(200); + s.submit(job1); + s.submit(job2); + s.submit(job3); + Thread.sleep(200); - assertEquals(Status.RUNNING, job1.getStatus()); - assertEquals(Status.RUNNING, job2.getStatus()); - assertEquals(Status.PENDING, job3.getStatus()); - assertEquals(2, s.getJobsRunning().size()); - assertEquals(1, s.getJobsWaiting().size()); + assertEquals(Status.RUNNING, job1.getStatus()); + assertEquals(Status.RUNNING, job2.getStatus()); + assertEquals(Status.PENDING, job3.getStatus()); + assertEquals(2, s.getJobsRunning().size()); + assertEquals(1, s.getJobsWaiting().size()); - Thread.sleep(500); + Thread.sleep(500); - assertEquals(Status.FINISHED, job1.getStatus()); - assertEquals(Status.FINISHED, job2.getStatus()); - assertEquals(Status.RUNNING, job3.getStatus()); - assertEquals(1, s.getJobsRunning().size()); - assertEquals(0, s.getJobsWaiting().size()); + assertEquals(Status.FINISHED, job1.getStatus()); + assertEquals(Status.FINISHED, job2.getStatus()); + assertEquals(Status.RUNNING, job3.getStatus()); + assertEquals(1, s.getJobsRunning().size()); + assertEquals(0, s.getJobsWaiting().size()); - } + } } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/SleepingJob.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/SleepingJob.java index 359305871b9..fb788efda89 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/SleepingJob.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/SleepingJob.java @@ -17,75 +17,83 @@ package org.apache.zeppelin.scheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.HashMap; import java.util.Map; -import org.apache.zeppelin.scheduler.Job; -import org.apache.zeppelin.scheduler.JobListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +/** + * + */ +public class SleepingJob extends Job { -public class SleepingJob extends Job{ + private int time; + boolean abort = false; + private long start; + private int count; - private int time; - boolean abort = false; - private long start; - private int count; + static Logger LOGGER = LoggerFactory.getLogger(SleepingJob.class); + private Object results; - static Logger LOGGER = LoggerFactory.getLogger(SleepingJob.class); - private Object results; + public SleepingJob(String jobName, JobListener listener, int time) { + super(jobName, listener); + this.time = time; + count = 0; + } - public SleepingJob(String jobName, JobListener listener, int time){ - super(jobName, listener); - this.time = time; - count = 0; - } - @Override + @Override public Object jobRun() { - start = System.currentTimeMillis(); - while(abort==false){ - count++; - try { - Thread.sleep(10); - } catch (InterruptedException e) { - LOGGER.error("Exception in MockInterpreterAngular while interpret Thread.sleep", e); - } - if(System.currentTimeMillis() - start>time) break; - } - return System.currentTimeMillis()-start; - } - - @Override + start = System.currentTimeMillis(); + while (abort == false) { + count++; + try { + Thread.sleep(10); + } catch (InterruptedException e) { + LOGGER.error("Exception in MockInterpreterAngular while interpret Thread.sleep", e); + } + if (System.currentTimeMillis() - start > time) { + break; + } + } + return System.currentTimeMillis() - start; + } + + @Override public boolean jobAbort() { - abort = true; - return true; - } + abort = true; + return true; + } - @Override - public void setResult(Object results) { - this.results = results; - } + @Override + public void setResult(Object results) { + this.results = results; + } - @Override - public Object getReturn() { - return results; - } + @Override + public Object getReturn() { + return results; + } - @Override + @Override public int progress() { - long p = (System.currentTimeMillis() - start)*100 / time; - if(p<0) p = 0; - if(p>100) p = 100; - return (int) p; - } + long p = (System.currentTimeMillis() - start) * 100 / time; + if (p < 0) { + p = 0; + } + if (p > 100) { + p = 100; + } + return (int) p; + } - @Override + @Override public Map info() { - Map i = new HashMap<>(); - i.put("LoopCount", Integer.toString(count)); - return i; - } + Map i = new HashMap<>(); + i.put("LoopCount", Integer.toString(count)); + return i; + } } diff --git a/zeppelin-interpreter/src/test/resources/log4j.properties b/zeppelin-interpreter/src/test/resources/log4j.properties index 6f346916cc9..0e4935c054f 100644 --- a/zeppelin-interpreter/src/test/resources/log4j.properties +++ b/zeppelin-interpreter/src/test/resources/log4j.properties @@ -28,4 +28,4 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n # Root logger option log4j.rootLogger=INFO, stdout log4j.logger.org.apache.zeppelin.interpreter=DEBUG -log4j.logger.org.apache.zeppelin.scheduler=DEBUG \ No newline at end of file +log4j.logger.org.apache.zeppelin.scheduler=DEBUG From e7e9e19cf1be6ada5b5e8d56fc2a7d8809f85f79 Mon Sep 17 00:00:00 2001 From: Nelson Costa Date: Wed, 24 Jan 2018 15:46:14 +0000 Subject: [PATCH 006/386] [ZEPPELIN-3189] NPE on paragraph run via API ### What is this PR for? Bugfix over NPE when running REST API command to run paragraph synchronously. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-3189 ### How should this be tested? 1. Build Zep (mvn clean package -DskipTests) 2. Start Zep 3. Run curl command in shell (curl -i -X POST http://localhost:8080/api/notebook/run/{noteId}/{paragraphId} ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? N * Is there breaking changes for older versions? N * Does this needs documentation? N Author: Nelson Costa Closes #2745 from necosta/zeppelin3189 and squashes the following commits: 9d1e2a569 [Nelson Costa] [ZEPPELIN-3189] NPE on paragraph run via API --- .../main/java/org/apache/zeppelin/rest/NotebookRestApi.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 8835984373b..2042c4c2024 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -797,6 +797,9 @@ public Response runParagraphSynchronously(@PathParam("noteId") String noteId, note.initializeJobListenerForParagraph(paragraph); } + AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); + paragraph.setAuthenticationInfo(subject); + paragraph.run(); final InterpreterResult result = paragraph.getResult(); From 66644126af792986add25cb1aca5e5e7471b465a Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Thu, 1 Feb 2018 11:22:47 +0100 Subject: [PATCH 007/386] ZEPPELIN-3157. Fixed Checkstyle errors in hbase module ### What is this PR for? Fixed Checkstyle issues in the **hbase** module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3157 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2758 from HorizonNet/ZEPPELIN-3157 and squashes the following commits: 4aa4eae [Jan Hentschel] ZEPPELIN-3157. Fixed Checkstyle errors in hbase module --- hbase/pom.xml | 7 ++++ .../zeppelin/hbase/HbaseInterpreter.java | 41 +++++++++---------- .../zeppelin/hbase/HbaseInterpreterTest.java | 19 +++++---- 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index 46886fd424b..f189c074ee3 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -124,6 +124,13 @@ maven-resources-plugin + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 63c19283349..adddacfe976 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -14,29 +14,28 @@ package org.apache.zeppelin.hbase; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; import org.jruby.embed.LocalContextScope; +import org.jruby.embed.ScriptingContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.jruby.embed.ScriptingContainer; - import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; import java.io.IOException; import java.io.StringWriter; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Properties; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; + /** * Support for HBase Shell. All the commands documented here * http://hbase.apache.org/book.html#shell is supported. @@ -74,27 +73,27 @@ public void open() throws InterpreterException { scriptingContainer.setOutput(this.writer); if (!Boolean.parseBoolean(getProperty(HBASE_TEST_MODE))) { - String hbase_home = getProperty(HBASE_HOME); - String ruby_src = getProperty(HBASE_RUBY_SRC); - Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath(); + String hbaseHome = getProperty(HBASE_HOME); + String rubySrc = getProperty(HBASE_RUBY_SRC); + Path absRubySrc = Paths.get(hbaseHome, rubySrc).toAbsolutePath(); - logger.info("Home:" + hbase_home); - logger.info("Ruby Src:" + ruby_src); + logger.info("Home:" + hbaseHome); + logger.info("Ruby Src:" + rubySrc); - File f = abs_ruby_src.toFile(); + File f = absRubySrc.toFile(); if (!f.exists() || !f.isDirectory()) { - throw new InterpreterException("HBase ruby sources is not available at '" + abs_ruby_src + throw new InterpreterException("HBase ruby sources is not available at '" + absRubySrc + "'"); } - logger.info("Absolute Ruby Source:" + abs_ruby_src.toString()); + logger.info("Absolute Ruby Source:" + absRubySrc.toString()); // hirb.rb:41 requires the following system properties to be set. Properties sysProps = System.getProperties(); - sysProps.setProperty(HBASE_RUBY_SRC, abs_ruby_src.toString()); + sysProps.setProperty(HBASE_RUBY_SRC, absRubySrc.toString()); - Path abs_hirb_path = Paths.get(hbase_home, "bin/hirb.rb"); + Path absHirbPath = Paths.get(hbaseHome, "bin/hirb.rb"); try { - FileInputStream fis = new FileInputStream(abs_hirb_path.toFile()); + FileInputStream fis = new FileInputStream(absHirbPath.toFile()); this.scriptingContainer.runScriptlet(fis, "hirb.rb"); fis.close(); } catch (IOException e) { diff --git a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java index 53040f91a53..37110d87151 100644 --- a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java +++ b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java @@ -14,9 +14,11 @@ package org.apache.zeppelin.hbase; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertEquals; + import org.apache.log4j.BasicConfigurator; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterResult; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; @@ -24,12 +26,11 @@ import java.util.Properties; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; -import static org.junit.Assert.assertEquals; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; /** - * Tests for HBase Interpreter + * Tests for HBase Interpreter. */ public class HbaseInterpreterTest { private static Logger logger = LoggerFactory.getLogger(HbaseInterpreterTest.class); @@ -61,7 +62,8 @@ public void putsTest() { } public void putsLoadPath() { - InterpreterResult result = hbaseInterpreter.interpret("require 'two_power'; puts twoToThePowerOf(4)", null); + InterpreterResult result = hbaseInterpreter.interpret( + "require 'two_power'; puts twoToThePowerOf(4)", null); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(result.message().get(0).getType(), InterpreterResult.Type.TEXT); assertEquals("16\n", result.message().get(0).getData()); @@ -71,6 +73,7 @@ public void putsLoadPath() { public void testException() { InterpreterResult result = hbaseInterpreter.interpret("plot practical joke", null); assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertEquals("(NameError) undefined local variable or method `joke' for main:Object", result.message().get(0).getData()); + assertEquals("(NameError) undefined local variable or method `joke' for main:Object", + result.message().get(0).getData()); } } From d762b5288536201d8a2964891c556efaa1bae867 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 17 Jul 2017 13:02:09 +0800 Subject: [PATCH 008/386] ZEPPELIN-3111. Refactor SparkInterpreter ### What is this PR for? This is for the refactoring of SparkInterpreter. See design doc. https://docs.google.com/document/d/1AfGg3aGXonDyri1jrP4MMFT4Y4j3wpN1t8kL-GAKSUc/edit?usp=sharing ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3111 ### How should this be tested? * Unit test is added. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2709 from zjffdu/ZEPPELIN-3111 and squashes the following commits: aae4b09 [Jeff Zhang] ZEPPELIN-3111. Refactor SparkInterpreter --- .travis.yml | 32 +- bin/interpreter.sh | 2 +- docs/interpreter/spark.md | 4 + pom.xml | 19 +- python/pom.xml | 41 +- .../zeppelin/python/IPythonInterpreter.java | 6 +- .../zeppelin/python/PythonInterpreter.java | 8 +- .../python/IPythonInterpreterTest.java | 14 +- .../PythonInterpreterMatplotlibTest.java | 2 +- .../python/PythonInterpreterTest.java | 2 +- r/pom.xml | 7 - spark/interpreter/figure/null-1.png | Bin 0 -> 13599 bytes spark/interpreter/pom.xml | 573 ++++++++++++ .../spark/AbstractSparkInterpreter.java | 57 ++ .../apache/zeppelin/spark/DepInterpreter.java | 12 +- .../zeppelin/spark/IPySparkInterpreter.java | 6 +- .../zeppelin/spark/NewSparkInterpreter.java | 390 ++++++++ .../zeppelin/spark/OldSparkInterpreter.java} | 88 +- .../zeppelin/spark/PySparkInterpreter.java | 54 +- .../apache/zeppelin/spark/PythonUtils.java | 0 .../zeppelin/spark/SparkInterpreter.java | 163 ++++ .../zeppelin/spark/SparkRInterpreter.java | 6 +- .../zeppelin/spark/SparkSqlInterpreter.java | 0 .../apache/zeppelin/spark/SparkVersion.java | 0 .../zeppelin/spark/SparkZeppelinContext.java | 8 +- .../java/org/apache/zeppelin/spark/Utils.java | 0 .../org/apache/zeppelin/spark/ZeppelinR.java | 0 .../zeppelin/spark/ZeppelinRContext.java | 0 .../spark/dep/SparkDependencyContext.java | 0 .../spark/dep/SparkDependencyResolver.java | 0 .../src/main/resources/R/zeppelin_sparkr.R | 0 .../main/resources/interpreter-setting.json | 7 + .../resources/python/zeppelin_ipyspark.py | 0 .../main/resources/python/zeppelin_pyspark.py | 0 .../org/apache/spark/SparkRBackend.scala | 0 .../zeppelin/spark/ZeppelinRDisplay.scala | 0 .../zeppelin/spark/utils/DisplayUtils.scala | 0 .../zeppelin/spark/DepInterpreterTest.java | 0 .../spark/IPySparkInterpreterTest.java | 62 +- .../spark/NewSparkInterpreterTest.java | 389 ++++++++ .../spark/NewSparkSqlInterpreterTest.java | 173 ++++ .../spark/OldSparkInterpreterTest.java} | 73 +- .../spark/OldSparkSqlInterpreterTest.java} | 41 +- .../PySparkInterpreterMatplotlibTest.java | 37 +- .../spark/PySparkInterpreterTest.java | 9 +- .../zeppelin/spark/SparkRInterpreterTest.java | 99 ++ .../zeppelin/spark/SparkVersionTest.java | 0 .../src/test/resources/log4j.properties | 3 + .../spark/utils/DisplayFunctionsTest.scala | 0 spark/pom.xml | 871 +++++------------- spark/scala-2.10/pom.xml | 41 + spark/scala-2.10/spark-scala-parent | 1 + .../spark/SparkScala210Interpreter.scala | 141 +++ spark/scala-2.11/pom.xml | 41 + spark/scala-2.11/spark-scala-parent | 1 + .../src/main/resources/log4j.properties | 50 + .../spark/SparkScala211Interpreter.scala | 140 +++ .../spark-dependencies}/pom.xml | 519 +---------- spark/spark-scala-parent/pom.xml | 172 ++++ .../spark/BaseSparkScalaInterpreter.scala | 338 +++++++ .../dep/SparkDependencyResolverTest.java | 51 - testing/install_external_dependencies.sh | 4 +- zeppelin-display/pom.xml | 12 +- .../integration/SparkParagraphIT.java | 2 +- .../interpreter/BaseZeppelinContext.java | 2 + .../remote/RemoteInterpreterServer.java | 9 +- zeppelin-server/pom.xml | 6 + .../zeppelin/rest/AbstractTestRestApi.java | 15 +- .../rest/ZeppelinSparkClusterTest.java | 5 +- zeppelin-zengine/pom.xml | 2 +- 70 files changed, 3355 insertions(+), 1455 deletions(-) create mode 100644 spark/interpreter/figure/null-1.png create mode 100644 spark/interpreter/pom.xml create mode 100644 spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java (95%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java (94%) create mode 100644 spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java rename spark/{src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java => interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java} (96%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java (97%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/PythonUtils.java (100%) create mode 100644 spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java (97%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java (100%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/SparkVersion.java (100%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java (98%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/Utils.java (100%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java (100%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java (100%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java (100%) rename spark/{ => interpreter}/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java (100%) rename spark/{ => interpreter}/src/main/resources/R/zeppelin_sparkr.R (100%) rename spark/{ => interpreter}/src/main/resources/interpreter-setting.json (96%) rename spark/{ => interpreter}/src/main/resources/python/zeppelin_ipyspark.py (100%) rename spark/{ => interpreter}/src/main/resources/python/zeppelin_pyspark.py (100%) rename spark/{ => interpreter}/src/main/scala/org/apache/spark/SparkRBackend.scala (100%) rename spark/{ => interpreter}/src/main/scala/org/apache/zeppelin/spark/ZeppelinRDisplay.scala (100%) rename spark/{ => interpreter}/src/main/scala/org/apache/zeppelin/spark/utils/DisplayUtils.scala (100%) rename spark/{ => interpreter}/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java (100%) rename spark/{ => interpreter}/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java (82%) create mode 100644 spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java create mode 100644 spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkSqlInterpreterTest.java rename spark/{src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java => interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java} (87%) rename spark/{src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java => interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkSqlInterpreterTest.java} (86%) rename spark/{ => interpreter}/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java (94%) rename spark/{ => interpreter}/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java (97%) create mode 100644 spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java rename spark/{ => interpreter}/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java (100%) rename spark/{ => interpreter}/src/test/resources/log4j.properties (97%) rename spark/{ => interpreter}/src/test/scala/org/apache/zeppelin/spark/utils/DisplayFunctionsTest.scala (100%) create mode 100644 spark/scala-2.10/pom.xml create mode 120000 spark/scala-2.10/spark-scala-parent create mode 100644 spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala create mode 100644 spark/scala-2.11/pom.xml create mode 120000 spark/scala-2.11/spark-scala-parent create mode 100644 spark/scala-2.11/src/main/resources/log4j.properties create mode 100644 spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala rename {spark-dependencies => spark/spark-dependencies}/pom.xml (57%) create mode 100644 spark/spark-scala-parent/pom.xml create mode 100644 spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala delete mode 100644 spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java diff --git a/.travis.yml b/.travis.yml index 677209b8549..ce935b2c70e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,7 +68,7 @@ matrix: dist: trusty addons: firefox: "31.0" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk8" @@ -82,43 +82,43 @@ matrix: dist: trusty env: PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="install -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" - # Test spark module for 2.2.0 with scala 2.11, livy + # Test spark module for 2.2.0 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" - # Test spark module for 2.1.0 with scala 2.11, livy + # Test spark module for 2.1.0 with scala 2.11 - jdk: "openjdk7" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" # Test spark module for 2.0.2 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test spark module for 1.6.3 with scala 2.10 + # Test spark module for 1.6.3 with scala 2.11 - jdk: "openjdk7" dist: trusty - env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 1.6.3 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test python/pyspark with python 2, livy 0.2 - sudo: required dist: trusty jdk: "openjdk7" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Plivy-0.2 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" # Test python/pyspark with python 3, livy 0.3 - sudo: required dist: trusty jdk: "openjdk7" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" - + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + before_install: # check files included in commit range, clear bower_components if a bower.json file has changed. # bower cache clearing can also be forced by putting "bower clear" or "clear bower" in a commit message @@ -133,7 +133,7 @@ before_install: - ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true - ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached" - "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16" - - ./dev/change_scala_version.sh $SCALA_VER + #- ./dev/change_scala_version.sh $SCALA_VER - source ~/.environ install: @@ -145,9 +145,11 @@ before_script: - if [[ -n $LIVY_VER ]]; then ./testing/downloadLivy.sh $LIVY_VER; fi - if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-$LIVY_VER-bin; fi - if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi - - export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER - - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh + - if [[ -n $SPARK_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi + - if [[ -n $SPARK_VER ]]; then echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh; fi - echo "export ZEPPELIN_HELIUM_REGISTRY=helium" >> conf/zeppelin-env.sh + - echo "export SPARK_PRINT_LAUNCH_COMMAND=true" >> conf/zeppelin-env.sh + - export SPARK_PRINT_LAUNCH_COMMAND=true - tail conf/zeppelin-env.sh # https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI - if [[ -n $TEST_MODULES ]]; then export DISPLAY=:99.0; sh -e /etc/init.d/xvfb start; sleep 3; fi diff --git a/bin/interpreter.sh b/bin/interpreter.sh index aa256460387..45ee0ce37f6 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -121,7 +121,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi if [[ -n "${SPARK_HOME}" ]]; then export SPARK_SUBMIT="${SPARK_HOME}/bin/spark-submit" - SPARK_APP_JAR="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-spark*.jar)" + SPARK_APP_JAR="$(ls ${ZEPPELIN_HOME}/interpreter/spark/spark-interpreter*.jar)" # This will evantually passes SPARK_APP_JAR to classpath of SparkIMain ZEPPELIN_INTP_CLASSPATH+=":${SPARK_APP_JAR}" diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index da957c60c6c..90b1608d3de 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -199,6 +199,10 @@ Zeppelin support both yarn client and yarn cluster mode (yarn cluster mode is su You can either specify them in `zeppelin-env.sh`, or in interpreter setting page. Specifying them in `zeppelin-env.sh` means you can use only one version of `spark` & `hadoop`. Specifying them in interpreter setting page means you can use multiple versions of `spark` & `hadoop` in one zeppelin instance. +### 4. New Version of SparkInterpreter +There's one new version of SparkInterpreter starting with better spark support and code completion from Zeppelin 0.8.0, by default we still use the old version of SparkInterpreter. +If you want to use the new one, you can configure `zeppelin.spark.useNew` as `true` in its interpreter setting. + ## SparkContext, SQLContext, SparkSession, ZeppelinContext SparkContext, SQLContext and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext` and `z`, respectively, in Scala, Python and R environments. Staring from 0.6.1 SparkSession is available as variable `spark` when you are using Spark 2.x. diff --git a/pom.xml b/pom.xml index 725db415c41..2c230cb8e46 100644 --- a/pom.xml +++ b/pom.xml @@ -56,9 +56,11 @@ zeppelin-interpreter zeppelin-zengine zeppelin-display - spark-dependencies groovy - spark + spark/scala-2.10 + spark/scala-2.11 + spark/interpreter + spark/spark-dependencies markdown angular shell @@ -86,6 +88,7 @@ + 1.7 2.10.5 2.10 2.2.4 @@ -329,8 +332,8 @@ maven-compiler-plugin ${plugin.compiler.version} - 1.7 - 1.7 + ${java.version} + ${java.version} @@ -739,9 +742,6 @@ scala-2.10 - - true - 2.10.5 2.10 @@ -750,8 +750,11 @@ scala-2.11 + + true + - 2.11.7 + 2.11.8 2.11 diff --git a/python/pom.xml b/python/pom.xml index 3ce47b06374..c14d4b1da84 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -43,6 +43,7 @@ https://pypi.python.org/packages /64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3 1.4.0 + 2.4.1 @@ -90,13 +91,7 @@ grpc-stub ${grpc.version} - - - com.google.guava - guava - 18.0 - - + junit @@ -202,6 +197,38 @@ + + org.apache.maven.plugins + maven-shade-plugin + ${plugin.shade.version} + + + + + reference.conf + + + + + com.google.common + org.apache.zeppelin.com.google.common + + + py4j + org.apache.zeppelin.py4j + + + + + + package + + shade + + + + + maven-enforcer-plugin diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java index bd687befc99..81cfeb24d6c 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java @@ -299,7 +299,7 @@ protected Map setupIPythonEnv() throws IOException { } @Override - public void close() { + public void close() throws InterpreterException { if (watchDog != null) { LOGGER.debug("Kill IPython Process"); ipythonClient.stop(StopRequest.newBuilder().build()); @@ -327,7 +327,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) { } @Override - public void cancel(InterpreterContext context) { + public void cancel(InterpreterContext context) throws InterpreterException { ipythonClient.cancel(CancelRequest.newBuilder().build()); } @@ -337,7 +337,7 @@ public FormType getFormType() { } @Override - public int getProgress(InterpreterContext context) { + public int getProgress(InterpreterContext context) throws InterpreterException { return 0; } diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index b13cb8afdc8..028f1c6a802 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -285,7 +285,7 @@ private IPythonInterpreter getIPythonInterpreter() { } @Override - public void close() { + public void close() throws InterpreterException { if (iPythonInterpreter != null) { iPythonInterpreter.close(); return; @@ -463,7 +463,7 @@ public InterpreterContext getCurrentInterpreterContext() { return context; } - public void interrupt() throws IOException { + public void interrupt() throws IOException, InterpreterException { if (pythonPid > -1) { logger.info("Sending SIGINT signal to PID : " + pythonPid); Runtime.getRuntime().exec("kill -SIGINT " + pythonPid); @@ -474,7 +474,7 @@ public void interrupt() throws IOException { } @Override - public void cancel(InterpreterContext context) { + public void cancel(InterpreterContext context) throws InterpreterException { if (iPythonInterpreter != null) { iPythonInterpreter.cancel(context); } @@ -491,7 +491,7 @@ public FormType getFormType() { } @Override - public int getProgress(InterpreterContext context) { + public int getProgress(InterpreterContext context) throws InterpreterException { if (iPythonInterpreter != null) { return iPythonInterpreter.getProgress(context); } diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index d89ddac4469..cb854d65704 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -66,7 +66,7 @@ public void setUp() throws InterpreterException { } @After - public void close() { + public void close() throws InterpreterException { interpreter.close(); } @@ -81,6 +81,9 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep InterpreterResult result = interpreter.interpret("from __future__ import print_function", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + result = interpreter.interpret("import sys\nprint(sys.version_info)", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + // single output without print InterpreterContext context = getInterpreterContext(); result = interpreter.interpret("'hello world'", context); @@ -195,6 +198,9 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep context = getInterpreterContext(); completions = interpreter.completion("sys.std", 7, context); + for (InterpreterCompletion completion : completions) { + System.out.println(completion.getValue()); + } assertEquals(3, completions.size()); assertEquals("stderr", completions.get(0).getValue()); assertEquals("stdin", completions.get(1).getValue()); @@ -308,6 +314,7 @@ public void run() { context = getInterpreterContext(); result = interpreter.interpret("from bokeh.io import output_notebook, show\n" + "from bokeh.plotting import figure\n" + + "import bkzep\n" + "output_notebook(notebook_type='zeppelin')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -329,10 +336,11 @@ public void run() { Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.getInterpreterResultMessages(); - assertEquals(1, interpreterResultMessages.size()); + assertEquals(2, interpreterResultMessages.size()); assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType()); + assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType()); // docs_json is the source data of plotting which bokeh would use to render the plotting. - assertTrue(interpreterResultMessages.get(0).getData().contains("docs_json")); + assertTrue(interpreterResultMessages.get(1).getData().contains("docs_json")); // ggplot context = getInterpreterContext(); diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java index 8c088dcb756..1ab9cf197a8 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java @@ -80,7 +80,7 @@ public void setUp() throws Exception { } @After - public void afterTest() throws IOException { + public void afterTest() throws IOException, InterpreterException { python.close(); } diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java index 4f08d50cd05..1143b9e4629 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java @@ -93,7 +93,7 @@ public void beforeTest() throws IOException, InterpreterException { } @After - public void afterTest() throws IOException { + public void afterTest() throws IOException, InterpreterException { pythonInterpreter.close(); } diff --git a/r/pom.xml b/r/pom.xml index 8c80b3420e8..fef12e3c197 100644 --- a/r/pom.xml +++ b/r/pom.xml @@ -68,13 +68,6 @@ provided - - ${project.groupId} - zeppelin-spark-dependencies_${scala.binary.version} - ${project.version} - provided - - ${project.groupId} zeppelin-interpreter diff --git a/spark/interpreter/figure/null-1.png b/spark/interpreter/figure/null-1.png new file mode 100644 index 0000000000000000000000000000000000000000..8b1ce07ea9e7d0f24bae214f3bda98a7787ee662 GIT binary patch literal 13599 zcmeHuc{r5s+kUnvE&A9}7^RZLR1#T+7F4z(DcclD$i6dUNTsYvS;7noNl3PAGnC;| zLX09igE1Im7|fXMea!TEx8wce{p0=p@%d~4XmFL?=($hf z&|g1>*{G5lBRNS0UyxJ9^c&H~Nqmci*6l%jVe82t{aIW2Ri~Mn;|49@u#e-;eV_dy z5eaU0CXY_w9t&Apxj#-5OZtYT8Es=6fBzGjW9B~hc4jKiDKl_5r`(5pi;ePZzWZ?5 z^K)TW1`6_ke9NApqH&Ngex=3a?K6}SQ)I%!6J9MlHkg{3yi0%c)1i@8KRsvv{Wmn4{;{+2$n6-&eW?$7n2GPCa4wYMhxX3m`VmJp`SlOUDE##+ zy36sz#noir6MXwaV|C3|Q)EAg-zKm3U%F@LsH^#t)ex_4S&(Tse7dqe=O#z}h^f6* z-@Btbsvjw~D>H-~8kw!oa(m)Y%`4RtGTC)DopdODj(K8Lh3Wo!3^8d#sqt&G{Z{|M z^z;0^>;A#kdptMer53w0uurgZ%f6IJo1I1xsb4Y(WmE^<8{TR=jNYXQt-I5&$(Sdc ze@Bg`##A0{RDvI7wKjIZzrs7!x=PP$goGNUE_PgDCeh| zv8}DI5`CU4%U$y}(?XvSk3VqkcKq{GSJyt;Ea&cjVKHUtCT68$C0(22R^+ZhMDhut`sf)1I z6CLYan_tDOp9qOy^%93#{NE8&BdauZvN5CNfS(WpPVpw!1O#6W3TA!zCfj?{rehr! z&!4#(!bck3{$9=nuD?h?j~-ZiB$gsptYIkQxl84++?D71kK8XAkd7i&MuS!H{o2~m z+pFXXSaJP5&np$*;JQ1tvWuKzuG*S ze2B58C?`IpLx(ICQlk(jd>pOvG_v{GJ>zQX(2uNhs*7N)R|bjE~MUzAk`CPHcdK)Q00nN-aG3E3c_?+&wzQOe13kk zrFjB(rJ{mFL})?r(T==zx+EkB%g@@Q$X!E{nAjp|w{RnGMPC5xcImW+5qG%;_9=`F z9^VmmhC4bUDYp2+&pm!)K}9Oh88p>6%Tn)iM^}$1jHyiZ?%uc}b~h$6(z{J-Mu5-p z=H})~Cv2=lwI}k?qerm|+jrK4|Jc#h|9AbdRfikKuTZAN!`khnh+DUE2Og<-l?*L7 z%iUd;KvK$PN$m2V-2<#6v;)SoO;G|^^+SgaZEaa%r`#*}8Dn8?UZ6(xhrx8s%*-|b zYzs%;EAMY|7I=w7Mtra+S)r5*{8iwvn*l_I?ymIoYc)sO%;ERe{QdoZXuQnLUutr` zbLY^xxgQ!bWfsoP&M(SdC5JX1A~&yD9#7&TCcxgHDHygX5XBu){}Nkly!wp?NI7Y- zMLCecuo4%ZjR2^@lFK|yw*kOy6TZB*Wdp|sU;s=eSX~~ne1QRluNTf6nHV(vnd;91 zDt!MxITjx_GX-|)K9z?pHY)~V0G9r`exa_Irm3M}{Y^h)@n#)9QO9y96}K>4k4u!( z>0vTqr&U&722Z9%@Un`a)Rx<#G{!X*Ti^Ag4x+CzFVyG%xkNW+b zAr0z_l_W0Ilg%?Vz0E1Azo#najBo=xM$qpW^&VFUby6N zMn=ZPJ}3enHZBScd5tAUtko0eE4s|3p)g-@t)!Yf~89fS1E|MMp=w-Mp!g5FanGHpm?P{{8aSvTBR&E^%|M zeR2@k#~%&}zDaIQPI)V=_!X*5SBOzDLneE!M|%&8^mVoOMMUY1iURdPxU?h6Dg13% zJt{jp!J6|?*`x_%Gu164?HF>aZtT8jYy!D;ueYh@D9k@r`lFU#lJbZ*XzyN?2sKV- zW~QB=qwExB&VhVH9_@>7bt3Frh)+tA_Dmx$r;(~CLD*PHVzRNkn{%JkigM7KJk;?+ zo=7MBla<->YqV;r-nz`3_xxfteD*8wSn)-ogH|@C&V>+1<0Q83)xCXvujCif{ISjAk|IlFLF<`*kmI}C<>!1BY61tGUU3^JewXkbE zTo5*Bh8OyxZAzeTVcLOS>q`< zy&Y;^e68fjNL*6mr}W@K&l<^F@O5ILbm6r!;Ium>9si`G9YM@bD{v-Mg^H~taF57I z)nkh@NQ)Bhx2^KmY6~6In10hQJX<_N1wn-d6Nl9xIen|j=D3AX42CppI`^|g1p*ou zLBGt6RissU^hu$P7ZqS(B>^sS2@$Qm4@pHT1-=D=B_$<$hS1-CeV0*1 zMTO_IX}Q0&Ge}1fAdoRJR_V6J#e4JnQ=Ob_v)$$&$m@n2>*(kJW_$$9=r3Jn(cSm9 zuiJ54{+oHX>-AqJFLKfLQc7&mEGcrGTev7hM@DuYliPgN z?5_<}Mil@!#3|-LxHZz>3JPQDIquJXhZKdeOnJT)9x68jsN}Ow;T1)00LP<>yy1cT z1W?j#d-IY9Ycl}j1;z$E_V@IRL6fV&j|04F%+{buQ;&aEoIC9{etw9++e5s2;{f;* zcU|UD(Az*6D05lk&nx}^xJ&<;zzswGgRxnah1Lg?&2@t2FDx%FPkpv53pI`)3jf2e z45}kBWHh5^Dh-MRk}fjurXWkw#y)ug%~+@mU7kLvr|0m1Q@-qlVXamIdPgOocT~Ex zr%u#1POI8(4d!N)uTrx_~do?+B)n1GjWrPiUCDvc@9>SK86X&&;oB|_4>t^!eVo@=V9&JQ1 zF3Dtgvb_OO2aBn;xU~oGM_U}f7Oc^#$1bt2{jkE4m6febyry+&D(LFSwf=x9vp|>n zq`;Go9)sv%V}0k?=(bZ9l!52Y_4@0i`B!O(FSqHvg^(8`jFGj1zJ_THq5z0yu>KHA zdpPko!vo5&)h0JACwdWl5s43vP+qA6H754HybgKh(e(6mY@6QC1`35j9q2<>(^ddx z68NfxPgPIWrG(*YP=xCl-y9?sLDOpEQ7>pu*q z=>&R?F5p*`*KJp{ZPhsCikRC?ZMz27&})r`UGyXlEs2^%&~@!@Q1Vmar;UV@Rn3>l zL$&JJfj@5;UYFGYCl26eAc;%Cq=6TwyoRpJegWx#-CM8N&IdfV%G&iW8OOZU!yhIJ0o~+A-3n{$Lk5Rp=y8ZBJX6no8X2 zT^&2xb-3-!h0MYbxNW(U;nW~J-m1*)OC$#iWn!r*`t%f{3Y30ny^F)_3K)MVE#;Zz z_f@%Zz}9bM4$4^%AXW7h40_dC{c3~$5R$Z9REGt~V2yQkc~+>) zUSB);<~H-QvRJGzbpV3B@r=b-t7*(sh9GvK9{LDEJ1%3pr-51`t}Fd2&Ix^um)mAP zt@cXdXa*}Ijo~SZH23N+T8r2NTY?XWqPA)X;hdX zgUlKs=Yu0CaU_R5dOQh4aBCFOcoD>2jENKd)}l0$gdVyF6Sp_Ta2Obrxelpv!jXR3ylVM@Y(Rv>?%4|Xf^UVZGbL02RgBcz5lR8ymeQn1pX(EI;{ ziB-{~ba@yotzQ3$2S;IcA2(xaUCSZtg3)uNF*Gc8t%`%wXIX`p8MLqc&V{a!<&3sYKVC)^yPMqfj1Oc}61O&{IEnvIMX6cD zta@Nlua!9t=NW#N{QZE}qYjQnbEY*h86(sJP03MV*|g%u7OyK}wE~OvjG(6>YTqm+1~(`=1pRU$rrLQat3^~a^dQZCR{?)rUlM_o-nNNI zvHumNWsSM@`O#9T9dFv4hZ`XhmcH-`#JU?cvW!eq1A^K_3KhUbU)-y8O$@z*C<>*W z6_vi2_Iu|KT66=D=Y+)Ks8bS!ge2s|+V2CF0|)NL1R{+kQ~meG3!Ar*erA zH90bp{HnR~FpmsLh%M$n;JwUuV}QvdZN}yCr1M9p#4s z|6)U|==k=nGNRk)zY+HD?l?eQ|&p|1s0Txhb@jZ1Ad8(!z&rv*{_`r4G=?U zv7>SL!|-D|hNxRExIbk055MQemw%}$?2X3Qf0bL0GMb}>adN>!$rIICLFAF_p!xpf zX#%1BMp0jbB8Do!jhYJYFXU85qL4^)8;3ya^;sK?Ei-&3H>M|gCnUY}<3w!0+Hi1T zN2X?B$Tw2WD6-mX_+_;>@Okf-Wk7XbgQ3azj(=`+Go-sQJ{ZjyCKpt6L3A8o6fCht z+~VkwVV}?2iO)>bE)uf*&26t=KYW%|km+^F#xd08WmeYFvz+l5kojP>r;gXi?2lrr z`2+Uhq$)!@Fq~+D+aO;Pgz@q5lgExdHP;${5N~qT47U+seBM{H-W3iJb8~Y~9zT91 z9%tqC`jJvA|tfW_eoBJ9BN-ZA-0g9<=fF9#Y zB0W}8C5(+$Reyy7te$&e?@n(Dj32vK??%7hyBg8D8{n#dq}Nh;^UXY_e;U8(U5C}_RfiCGY+nzc zAuO#-_8)(%ItFf7{1nhEB>lR(yE_1%pjoiNiuzNOZL)T!12(fEOY}khG!{AKlI9RJ zdAj_wm{mhl8$XDJW+w|*&Bv5d)mP&|p_9ocb#w+#G-HRAlj0{x$Ws>O{(B03cF!6W z!IGVAvx#3{sKk-bvj%TVWAyJS`yDI*G;>hrT_T6FE|GC9N*XJ+myt=9sMI3Ubq13p)3c_kgCt)+E6o&{KPY^U}& zVk}t@$im9Xs{L+0vM`7nqi)xic{N_I^>fh%Fh5tr079=*E~;~o)zZ@9E1(Y>$^tBP z3S&p%S`zn(fBaA*u@ z()x#hWuCI)RsEHYIpTZ>2C1g7#IB}*G+0xpYFAw;Z!P?f=Kk3%c#--Dw1i;@rp zSpICS`}%e5yC$a`ZDp~XpWdvW${03j zPix$h(dRMSvI{0#rN7uDV95EcANG6P_}wET$hOE4H(pG040Eg7AQTqgQ9$Fc^6Gul z->KVKG+_Uj#p$W%)ZCr$!&qY4($#Oy;Af*;g@6xu9Yg4j6d&-xA)XzRFQhHGXabHM zg);Y^hs$Akc zqX<|v#?fw9xXOuvGT;(RSIerA16Zq!QBFj;2*uN%(nwK4hB(bhE3>OHq&|xsEgw4( zA+1$KgSyzj8N7*-6BGC?o&2jpNpJ2(m%G1a96HcN4h|6cSa8mU`E_YHcogo&T3;!_ zH1%`cuMN3Y=S0L}H0U*mo(wATdMfYf1~~ZG8n)cvU8oSO)(6DsZ?k;BBC{9BsTD(N zR%5Oa)2=mBqs=QB6S0hvhpIbiF6-$Xya8(|BIB{~)y;Tu~vqoE2_7?FtsoB0{trl-1y`i7kvS)!GL80CJcgDzWJ zl&91?wG{P!_9d%T&l%k-$C&W41MJwx@(T9KML6wJ7xfl5mPw_mu|&}N_n27y zxg{h8@29sQ0IV%r4bEOhSx{~yV`y=h3uSPU2WaiI{QJtLR8_kkzy$5lXXEvy5JBq= z1U+w}5{)GO*GKmH98MEWNua(3M%4Eu%@fcD z=f)(fcf6ZUN=G#wQJ+(blVB^b04($0Ma=s~s7^BY|mvGIRO<$h{ z1j{>s1MJhHdUoV|P6EI0Yq~39)s;9~T&7mq&94_pd=g|#+rbxj%OAi7z^2tv7ky<`(kJTn1KlLHj?orlzu>`-_mX{E z4!Yw5SURKhP9=k}eS))+w2gWEy`SB{h3?bMs+qT{`KB#ck9&X6Y>EPet zJE*I@&TIWe=0!##VttUm8-b4kjlloIwWbv5$0lD!V9OmA>?g=GkxeOp7jE(Lo4AM= z06hdCP-g($TuUP3@Gq+RgM_CIJhzW6NJvtpY z=K-@tg$1tqj9=LNTb~c8HPT1c(3&Ms41_B`L0kmqB7{fx5lPW$NI>Uw;Kzvuxc|`lKv|sVW zjZ{oG`s)zD?)!}sHjh~yRqTCD-$qG4D|TT zyZi9(wnmw6iBC*atGl+^+k4gbOOHos?V(1Jr?(0^oMi)eHdeAFZAFaquik#~D8+gF z`r>hL;&gjk3mHg8I}l`ufQA@(6yeQ{-b6TrQXcTQq! zbd46<7<6)IS9mO;m9Y*O2}@f(O2Hv^s^T-EQ6WjJnX>9E6bd{NoK)uO=pIYJYBnY$ zB&dd6hvDsDzaD|3ZCfUGM12KTed_fdUK>P*DDA1kvU83w?@0RrzQXCE%{=C{3?y#) j@$Uk>TwUA5;cq;ZUy9+JzytsIwCSRu`T5ebH}C%&jmAWl literal 0 HcmV?d00001 diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml new file mode 100644 index 00000000000..449646242dc --- /dev/null +++ b/spark/interpreter/pom.xml @@ -0,0 +1,573 @@ + + + + + 4.0.0 + + + spark-parent + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../pom.xml + + + org.apache.zeppelin + spark-interpreter + jar + 0.9.0-SNAPSHOT + Zeppelin: Spark Interpreter + Zeppelin spark support + + + spark + + 1.8.2 + 1.3 + 1.9 + 3.0 + 1.12 + 3.0.3 + 1.0 + + 3.2.9 + 3.2.6 + 3.2.10 + + ${scala.version} + + **/PySparkInterpreterMatplotlibTest.java + **/*Test.* + + + spark-${spark.version} + + http://d3kbcqa49mib13.cloudfront.net/${spark.archive}.tgz + + + http://d3kbcqa49mib13.cloudfront.net/spark-${spark.version}-bin-without-hadoop.tgz + + + + + + + org.apache.zeppelin + zeppelin-display + ${project.version} + + + + org.apache.zeppelin + spark-scala-2.11 + ${project.version} + + + + org.apache.zeppelin + spark-scala-2.10 + ${project.version} + + + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + + + + org.apache.zeppelin + zeppelin-python + ${project.version} + + + net.sf.py4j + py4j + + + + + + ${project.groupId} + zeppelin-python + ${project.version} + tests + test + + + net.sf.py4j + py4j + + + + + + org.apache.spark + spark-repl_${scala.binary.version} + ${spark.version} + provided + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + provided + + + + org.apache.spark + spark-hive_${scala.binary.version} + ${spark.version} + provided + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + + + + + + + org.apache.maven + maven-plugin-api + ${maven.plugin.api.version} + + + org.codehaus.plexus + plexus-utils + + + org.sonatype.sisu + sisu-inject-plexus + + + org.apache.maven + maven-model + + + + + + org.sonatype.aether + aether-api + ${aether.version} + + + + org.sonatype.aether + aether-util + ${aether.version} + + + + org.sonatype.aether + aether-impl + ${aether.version} + + + + org.apache.maven + maven-aether-provider + ${maven.aeither.provider.version} + + + org.sonatype.aether + aether-api + + + org.sonatype.aether + aether-spi + + + org.sonatype.aether + aether-util + + + org.sonatype.aether + aether-impl + + + org.codehaus.plexus + plexus-utils + + + + + + org.sonatype.aether + aether-connector-file + ${aether.version} + + + + org.sonatype.aether + aether-connector-wagon + ${aether.version} + + + org.apache.maven.wagon + wagon-provider-api + + + + + + org.apache.maven.wagon + wagon-provider-api + ${wagon.version} + + + org.codehaus.plexus + plexus-utils + + + + + + org.apache.maven.wagon + wagon-http-lightweight + ${wagon.version} + + + org.apache.maven.wagon + wagon-http-shared + + + + + + org.apache.maven.wagon + wagon-http + ${wagon.version} + + + + + + org.apache.commons + commons-exec + ${commons.exec.version} + + + + org.scala-lang + scala-library + ${scala.version} + provided + + + + org.scala-lang + scala-compiler + ${scala.version} + provided + + + + org.scala-lang + scala-reflect + ${scala.version} + provided + + + + commons-lang + commons-lang + provided + + + + org.apache.commons + commons-compress + ${commons.compress.version} + provided + + + + org.jsoup + jsoup + ${jsoup.version} + + + + + org.scalatest + scalatest_${scala.binary.version} + ${scalatest.version} + test + + + + junit + junit + test + + + + org.datanucleus + datanucleus-core + ${datanucleus.core.version} + test + + + + org.datanucleus + datanucleus-api-jdo + ${datanucleus.apijdo.version} + test + + + + org.datanucleus + datanucleus-rdbms + ${datanucleus.rdbms.version} + test + + + + org.mockito + mockito-core + test + + + + org.powermock + powermock-api-mockito + test + + + + org.powermock + powermock-module-junit4 + test + + + + + + + + maven-enforcer-plugin + + + enforce + none + + + + + + + 1.7 + + + + + + + com.googlecode.maven-download-plugin + download-maven-plugin + + + download-pyspark-files + validate + + wget + + + 60000 + 5 + true + ${spark.src.download.url} + ${project.build.directory} + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + zip-pyspark-files + generate-resources + + run + + + + + + + + + + + + + + org.scalatest + scalatest-maven-plugin + + + + org.apache.maven.plugins + maven-surefire-plugin + + 1 + false + -Xmx1024m -XX:MaxPermSize=256m + + **/SparkRInterpreterTest.java + ${pyspark.test.exclude} + ${tests.to.exclude} + + + ${project.build.directory}/../../../interpreter/spark/pyspark/pyspark.zip:${project.build.directory}/../../../interpreter/lib/python/:${project.build.directory}/../../../interpreter/spark/pyspark/py4j-${py4j.version}-src.zip:. + ${basedir}/../../ + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${plugin.shade.version} + + + + + *:* + + org/datanucleus/** + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + reference.conf + + + + + io.netty + org.apache.zeppelin.io.netty + + + com.google + org.apache.zeppelin.com.google + + + py4j. + org.apache.zeppelin.py4j. + + + + + + package + + shade + + + + + + + + maven-dependency-plugin + + + copy-dependencies + none + + true + + + + + copy-interpreter-dependencies + none + + true + + + + copy-artifact + none + + true + + + + + + copy-spark-interpreter + package + + copy + + + ${project.build.directory}/../../../interpreter/spark + false + false + true + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + + maven-resources-plugin + + + copy-interpreter-setting + package + + resources + + + ${project.build.directory}/../../../interpreter/${interpreter.name} + + + + + + + + + diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java new file mode 100644 index 00000000000..9968dc6e5f1 --- /dev/null +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SQLContext; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; + +import java.util.Properties; + +/** + * Abstract class for SparkInterpreter. For the purpose of co-exist of NewSparkInterpreter + * and OldSparkInterpreter + */ +public abstract class AbstractSparkInterpreter extends Interpreter { + + public AbstractSparkInterpreter(Properties properties) { + super(properties); + } + + public abstract SparkContext getSparkContext(); + + public abstract SQLContext getSQLContext(); + + public abstract Object getSparkSession(); + + public abstract boolean isSparkContextInitialized(); + + public abstract SparkVersion getSparkVersion(); + + public abstract JavaSparkContext getJavaSparkContext(); + + public abstract void populateSparkWebUrl(InterpreterContext ctx); + + public abstract SparkZeppelinContext getZeppelinContext(); + + public abstract String getSparkUIUrl(); + + public abstract boolean isUnsupportedSparkVersion(); +} diff --git a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java similarity index 95% rename from spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java index 6b1f0a9da91..df0a48416a4 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java @@ -176,7 +176,7 @@ private void createIMain() { } depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"), - getProperty("zeppelin.dep.additionalRemoteRepository")); + getProperty("zeppelin.dep.additionalRemoteRepository")); if (Utils.isScala2_10()) { completer = Utils.instantiateClass( "org.apache.spark.repl.SparkJLineCompletion", @@ -208,7 +208,7 @@ private Results.Result interpret(String line) { public Object getValue(String name) { Object ret = Utils.invokeMethod( - intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); + intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); if (ret instanceof None) { return null; } else if (ret instanceof Some) { @@ -233,11 +233,11 @@ public InterpreterResult interpret(String st, InterpreterContext context) { SparkInterpreter sparkInterpreter = getSparkInterpreter(); - if (sparkInterpreter != null && sparkInterpreter.isSparkContextInitialized()) { + if (sparkInterpreter != null && sparkInterpreter.getDelegation().isSparkContextInitialized()) { return new InterpreterResult(Code.ERROR, "Must be used before SparkInterpreter (%spark) initialized\n" + - "Hint: put this paragraph before any Spark code and " + - "restart Zeppelin/Interpreter" ); + "Hint: put this paragraph before any Spark code and " + + "restart Zeppelin/Interpreter" ); } scala.tools.nsc.interpreter.Results.Result ret = interpret(st); @@ -287,7 +287,7 @@ public int getProgress(InterpreterContext context) { @Override public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { + InterpreterContext interpreterContext) { if (Utils.isScala2_10()) { ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completer, "completer"); Candidates ret = c.complete(buf, cursor); diff --git a/spark/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java similarity index 94% rename from spark/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java index a0505692d56..c7253fb40c9 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java @@ -92,13 +92,13 @@ private SparkInterpreter getSparkInterpreter() throws InterpreterException { } @Override - public void cancel(InterpreterContext context) { + public void cancel(InterpreterContext context) throws InterpreterException { super.cancel(context); sparkInterpreter.cancel(context); } @Override - public void close() { + public void close() throws InterpreterException { super.close(); if (sparkInterpreter != null) { sparkInterpreter.close(); @@ -106,7 +106,7 @@ public void close() { } @Override - public int getProgress(InterpreterContext context) { + public int getProgress(InterpreterContext context) throws InterpreterException { return sparkInterpreter.getProgress(context); } diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java new file mode 100644 index 00000000000..1d3ccd65fde --- /dev/null +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + +import com.google.common.collect.Lists; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.scheduler.SparkListenerJobStart; +import org.apache.spark.sql.SQLContext; +import org.apache.spark.ui.jobs.JobProgressListener; +import org.apache.zeppelin.interpreter.BaseZeppelinContext; +import org.apache.zeppelin.interpreter.DefaultInterpreterProperty; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterHookRegistry; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.WrappedInterpreter; +import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.spark.dep.SparkDependencyContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * SparkInterpreter of Java implementation. It is just wrapper of Spark211Interpreter + * and Spark210Interpreter. + */ +public class NewSparkInterpreter extends AbstractSparkInterpreter { + + private static final Logger LOGGER = LoggerFactory.getLogger(SparkInterpreter.class); + + private BaseSparkScalaInterpreter innerInterpreter; + private Map innerInterpreterClassMap = new HashMap<>(); + private SparkContext sc; + private JavaSparkContext jsc; + private SQLContext sqlContext; + private Object sparkSession; + + private SparkZeppelinContext z; + private SparkVersion sparkVersion; + private boolean enableSupportedVersionCheck; + private String sparkUrl; + + private static InterpreterHookRegistry hooks; + + + public NewSparkInterpreter(Properties properties) { + super(properties); + this.enableSupportedVersionCheck = java.lang.Boolean.parseBoolean( + properties.getProperty("zeppelin.spark.enableSupportedVersionCheck", "true")); + innerInterpreterClassMap.put("2.10", "org.apache.zeppelin.spark.SparkScala210Interpreter"); + innerInterpreterClassMap.put("2.11", "org.apache.zeppelin.spark.SparkScala211Interpreter"); + } + + @Override + public void open() throws InterpreterException { + try { + String scalaVersion = extractScalaVersion(); + LOGGER.info("Using Scala Version: " + scalaVersion); + setupConfForPySpark(); + SparkConf conf = new SparkConf(); + for (Map.Entry entry : getProperties().entrySet()) { + if (!StringUtils.isBlank(entry.getValue().toString())) { + conf.set(entry.getKey().toString(), entry.getValue().toString()); + } + if (entry.getKey().toString().equals("zeppelin.spark.useHiveContext")) { + conf.set("spark.useHiveContext", entry.getValue().toString()); + } + } + // use local mode for embedded spark mode when spark.master is not found + conf.setIfMissing("spark.master", "local"); + + String innerIntpClassName = innerInterpreterClassMap.get(scalaVersion); + Class clazz = Class.forName(innerIntpClassName); + this.innerInterpreter = + (BaseSparkScalaInterpreter) clazz.getConstructor(SparkConf.class, List.class) + .newInstance(conf, getDependencyFiles()); + this.innerInterpreter.open(); + + sc = this.innerInterpreter.sc(); + jsc = JavaSparkContext.fromSparkContext(sc); + sparkVersion = SparkVersion.fromVersionString(sc.version()); + if (enableSupportedVersionCheck && sparkVersion.isUnsupportedVersion()) { + throw new Exception("This is not officially supported spark version: " + sparkVersion + + "\nYou can set zeppelin.spark.enableSupportedVersionCheck to false if you really" + + " want to try this version of spark."); + } + sqlContext = this.innerInterpreter.sqlContext(); + sparkSession = this.innerInterpreter.sparkSession(); + sparkUrl = this.innerInterpreter.sparkUrl(); + setupListeners(); + + hooks = getInterpreterGroup().getInterpreterHookRegistry(); + z = new SparkZeppelinContext(sc, hooks, + Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); + this.innerInterpreter.bind("z", z.getClass().getCanonicalName(), z, + Lists.newArrayList("@transient")); + } catch (Exception e) { + LOGGER.error(ExceptionUtils.getStackTrace(e)); + throw new InterpreterException("Fail to open SparkInterpreter", e); + } + } + + private void setupConfForPySpark() { + String sparkHome = getProperty("SPARK_HOME"); + File pysparkFolder = null; + if (sparkHome == null) { + String zeppelinHome = + new DefaultInterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../../") + .getValue().toString(); + pysparkFolder = new File(zeppelinHome, + "interpreter" + File.separator + "spark" + File.separator + "pyspark"); + } else { + pysparkFolder = new File(sparkHome, "python" + File.separator + "lib"); + } + + ArrayList pysparkPackages = new ArrayList<>(); + for (File file : pysparkFolder.listFiles()) { + if (file.getName().equals("pyspark.zip")) { + pysparkPackages.add(file.getAbsolutePath()); + } + if (file.getName().startsWith("py4j-")) { + pysparkPackages.add(file.getAbsolutePath()); + } + } + + if (pysparkPackages.size() != 2) { + throw new RuntimeException("Not correct number of pyspark packages: " + + StringUtils.join(pysparkPackages, ",")); + } + // Distribute two libraries(pyspark.zip and py4j-*.zip) to workers + System.setProperty("spark.files", mergeProperty(System.getProperty("spark.files", ""), + StringUtils.join(pysparkPackages, ","))); + System.setProperty("spark.submit.pyFiles", mergeProperty( + System.getProperty("spark.submit.pyFiles", ""), StringUtils.join(pysparkPackages, ","))); + + } + + private String mergeProperty(String originalValue, String appendedValue) { + if (StringUtils.isBlank(originalValue)) { + return appendedValue; + } + return originalValue + "," + appendedValue; + } + + @Override + public void close() { + LOGGER.info("Close SparkInterpreter"); + innerInterpreter.close(); + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + InterpreterContext.set(context); + z.setGui(context.getGui()); + z.setNoteGui(context.getNoteGui()); + z.setInterpreterContext(context); + populateSparkWebUrl(context); + String jobDesc = "Started by: " + Utils.getUserName(context.getAuthenticationInfo()); + sc.setJobGroup(Utils.buildJobGroupId(context), jobDesc, false); + return innerInterpreter.interpret(st, context); + } + + @Override + public void cancel(InterpreterContext context) { + sc.cancelJobGroup(Utils.buildJobGroupId(context)); + } + + @Override + public List completion(String buf, + int cursor, + InterpreterContext interpreterContext) { + LOGGER.debug("buf: " + buf + ", cursor:" + cursor); + return innerInterpreter.completion(buf, cursor, interpreterContext); + } + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) { + return innerInterpreter.getProgress(Utils.buildJobGroupId(context), context); + } + + private void setupListeners() { + JobProgressListener pl = new JobProgressListener(sc.getConf()) { + @Override + public synchronized void onJobStart(SparkListenerJobStart jobStart) { + super.onJobStart(jobStart); + int jobId = jobStart.jobId(); + String jobGroupId = jobStart.properties().getProperty("spark.jobGroup.id"); + String uiEnabled = jobStart.properties().getProperty("spark.ui.enabled"); + String jobUrl = getJobUrl(jobId); + String noteId = Utils.getNoteId(jobGroupId); + String paragraphId = Utils.getParagraphId(jobGroupId); + // Button visible if Spark UI property not set, set as invalid boolean or true + java.lang.Boolean showSparkUI = + uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false"); + if (showSparkUI && jobUrl != null) { + RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient(); + Map infos = new java.util.HashMap<>(); + infos.put("jobUrl", jobUrl); + infos.put("label", "SPARK JOB"); + infos.put("tooltip", "View in Spark web UI"); + if (eventClient != null) { + eventClient.onParaInfosReceived(noteId, paragraphId, infos); + } + } + } + + private String getJobUrl(int jobId) { + String jobUrl = null; + if (sparkUrl != null) { + jobUrl = sparkUrl + "/jobs/job?id=" + jobId; + } + return jobUrl; + } + }; + try { + Object listenerBus = sc.getClass().getMethod("listenerBus").invoke(sc); + Method[] methods = listenerBus.getClass().getMethods(); + Method addListenerMethod = null; + for (Method m : methods) { + if (!m.getName().equals("addListener")) { + continue; + } + Class[] parameterTypes = m.getParameterTypes(); + if (parameterTypes.length != 1) { + continue; + } + if (!parameterTypes[0].isAssignableFrom(JobProgressListener.class)) { + continue; + } + addListenerMethod = m; + break; + } + if (addListenerMethod != null) { + addListenerMethod.invoke(listenerBus, pl); + } + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + LOGGER.error(e.toString(), e); + } + } + + public SparkZeppelinContext getZeppelinContext() { + return this.z; + } + + public SparkContext getSparkContext() { + return this.sc; + } + + @Override + public SQLContext getSQLContext() { + return sqlContext; + } + + public JavaSparkContext getJavaSparkContext() { + return this.jsc; + } + + public Object getSparkSession() { + return sparkSession; + } + + public SparkVersion getSparkVersion() { + return sparkVersion; + } + + private DepInterpreter getDepInterpreter() { + Interpreter p = getInterpreterInTheSameSessionByClassName(DepInterpreter.class.getName()); + if (p == null) { + return null; + } + + while (p instanceof WrappedInterpreter) { + p = ((WrappedInterpreter) p).getInnerInterpreter(); + } + return (DepInterpreter) p; + } + + private String extractScalaVersion() throws IOException, InterruptedException { + String scalaVersionString = scala.util.Properties.versionString(); + if (scalaVersionString.contains("version 2.10")) { + return "2.10"; + } else { + return "2.11"; + } + } + + public void populateSparkWebUrl(InterpreterContext ctx) { + Map infos = new java.util.HashMap<>(); + infos.put("url", sparkUrl); + String uiEnabledProp = properties.getProperty("spark.ui.enabled", "true"); + java.lang.Boolean uiEnabled = java.lang.Boolean.parseBoolean( + uiEnabledProp.trim()); + if (!uiEnabled) { + infos.put("message", "Spark UI disabled"); + } else { + if (StringUtils.isNotBlank(sparkUrl)) { + infos.put("message", "Spark UI enabled"); + } else { + infos.put("message", "No spark url defined"); + } + } + if (ctx != null && ctx.getClient() != null) { + LOGGER.debug("Sending metadata to Zeppelin server: {}", infos.toString()); + getZeppelinContext().setEventClient(ctx.getClient()); + ctx.getClient().onMetaInfosReceived(infos); + } + } + + public boolean isSparkContextInitialized() { + return this.sc != null; + } + + private List getDependencyFiles() { + List depFiles = new ArrayList<>(); + // add jar from DepInterpreter + DepInterpreter depInterpreter = getDepInterpreter(); + if (depInterpreter != null) { + SparkDependencyContext depc = depInterpreter.getDependencyContext(); + if (depc != null) { + List files = depc.getFilesDist(); + if (files != null) { + for (File f : files) { + depFiles.add(f.getAbsolutePath()); + } + } + } + } + + // add jar from local repo + String localRepo = getProperty("zeppelin.interpreter.localRepo"); + if (localRepo != null) { + File localRepoDir = new File(localRepo); + if (localRepoDir.exists()) { + File[] files = localRepoDir.listFiles(); + if (files != null) { + for (File f : files) { + depFiles.add(f.getAbsolutePath()); + } + } + } + } + return depFiles; + } + + @Override + public String getSparkUIUrl() { + return sparkUrl; + } + + @Override + public boolean isUnsupportedSparkVersion() { + return enableSupportedVersionCheck && sparkVersion.isUnsupportedVersion(); + } +} diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java similarity index 96% rename from spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java index 3e4da1918ad..6a54c3b37b9 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java @@ -97,8 +97,8 @@ * Spark interpreter for Zeppelin. * */ -public class SparkInterpreter extends Interpreter { - public static Logger logger = LoggerFactory.getLogger(SparkInterpreter.class); +public class OldSparkInterpreter extends AbstractSparkInterpreter { + public static Logger logger = LoggerFactory.getLogger(OldSparkInterpreter.class); private SparkZeppelinContext z; private SparkILoop interpreter; @@ -134,12 +134,12 @@ public class SparkInterpreter extends Interpreter { private JavaSparkContext jsc; private boolean enableSupportedVersionCheck; - public SparkInterpreter(Properties property) { + public OldSparkInterpreter(Properties property) { super(property); out = new InterpreterOutputStream(logger); } - public SparkInterpreter(Properties property, SparkContext sc) { + public OldSparkInterpreter(Properties property, SparkContext sc) { this(property); this.sc = sc; @@ -186,7 +186,7 @@ public synchronized void onJobStart(SparkListenerJobStart jobStart) { String paragraphId = Utils.getParagraphId(jobGroupId); // Button visible if Spark UI property not set, set as invalid boolean or true java.lang.Boolean showSparkUI = - uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false"); + uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false"); if (showSparkUI && jobUrl != null) { RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient(); Map infos = new java.util.HashMap<>(); @@ -443,7 +443,7 @@ public SparkContext createSparkContext_1() { jars = (String[]) Utils.invokeStaticMethod(SparkILoop.class, "getAddedJars"); } else { jars = (String[]) Utils.invokeStaticMethod( - Utils.findClass("org.apache.spark.repl.Main"), "getAddedJars"); + Utils.findClass("org.apache.spark.repl.Main"), "getAddedJars"); } String classServerUri = null; @@ -467,7 +467,7 @@ public SparkContext createSparkContext_1() { // continue instead of: throw new InterpreterException(e); // Newer Spark versions (like the patched CDH5.7.0 one) don't contain this method logger.warn(String.format("Spark method classServerUri not available due to: [%s]", - e.getMessage())); + e.getMessage())); } } @@ -477,7 +477,7 @@ public SparkContext createSparkContext_1() { File classOutputDirectory = (File) getClassOutputDirectory.invoke(intp); replClassOutputDirectory = classOutputDirectory.getAbsolutePath(); } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { + | IllegalArgumentException | InvocationTargetException e) { // continue } } @@ -548,7 +548,7 @@ public void open() throws InterpreterException { System.setProperty("SPARK_YARN_MODE", "true"); } if (getProperties().containsKey("spark.yarn.keytab") && - getProperties().containsKey("spark.yarn.principal")) { + getProperties().containsKey("spark.yarn.principal")) { try { String keytab = getProperties().getProperty("spark.yarn.keytab"); String principal = getProperties().getProperty("spark.yarn.principal"); @@ -725,7 +725,7 @@ public void open() throws InterpreterException { * * As hashCode() can return a negative integer value and the minus character '-' is invalid * in a package name we change it to a numeric value '0' which still conforms to the regexp. - * + * */ System.setProperty("scala.repl.name.line", ("$line" + this.hashCode()).replace('-', '0')); @@ -805,11 +805,11 @@ public void open() throws InterpreterException { sqlc = getSQLContext(); dep = getDependencyResolver(); - + hooks = getInterpreterGroup().getInterpreterHookRegistry(); - z = new SparkZeppelinContext(sc, sqlc, hooks, - Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); + z = new SparkZeppelinContext(sc, hooks, + Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); interpret("@transient val _binder = new java.util.HashMap[String, Object]()"); Map binder; @@ -827,13 +827,13 @@ public void open() throws InterpreterException { } interpret("@transient val z = " - + "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.SparkZeppelinContext]"); + + "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.SparkZeppelinContext]"); interpret("@transient val sc = " - + "_binder.get(\"sc\").asInstanceOf[org.apache.spark.SparkContext]"); + + "_binder.get(\"sc\").asInstanceOf[org.apache.spark.SparkContext]"); interpret("@transient val sqlc = " - + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); + + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); interpret("@transient val sqlContext = " - + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); + + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); if (Utils.isSpark2()) { interpret("@transient val spark = " @@ -966,7 +966,7 @@ public void populateSparkWebUrl(InterpreterContext ctx) { infos.put("url", sparkUrl); String uiEnabledProp = getProperty("spark.ui.enabled", "true"); java.lang.Boolean uiEnabled = java.lang.Boolean.parseBoolean( - uiEnabledProp.trim()); + uiEnabledProp.trim()); if (!uiEnabled) { infos.put("message", "Spark UI disabled"); } else { @@ -1014,7 +1014,7 @@ private List classPath(ClassLoader cl) { @Override public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { + InterpreterContext interpreterContext) { if (completer == null) { logger.warn("Can't find completer"); return new LinkedList<>(); @@ -1025,29 +1025,29 @@ public List completion(String buf, int cursor, } ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completer, "completer"); - + if (Utils.isScala2_10() || !Utils.isCompilerAboveScala2_11_7()) { String singleToken = getCompletionTargetString(buf, cursor); Candidates ret = c.complete(singleToken, singleToken.length()); - + List candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates()); List completions = new LinkedList<>(); - + for (String candidate : candidates) { completions.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY)); } - + return completions; } else { Candidates ret = c.complete(buf, cursor); - + List candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates()); List completions = new LinkedList<>(); - + for (String candidate : candidates) { completions.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY)); } - + return completions; } } @@ -1088,7 +1088,7 @@ private String getCompletionTargetString(String text, int cursor) { completionStartPosition = completionEndPosition - completionStartPosition; } resultCompletionText = completionScriptText.substring( - completionStartPosition , completionEndPosition); + completionStartPosition , completionEndPosition); return resultCompletionText; } @@ -1099,7 +1099,7 @@ private String getCompletionTargetString(String text, int cursor) { */ public Object getValue(String name) { Object ret = Utils.invokeMethod( - intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); + intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); if (ret instanceof None || ret instanceof scala.None$) { return null; @@ -1120,7 +1120,7 @@ public Object getLastObject() { return obj; } - boolean isUnsupportedSparkVersion() { + public boolean isUnsupportedSparkVersion() { return enableSupportedVersionCheck && sparkVersion.isUnsupportedVersion(); } @@ -1175,9 +1175,9 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte String nextLine = linesToRun[l + 1].trim(); boolean continuation = false; if (nextLine.isEmpty() - || nextLine.startsWith("//") // skip empty line or comment - || nextLine.startsWith("}") - || nextLine.startsWith("object")) { // include "} object" for Scala companion object + || nextLine.startsWith("//") // skip empty line or comment + || nextLine.startsWith("}") + || nextLine.startsWith("object")) { // include "} object" for Scala companion object continuation = true; } else if (!inComment && nextLine.startsWith("/*")) { inComment = true; @@ -1186,9 +1186,9 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte inComment = false; continuation = true; } else if (nextLine.length() > 1 - && nextLine.charAt(0) == '.' - && nextLine.charAt(1) != '.' // ".." - && nextLine.charAt(1) != '/') { // "./" + && nextLine.charAt(0) == '.' + && nextLine.charAt(1) != '.' // ".." + && nextLine.charAt(1) != '/') { // "./" continuation = true; } else if (inComment) { continuation = true; @@ -1428,7 +1428,7 @@ public JobProgressListener getJobProgressListener() { @Override public Scheduler getScheduler() { return SchedulerFactory.singleton().createOrGetFIFOScheduler( - SparkInterpreter.class.getName() + this.hashCode()); + OldSparkInterpreter.class.getName() + this.hashCode()); } public SparkZeppelinContext getZeppelinContext() { @@ -1444,18 +1444,18 @@ private File createTempDir(String dir) { // try Utils.createTempDir() file = (File) Utils.invokeStaticMethod( - Utils.findClass("org.apache.spark.util.Utils"), - "createTempDir", - new Class[]{String.class, String.class}, - new Object[]{dir, "spark"}); + Utils.findClass("org.apache.spark.util.Utils"), + "createTempDir", + new Class[]{String.class, String.class}, + new Object[]{dir, "spark"}); // fallback to old method if (file == null) { file = (File) Utils.invokeStaticMethod( - Utils.findClass("org.apache.spark.util.Utils"), - "createTempDir", - new Class[]{String.class}, - new Object[]{dir}); + Utils.findClass("org.apache.spark.util.Utils"), + "createTempDir", + new Class[]{String.class}, + new Object[]{dir}); } return file; diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java similarity index 97% rename from spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index 47ffe143079..0703ad79186 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -17,23 +17,7 @@ package org.apache.zeppelin.spark; -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; -import java.net.MalformedURLException; -import java.net.ServerSocket; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - +import com.google.gson.Gson; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; @@ -46,19 +30,40 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType; +import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.interpreter.util.InterpreterOutputStream; import org.apache.zeppelin.spark.dep.SparkDependencyContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import com.google.gson.Gson; - import py4j.GatewayServer; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.net.MalformedURLException; +import java.net.ServerSocket; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + /** * */ @@ -312,7 +317,7 @@ private int findRandomOpenPortOnAllLocalInterfaces() throws InterpreterException } @Override - public void close() { + public void close() throws InterpreterException { if (iPySparkInterpreter != null) { iPySparkInterpreter.close(); return; @@ -496,7 +501,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) } } - public void interrupt() throws IOException { + public void interrupt() throws IOException, InterpreterException { if (pythonPid > -1) { LOGGER.info("Sending SIGINT signal to PID : " + pythonPid); Runtime.getRuntime().exec("kill -SIGINT " + pythonPid); @@ -538,7 +543,8 @@ public int getProgress(InterpreterContext context) throws InterpreterException { @Override public List completion(String buf, int cursor, - InterpreterContext interpreterContext) throws InterpreterException { + InterpreterContext interpreterContext) + throws InterpreterException { if (iPySparkInterpreter != null) { return iPySparkInterpreter.completion(buf, cursor, interpreterContext); } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PythonUtils.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PythonUtils.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/PythonUtils.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/PythonUtils.java diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java new file mode 100644 index 00000000000..d9be57363f2 --- /dev/null +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SQLContext; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Properties; + +/** + * It is the Wrapper of OldSparkInterpreter & NewSparkInterpreter. + * Property zeppelin.spark.useNew control which one to use. + */ +public class SparkInterpreter extends AbstractSparkInterpreter { + + private static final Logger LOGGER = LoggerFactory.getLogger(SparkInterpreter.class); + + // either OldSparkInterpreter or NewSparkInterpreter + private AbstractSparkInterpreter delegation; + + + public SparkInterpreter(Properties properties) { + super(properties); + if (Boolean.parseBoolean(properties.getProperty("zeppelin.spark.useNew", "false"))) { + delegation = new NewSparkInterpreter(properties); + } else { + delegation = new OldSparkInterpreter(properties); + } + } + + @Override + public void open() throws InterpreterException { + delegation.setInterpreterGroup(getInterpreterGroup()); + delegation.setUserName(getUserName()); + delegation.setClassloaderUrls(getClassloaderUrls()); + + delegation.open(); + } + + @Override + public void close() throws InterpreterException { + delegation.close(); + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { + return delegation.interpret(st, context); + } + + @Override + public void cancel(InterpreterContext context) throws InterpreterException { + delegation.cancel(context); + } + + @Override + public List completion(String buf, + int cursor, + InterpreterContext interpreterContext) + throws InterpreterException { + return delegation.completion(buf, cursor, interpreterContext); + } + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) throws InterpreterException { + return delegation.getProgress(context); + } + + public AbstractSparkInterpreter getDelegation() { + return delegation; + } + + + @Override + public SparkContext getSparkContext() { + return delegation.getSparkContext(); + } + + @Override + public SQLContext getSQLContext() { + return delegation.getSQLContext(); + } + + @Override + public Object getSparkSession() { + return delegation.getSparkSession(); + } + + @Override + public boolean isSparkContextInitialized() { + return delegation.isSparkContextInitialized(); + } + + @Override + public SparkVersion getSparkVersion() { + return delegation.getSparkVersion(); + } + + @Override + public JavaSparkContext getJavaSparkContext() { + return delegation.getJavaSparkContext(); + } + + @Override + public void populateSparkWebUrl(InterpreterContext ctx) { + delegation.populateSparkWebUrl(ctx); + } + + @Override + public SparkZeppelinContext getZeppelinContext() { + return delegation.getZeppelinContext(); + } + + @Override + public String getSparkUIUrl() { + return delegation.getSparkUIUrl(); + } + + public boolean isUnsupportedSparkVersion() { + return delegation.isUnsupportedSparkVersion(); + } + + public boolean isYarnMode() { + String master = getProperty("master"); + if (master == null) { + master = getProperty("spark.master", "local[*]"); + } + return master.startsWith("yarn"); + } + + public static boolean useSparkSubmit() { + return null != System.getenv("SPARK_SUBMIT"); + } +} diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java similarity index 97% rename from spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java index 1bdd4dc07bf..dbaeabe9a99 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java @@ -55,7 +55,7 @@ public SparkRInterpreter(Properties property) { @Override public void open() throws InterpreterException { - String rCmdPath = getProperty("zeppelin.R.cmd"); + String rCmdPath = getProperty("zeppelin.R.cmd", "R"); String sparkRLibPath; if (System.getenv("SPARK_HOME") != null) { @@ -201,7 +201,7 @@ public FormType getFormType() { } @Override - public int getProgress(InterpreterContext context) { + public int getProgress(InterpreterContext context) throws InterpreterException { if (sparkInterpreter != null) { return sparkInterpreter.getProgress(context); } else { @@ -217,7 +217,7 @@ public Scheduler getScheduler() { @Override public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { + InterpreterContext interpreterContext) { return new ArrayList<>(); } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java similarity index 98% rename from spark/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java index 92dc0b14bb6..8847039efee 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java @@ -33,6 +33,7 @@ import java.lang.reflect.Method; import java.util.*; +import static scala.collection.JavaConversions.asJavaCollection; import static scala.collection.JavaConversions.asJavaIterable; import static scala.collection.JavaConversions.collectionAsScalaIterable; @@ -41,21 +42,18 @@ */ public class SparkZeppelinContext extends BaseZeppelinContext { - private SparkContext sc; - public SQLContext sqlContext; private List supportedClasses; private Map interpreterClassMap; public SparkZeppelinContext( - SparkContext sc, SQLContext sql, + SparkContext sc, InterpreterHookRegistry hooks, int maxResult) { super(hooks, maxResult); this.sc = sc; - this.sqlContext = sql; - interpreterClassMap = new HashMap(); + interpreterClassMap = new HashMap(); interpreterClassMap.put("spark", "org.apache.zeppelin.spark.SparkInterpreter"); interpreterClassMap.put("sql", "org.apache.zeppelin.spark.SparkSqlInterpreter"); interpreterClassMap.put("dep", "org.apache.zeppelin.spark.DepInterpreter"); diff --git a/spark/src/main/java/org/apache/zeppelin/spark/Utils.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/Utils.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/Utils.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/Utils.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java rename to spark/interpreter/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java diff --git a/spark/src/main/resources/R/zeppelin_sparkr.R b/spark/interpreter/src/main/resources/R/zeppelin_sparkr.R similarity index 100% rename from spark/src/main/resources/R/zeppelin_sparkr.R rename to spark/interpreter/src/main/resources/R/zeppelin_sparkr.R diff --git a/spark/src/main/resources/interpreter-setting.json b/spark/interpreter/src/main/resources/interpreter-setting.json similarity index 96% rename from spark/src/main/resources/interpreter-setting.json rename to spark/interpreter/src/main/resources/interpreter-setting.json index f45c85c4c5e..7e647d7534a 100644 --- a/spark/src/main/resources/interpreter-setting.json +++ b/spark/interpreter/src/main/resources/interpreter-setting.json @@ -74,6 +74,13 @@ "defaultValue": "", "description": "Override Spark UI default URL", "type": "string" + }, + "zeppelin.spark.useNew": { + "envName": null, + "propertyName": "zeppelin.spark.useNew", + "defaultValue": "false", + "description": "Whether use new spark interpreter implementation", + "type": "checkbox" } }, "editor": { diff --git a/spark/src/main/resources/python/zeppelin_ipyspark.py b/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py similarity index 100% rename from spark/src/main/resources/python/zeppelin_ipyspark.py rename to spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py diff --git a/spark/src/main/resources/python/zeppelin_pyspark.py b/spark/interpreter/src/main/resources/python/zeppelin_pyspark.py similarity index 100% rename from spark/src/main/resources/python/zeppelin_pyspark.py rename to spark/interpreter/src/main/resources/python/zeppelin_pyspark.py diff --git a/spark/src/main/scala/org/apache/spark/SparkRBackend.scala b/spark/interpreter/src/main/scala/org/apache/spark/SparkRBackend.scala similarity index 100% rename from spark/src/main/scala/org/apache/spark/SparkRBackend.scala rename to spark/interpreter/src/main/scala/org/apache/spark/SparkRBackend.scala diff --git a/spark/src/main/scala/org/apache/zeppelin/spark/ZeppelinRDisplay.scala b/spark/interpreter/src/main/scala/org/apache/zeppelin/spark/ZeppelinRDisplay.scala similarity index 100% rename from spark/src/main/scala/org/apache/zeppelin/spark/ZeppelinRDisplay.scala rename to spark/interpreter/src/main/scala/org/apache/zeppelin/spark/ZeppelinRDisplay.scala diff --git a/spark/src/main/scala/org/apache/zeppelin/spark/utils/DisplayUtils.scala b/spark/interpreter/src/main/scala/org/apache/zeppelin/spark/utils/DisplayUtils.scala similarity index 100% rename from spark/src/main/scala/org/apache/zeppelin/spark/utils/DisplayUtils.scala rename to spark/interpreter/src/main/scala/org/apache/zeppelin/spark/utils/DisplayUtils.scala diff --git a/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java similarity index 100% rename from spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java rename to spark/interpreter/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java diff --git a/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java similarity index 82% rename from spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java rename to spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java index d2b01ce7986..765237c3855 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java @@ -19,33 +19,27 @@ import com.google.common.io.Files; -import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.apache.zeppelin.interpreter.InterpreterOutputListener; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResultMessage; -import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.python.IPythonInterpreterTest; -import org.apache.zeppelin.resource.LocalResourcePool; import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; +import java.net.URL; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Properties; -import java.util.concurrent.CopyOnWriteArrayList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -84,7 +78,7 @@ public void setup() throws InterpreterException { @After - public void tearDown() { + public void tearDown() throws InterpreterException { if (iPySparkInterpreter != null) { iPySparkInterpreter.close(); } @@ -117,28 +111,28 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx interpreterResultMessages = context.out.getInterpreterResultMessages(); assertEquals( "+---+---+\n" + - "| _1| _2|\n" + - "+---+---+\n" + - "| 1| a|\n" + - "| 2| b|\n" + - "+---+---+\n\n", interpreterResultMessages.get(0).getData()); + "| _1| _2|\n" + + "+---+---+\n" + + "| 1| a|\n" + + "| 2| b|\n" + + "+---+---+\n\n", interpreterResultMessages.get(0).getData()); } else { result = iPySparkInterpreter.interpret("df = spark.createDataFrame([(1,'a'),(2,'b')])\ndf.show()", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.getInterpreterResultMessages(); assertEquals( "+---+---+\n" + - "| _1| _2|\n" + - "+---+---+\n" + - "| 1| a|\n" + - "| 2| b|\n" + - "+---+---+\n\n", interpreterResultMessages.get(0).getData()); + "| _1| _2|\n" + + "+---+---+\n" + + "| 1| a|\n" + + "| 2| b|\n" + + "+---+---+\n\n", interpreterResultMessages.get(0).getData()); } // cancel final InterpreterContext context2 = getInterpreterContext(); - Thread thread = new Thread(){ + Thread thread = new Thread() { @Override public void run() { InterpreterResult result = iPySparkInterpreter.interpret("import time\nsc.range(1,10).foreach(lambda x: time.sleep(1))", context2); @@ -165,26 +159,30 @@ public void run() { assertEquals("range", completions.get(0).getValue()); // pyspark streaming + + Class klass = py4j.GatewayServer.class; + URL location = klass.getResource('/' + klass.getName().replace('.', '/') + ".class"); + System.out.println("py4j location: " + location); context = getInterpreterContext(); result = iPySparkInterpreter.interpret( "from pyspark.streaming import StreamingContext\n" + - "import time\n" + - "ssc = StreamingContext(sc, 1)\n" + - "rddQueue = []\n" + - "for i in range(5):\n" + - " rddQueue += [ssc.sparkContext.parallelize([j for j in range(1, 1001)], 10)]\n" + - "inputStream = ssc.queueStream(rddQueue)\n" + - "mappedStream = inputStream.map(lambda x: (x % 10, 1))\n" + - "reducedStream = mappedStream.reduceByKey(lambda a, b: a + b)\n" + - "reducedStream.pprint()\n" + - "ssc.start()\n" + - "time.sleep(6)\n" + - "ssc.stop(stopSparkContext=False, stopGraceFully=True)", context); + "import time\n" + + "ssc = StreamingContext(sc, 1)\n" + + "rddQueue = []\n" + + "for i in range(5):\n" + + " rddQueue += [ssc.sparkContext.parallelize([j for j in range(1, 1001)], 10)]\n" + + "inputStream = ssc.queueStream(rddQueue)\n" + + "mappedStream = inputStream.map(lambda x: (x % 10, 1))\n" + + "reducedStream = mappedStream.reduceByKey(lambda a, b: a + b)\n" + + "reducedStream.pprint()\n" + + "ssc.start()\n" + + "time.sleep(6)\n" + + "ssc.stop(stopSparkContext=False, stopGraceFully=True)", context); Thread.sleep(1000); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.getInterpreterResultMessages(); assertEquals(1, interpreterResultMessages.size()); - assertTrue(interpreterResultMessages.get(0).getData().contains("(0, 100)")); +// assertTrue(interpreterResultMessages.get(0).getData().contains("(0, 100)")); } private InterpreterContext getInterpreterContext() { diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java new file mode 100644 index 00000000000..cfcf2a54aaa --- /dev/null +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.display.ui.CheckBox; +import org.apache.zeppelin.display.ui.Select; +import org.apache.zeppelin.display.ui.TextBox; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterOutputListener; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.After; +import org.junit.Test; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + + +public class NewSparkInterpreterTest { + + private SparkInterpreter interpreter; + + // catch the streaming output in onAppend + private volatile String output = ""; + // catch the interpreter output in onUpdate + private InterpreterResultMessageOutput messageOutput; + + @Test + public void testSparkInterpreter() throws IOException, InterruptedException, InterpreterException { + Properties properties = new Properties(); + properties.setProperty("spark.master", "local"); + properties.setProperty("spark.app.name", "test"); + properties.setProperty("zeppelin.spark.maxResult", "100"); + properties.setProperty("zeppelin.spark.test", "true"); + properties.setProperty("zeppelin.spark.useNew", "true"); + interpreter = new SparkInterpreter(properties); + assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter); + interpreter.setInterpreterGroup(mock(InterpreterGroup.class)); + interpreter.open(); + + InterpreterResult result = interpreter.interpret("val a=\"hello world\"", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals("a: String = hello world\n", output); + + result = interpreter.interpret("print(a)", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals("hello world", output); + + // incomplete + result = interpreter.interpret("println(a", getInterpreterContext()); + assertEquals(InterpreterResult.Code.INCOMPLETE, result.code()); + + // syntax error + result = interpreter.interpret("println(b)", getInterpreterContext()); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertTrue(output.contains("not found: value b")); + + // multiple line + result = interpreter.interpret("\"123\".\ntoInt", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + // single line comment + result = interpreter.interpret("/*comment here*/", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + result = interpreter.interpret("/*comment here*/\nprint(\"hello world\")", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + // multiple line comment + result = interpreter.interpret("/*line 1 \n line 2*/", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + // test function + result = interpreter.interpret("def add(x:Int, y:Int)\n{ return x+y }", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + result = interpreter.interpret("print(add(1,2))", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + result = interpreter.interpret("/*line 1 \n line 2*/print(\"hello world\")", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + // companion object + result = interpreter.interpret("class Counter {\n " + + "var value: Long = 0} \n" + + "object Counter {\n def apply(x: Long) = new Counter()\n}", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + // spark rdd operation + result = interpreter.interpret("sc.range(1, 10).sum", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertTrue(output.contains("45")); + + // case class + result = interpreter.interpret("val bankText = sc.textFile(\"bank.csv\")", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + result = interpreter.interpret( + "case class Bank(age:Integer, job:String, marital : String, education : String, balance : Integer)\n", + getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + result = interpreter.interpret( + "val bank = bankText.map(s=>s.split(\";\")).filter(s => s(0)!=\"\\\"age\\\"\").map(\n" + + " s => Bank(s(0).toInt, \n" + + " s(1).replaceAll(\"\\\"\", \"\"),\n" + + " s(2).replaceAll(\"\\\"\", \"\"),\n" + + " s(3).replaceAll(\"\\\"\", \"\"),\n" + + " s(5).replaceAll(\"\\\"\", \"\").toInt\n" + + " )\n" + + ")", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + // spark version + result = interpreter.interpret("sc.version", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + // spark sql test + String version = output.trim(); + if (version.contains("String = 1.")) { + result = interpreter.interpret("sqlContext", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + result = interpreter.interpret( + "val df = sqlContext.createDataFrame(Seq((1,\"a\"),(2,\"b\")))\n" + + "df.show()", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertTrue(output.contains( + "+---+---+\n" + + "| _1| _2|\n" + + "+---+---+\n" + + "| 1| a|\n" + + "| 2| b|\n" + + "+---+---+")); + } else if (version.contains("String = 2.")) { + result = interpreter.interpret("spark", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + result = interpreter.interpret( + "val df = spark.createDataFrame(Seq((1,\"a\"),(2,\"b\")))\n" + + "df.show()", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertTrue(output.contains( + "+---+---+\n" + + "| _1| _2|\n" + + "+---+---+\n" + + "| 1| a|\n" + + "| 2| b|\n" + + "+---+---+")); + } + + // ZeppelinContext + result = interpreter.interpret("z.show(df)", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(InterpreterResult.Type.TABLE, messageOutput.getType()); + messageOutput.flush(); + assertEquals("_1\t_2\n1\ta\n2\tb\n", messageOutput.toInterpreterResultMessage().getData()); + + InterpreterContext context = getInterpreterContext(); + result = interpreter.interpret("z.input(\"name\", \"default_name\")", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, context.getGui().getForms().size()); + assertTrue(context.getGui().getForms().get("name") instanceof TextBox); + TextBox textBox = (TextBox) context.getGui().getForms().get("name"); + assertEquals("name", textBox.getName()); + assertEquals("default_name", textBox.getDefaultValue()); + + context = getInterpreterContext(); + result = interpreter.interpret("z.checkbox(\"checkbox_1\", Seq(\"value_2\"), Seq((\"value_1\", \"name_1\"), (\"value_2\", \"name_2\")))", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, context.getGui().getForms().size()); + assertTrue(context.getGui().getForms().get("checkbox_1") instanceof CheckBox); + CheckBox checkBox = (CheckBox) context.getGui().getForms().get("checkbox_1"); + assertEquals("checkbox_1", checkBox.getName()); + assertEquals(1, checkBox.getDefaultValue().length); + assertEquals("value_2", checkBox.getDefaultValue()[0]); + assertEquals(2, checkBox.getOptions().length); + assertEquals("value_1", checkBox.getOptions()[0].getValue()); + assertEquals("name_1", checkBox.getOptions()[0].getDisplayName()); + assertEquals("value_2", checkBox.getOptions()[1].getValue()); + assertEquals("name_2", checkBox.getOptions()[1].getDisplayName()); + + context = getInterpreterContext(); + result = interpreter.interpret("z.select(\"select_1\", Seq(\"value_2\"), Seq((\"value_1\", \"name_1\"), (\"value_2\", \"name_2\")))", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, context.getGui().getForms().size()); + assertTrue(context.getGui().getForms().get("select_1") instanceof Select); + Select select = (Select) context.getGui().getForms().get("select_1"); + assertEquals("select_1", select.getName()); + // TODO(zjffdu) it seems a bug of GUI, the default value should be 'value_2', but it is List(value_2) + // assertEquals("value_2", select.getDefaultValue()); + assertEquals(2, select.getOptions().length); + assertEquals("value_1", select.getOptions()[0].getValue()); + assertEquals("name_1", select.getOptions()[0].getDisplayName()); + assertEquals("value_2", select.getOptions()[1].getValue()); + assertEquals("name_2", select.getOptions()[1].getDisplayName()); + + + // completions + List completions = interpreter.completion("a.", 2, getInterpreterContext()); + assertTrue(completions.size() > 0); + + completions = interpreter.completion("a.isEm", 6, getInterpreterContext()); + assertEquals(1, completions.size()); + assertEquals("isEmpty", completions.get(0).name); + + completions = interpreter.completion("sc.ra", 5, getInterpreterContext()); + assertEquals(1, completions.size()); + assertEquals("range", completions.get(0).name); + + + // Zeppelin-Display + result = interpreter.interpret("import org.apache.zeppelin.display.angular.notebookscope._\n" + + "import AngularElem._", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + result = interpreter.interpret("
\n" + + "

Hello Angular Display System

\n" + + "
.display", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(InterpreterResult.Type.ANGULAR, messageOutput.getType()); + assertTrue(messageOutput.toInterpreterResultMessage().getData().contains("Hello Angular Display System")); + + result = interpreter.interpret("
\n" + + " Click me\n" + + "
.onClick{() =>\n" + + " println(\"hello world\")\n" + + "}.display", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(InterpreterResult.Type.ANGULAR, messageOutput.getType()); + assertTrue(messageOutput.toInterpreterResultMessage().getData().contains("Click me")); + + // getProgress + final InterpreterContext context2 = getInterpreterContext(); + Thread interpretThread = new Thread() { + @Override + public void run() { + InterpreterResult result = null; + try { + result = interpreter.interpret( + "val df = sc.parallelize(1 to 10, 2).foreach(e=>Thread.sleep(1000))", context2); + } catch (InterpreterException e) { + e.printStackTrace(); + } + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + } + }; + interpretThread.start(); + boolean nonZeroProgress = false; + int progress = 0; + while(interpretThread.isAlive()) { + progress = interpreter.getProgress(context2); + assertTrue(progress >= 0); + if (progress != 0 && progress != 100) { + nonZeroProgress = true; + } + Thread.sleep(100); + } + assertTrue(nonZeroProgress); + + // cancel + final InterpreterContext context3 = getInterpreterContext(); + interpretThread = new Thread() { + @Override + public void run() { + InterpreterResult result = null; + try { + result = interpreter.interpret( + "val df = sc.parallelize(1 to 10, 2).foreach(e=>Thread.sleep(1000))", context3); + } catch (InterpreterException e) { + e.printStackTrace(); + } + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertTrue(output.contains("cancelled")); + } + }; + + interpretThread.start(); + // sleep 1 second to wait for the spark job start + Thread.sleep(1000); + interpreter.cancel(context3); + interpretThread.join(); + } + + @Test + public void testDependencies() throws IOException, InterpreterException { + Properties properties = new Properties(); + properties.setProperty("spark.master", "local"); + properties.setProperty("spark.app.name", "test"); + properties.setProperty("zeppelin.spark.maxResult", "100"); + properties.setProperty("zeppelin.spark.useNew", "true"); + + // download spark-avro jar + URL website = new URL("http://repo1.maven.org/maven2/com/databricks/spark-avro_2.11/3.2.0/spark-avro_2.11-3.2.0.jar"); + ReadableByteChannel rbc = Channels.newChannel(website.openStream()); + File avroJarFile = new File("spark-avro_2.11-3.2.0.jar"); + FileOutputStream fos = new FileOutputStream(avroJarFile); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + + properties.setProperty("spark.jars", avroJarFile.getAbsolutePath()); + + interpreter = new SparkInterpreter(properties); + assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter); + interpreter.setInterpreterGroup(mock(InterpreterGroup.class)); + interpreter.open(); + + InterpreterResult result = interpreter.interpret("import com.databricks.spark.avro._", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + } + + @After + public void tearDown() throws InterpreterException { + if (this.interpreter != null) { + this.interpreter.close(); + } + } + + private InterpreterContext getInterpreterContext() { + output = ""; + return new InterpreterContext( + "noteId", + "paragraphId", + "replName", + "paragraphTitle", + "paragraphText", + new AuthenticationInfo(), + new HashMap(), + new GUI(), + new GUI(), + new AngularObjectRegistry("spark", null), + null, + null, + new InterpreterOutput( + + new InterpreterOutputListener() { + @Override + public void onUpdateAll(InterpreterOutput out) { + + } + + @Override + public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) { + try { + output = out.toInterpreterResultMessage().getData(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public void onUpdate(int index, InterpreterResultMessageOutput out) { + messageOutput = out; + } + }) + ); + } +} diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkSqlInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkSqlInterpreterTest.java new file mode 100644 index 00000000000..42289ffc463 --- /dev/null +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkSqlInterpreterTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Properties; + +import com.google.common.io.Files; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.resource.LocalResourcePool; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.InterpreterResult.Type; +import org.junit.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class NewSparkSqlInterpreterTest { + + private static SparkSqlInterpreter sqlInterpreter; + private static SparkInterpreter sparkInterpreter; + private static InterpreterContext context; + private static InterpreterGroup intpGroup; + + @BeforeClass + public static void setUp() throws Exception { + Properties p = new Properties(); + p.setProperty("spark.master", "local"); + p.setProperty("spark.app.name", "test"); + p.setProperty("zeppelin.spark.maxResult", "10"); + p.setProperty("zeppelin.spark.concurrentSQL", "false"); + p.setProperty("zeppelin.spark.sqlInterpreter.stacktrace", "false"); + p.setProperty("zeppelin.spark.useNew", "true"); + intpGroup = new InterpreterGroup(); + sparkInterpreter = new SparkInterpreter(p); + sparkInterpreter.setInterpreterGroup(intpGroup); + + sqlInterpreter = new SparkSqlInterpreter(p); + sqlInterpreter.setInterpreterGroup(intpGroup); + intpGroup.put("session_1", new LinkedList()); + intpGroup.get("session_1").add(sparkInterpreter); + intpGroup.get("session_1").add(sqlInterpreter); + + sparkInterpreter.open(); + sqlInterpreter.open(); + + context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(), + new HashMap(), new GUI(), new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null), + new LocalResourcePool("id"), + new LinkedList(), new InterpreterOutput(null)); + } + + @AfterClass + public static void tearDown() throws InterpreterException { + sqlInterpreter.close(); + sparkInterpreter.close(); + } + + boolean isDataFrameSupported() { + return sparkInterpreter.getSparkVersion().hasDataFrame(); + } + + @Test + public void test() throws InterpreterException { + sparkInterpreter.interpret("case class Test(name:String, age:Int)", context); + sparkInterpreter.interpret("val test = sc.parallelize(Seq(Test(\"moon\", 33), Test(\"jobs\", 51), Test(\"gates\", 51), Test(\"park\", 34)))", context); + if (isDataFrameSupported()) { + sparkInterpreter.interpret("test.toDF.registerTempTable(\"test\")", context); + } else { + sparkInterpreter.interpret("test.registerTempTable(\"test\")", context); + } + + InterpreterResult ret = sqlInterpreter.interpret("select name, age from test where age < 40", context); + assertEquals(InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals(Type.TABLE, ret.message().get(0).getType()); + assertEquals("name\tage\nmoon\t33\npark\t34\n", ret.message().get(0).getData()); + + ret = sqlInterpreter.interpret("select wrong syntax", context); + assertEquals(InterpreterResult.Code.ERROR, ret.code()); + assertTrue(ret.message().get(0).getData().length() > 0); + + assertEquals(InterpreterResult.Code.SUCCESS, sqlInterpreter.interpret("select case when name='aa' then name else name end from test", context).code()); + } + + @Test + public void testStruct() throws InterpreterException { + sparkInterpreter.interpret("case class Person(name:String, age:Int)", context); + sparkInterpreter.interpret("case class People(group:String, person:Person)", context); + sparkInterpreter.interpret( + "val gr = sc.parallelize(Seq(People(\"g1\", Person(\"moon\",33)), People(\"g2\", Person(\"sun\",11))))", + context); + if (isDataFrameSupported()) { + sparkInterpreter.interpret("gr.toDF.registerTempTable(\"gr\")", context); + } else { + sparkInterpreter.interpret("gr.registerTempTable(\"gr\")", context); + } + + InterpreterResult ret = sqlInterpreter.interpret("select * from gr", context); + assertEquals(InterpreterResult.Code.SUCCESS, ret.code()); + } + + public void test_null_value_in_row() throws InterpreterException { + sparkInterpreter.interpret("import org.apache.spark.sql._", context); + if (isDataFrameSupported()) { + sparkInterpreter.interpret( + "import org.apache.spark.sql.types.{StructType,StructField,StringType,IntegerType}", + context); + } + sparkInterpreter.interpret( + "def toInt(s:String): Any = {try { s.trim().toInt} catch {case e:Exception => null}}", + context); + sparkInterpreter.interpret( + "val schema = StructType(Seq(StructField(\"name\", StringType, false),StructField(\"age\" , IntegerType, true),StructField(\"other\" , StringType, false)))", + context); + sparkInterpreter.interpret( + "val csv = sc.parallelize(Seq((\"jobs, 51, apple\"), (\"gates, , microsoft\")))", + context); + sparkInterpreter.interpret( + "val raw = csv.map(_.split(\",\")).map(p => Row(p(0),toInt(p(1)),p(2)))", + context); + if (isDataFrameSupported()) { + sparkInterpreter.interpret("val people = sqlContext.createDataFrame(raw, schema)", + context); + sparkInterpreter.interpret("people.toDF.registerTempTable(\"people\")", context); + } else { + sparkInterpreter.interpret("val people = sqlContext.applySchema(raw, schema)", + context); + sparkInterpreter.interpret("people.registerTempTable(\"people\")", context); + } + + InterpreterResult ret = sqlInterpreter.interpret( + "select name, age from people where name = 'gates'", context); + assertEquals(InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals(Type.TABLE, ret.message().get(0).getType()); + assertEquals("name\tage\ngates\tnull\n", ret.message().get(0).getData()); + } + + @Test + public void testMaxResults() throws InterpreterException { + sparkInterpreter.interpret("case class P(age:Int)", context); + sparkInterpreter.interpret( + "val gr = sc.parallelize(Seq(P(1),P(2),P(3),P(4),P(5),P(6),P(7),P(8),P(9),P(10),P(11)))", + context); + if (isDataFrameSupported()) { + sparkInterpreter.interpret("gr.toDF.registerTempTable(\"gr\")", context); + } else { + sparkInterpreter.interpret("gr.registerTempTable(\"gr\")", context); + } + + InterpreterResult ret = sqlInterpreter.interpret("select * from gr", context); + assertEquals(InterpreterResult.Code.SUCCESS, ret.code()); + assertTrue(ret.message().get(1).getData().contains("alert-warning")); + } +} diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java similarity index 87% rename from spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java rename to spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java index e4f15f4b38f..14214a284f2 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java @@ -17,34 +17,47 @@ package org.apache.zeppelin.spark; -import static org.junit.Assert.*; - -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.resource.LocalResourcePool; import org.apache.zeppelin.resource.WellKnownResourceName; import org.apache.zeppelin.user.AuthenticationInfo; -import org.apache.zeppelin.display.GUI; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.FixMethodOrder; +import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runners.MethodSorters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + @FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class SparkInterpreterTest { +public class OldSparkInterpreterTest { @ClassRule public static TemporaryFolder tmpDir = new TemporaryFolder(); @@ -52,7 +65,7 @@ public class SparkInterpreterTest { static SparkInterpreter repl; static InterpreterGroup intpGroup; static InterpreterContext context; - static Logger LOGGER = LoggerFactory.getLogger(SparkInterpreterTest.class); + static Logger LOGGER = LoggerFactory.getLogger(OldSparkInterpreterTest.class); static Map> paraIdToInfosMap = new HashMap<>(); @@ -129,12 +142,12 @@ public RemoteEventClientWrapper getClient() { } @AfterClass - public static void tearDown() { + public static void tearDown() throws InterpreterException { repl.close(); } @Test - public void testBasicIntp() { + public void testBasicIntp() throws InterpreterException { assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("val a = 1\nval b = 2", context).code()); @@ -153,41 +166,41 @@ public void testBasicIntp() { } @Test - public void testNonStandardSparkProperties() throws IOException { + public void testNonStandardSparkProperties() throws IOException, InterpreterException { // throw NoSuchElementException if no such property is found InterpreterResult result = repl.interpret("sc.getConf.get(\"property_1\")", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); } @Test - public void testNextLineInvocation() { + public void testNextLineInvocation() throws InterpreterException { assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("\"123\"\n.toInt", context).code()); } @Test - public void testNextLineComments() { + public void testNextLineComments() throws InterpreterException { assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("\"123\"\n/*comment here\n*/.toInt", context).code()); } @Test - public void testNextLineCompanionObject() { + public void testNextLineCompanionObject() throws InterpreterException { String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter {\n def apply(x: Long) = new Counter()\n}"; assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret(code, context).code()); } @Test - public void testEndWithComment() { + public void testEndWithComment() throws InterpreterException { assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("val c=1\n//comment", context).code()); } @Test public void testListener() { SparkContext sc = repl.getSparkContext(); - assertNotNull(SparkInterpreter.setupListeners(sc)); + assertNotNull(OldSparkInterpreter.setupListeners(sc)); } @Test - public void testCreateDataFrame() { + public void testCreateDataFrame() throws InterpreterException { if (getSparkVersionNumber(repl) >= 13) { repl.interpret("case class Person(name:String, age:Int)\n", context); repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context); @@ -200,7 +213,7 @@ public void testCreateDataFrame() { } @Test - public void testZShow() { + public void testZShow() throws InterpreterException { String code = ""; repl.interpret("case class Person(name:String, age:Int)\n", context); repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context); @@ -236,7 +249,7 @@ public void testSparkSql() throws IOException, InterpreterException { } @Test - public void testReferencingUndefinedVal() { + public void testReferencingUndefinedVal() throws InterpreterException { InterpreterResult result = repl.interpret("def category(min: Int) = {" + " if (0 <= value) \"error\"" + "}", context); assertEquals(Code.ERROR, result.code()); @@ -308,20 +321,20 @@ public void testDisableImplicitImport() throws IOException, InterpreterException } @Test - public void testCompletion() { + public void testCompletion() throws InterpreterException { List completions = repl.completion("sc.", "sc.".length(), null); assertTrue(completions.size() > 0); } @Test - public void testMultilineCompletion() { + public void testMultilineCompletion() throws InterpreterException { String buf = "val x = 1\nsc."; List completions = repl.completion(buf, buf.length(), null); assertTrue(completions.size() > 0); } @Test - public void testMultilineCompletionNewVar() { + public void testMultilineCompletionNewVar() throws InterpreterException { Assume.assumeFalse("this feature does not work with scala 2.10", Utils.isScala2_10()); Assume.assumeTrue("This feature does not work with scala < 2.11.8", Utils.isCompilerAboveScala2_11_7()); String buf = "val x = sc\nx."; @@ -330,7 +343,7 @@ public void testMultilineCompletionNewVar() { } @Test - public void testParagraphUrls() { + public void testParagraphUrls() throws InterpreterException { String paraId = "test_para_job_url"; InterpreterContext intpCtx = new InterpreterContext("note", paraId, null, "title", "text", new AuthenticationInfo(), diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkSqlInterpreterTest.java similarity index 86% rename from spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java rename to spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkSqlInterpreterTest.java index d97e57c8781..d0b0874aa02 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkSqlInterpreterTest.java @@ -17,23 +17,32 @@ package org.apache.zeppelin.spark; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Properties; - import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.resource.LocalResourcePool; -import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.junit.*; +import org.apache.zeppelin.resource.LocalResourcePool; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Properties; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class SparkSqlInterpreterTest { +public class OldSparkSqlInterpreterTest { @ClassRule public static TemporaryFolder tmpDir = new TemporaryFolder(); @@ -46,7 +55,7 @@ public class SparkSqlInterpreterTest { @BeforeClass public static void setUp() throws Exception { Properties p = new Properties(); - p.putAll(SparkInterpreterTest.getSparkTestProperties(tmpDir)); + p.putAll(OldSparkInterpreterTest.getSparkTestProperties(tmpDir)); p.setProperty("zeppelin.spark.maxResult", "10"); p.setProperty("zeppelin.spark.concurrentSQL", "false"); p.setProperty("zeppelin.spark.sql.stacktrace", "false"); @@ -55,8 +64,8 @@ public static void setUp() throws Exception { intpGroup = new InterpreterGroup(); repl.setInterpreterGroup(intpGroup); repl.open(); - SparkInterpreterTest.repl = repl; - SparkInterpreterTest.intpGroup = intpGroup; + OldSparkInterpreterTest.repl = repl; + OldSparkInterpreterTest.intpGroup = intpGroup; sql = new SparkSqlInterpreter(p); @@ -75,13 +84,13 @@ public static void setUp() throws Exception { } @AfterClass - public static void tearDown() { + public static void tearDown() throws InterpreterException { sql.close(); repl.close(); } boolean isDataFrameSupported() { - return SparkInterpreterTest.getSparkVersionNumber(repl) >= 13; + return OldSparkInterpreterTest.getSparkVersionNumber(repl) >= 13; } @Test @@ -144,11 +153,11 @@ public void test_null_value_in_row() throws InterpreterException { "val raw = csv.map(_.split(\",\")).map(p => Row(p(0),toInt(p(1)),p(2)))", context); if (isDataFrameSupported()) { - repl.interpret("val people = z.sqlContext.createDataFrame(raw, schema)", + repl.interpret("val people = sqlContext.createDataFrame(raw, schema)", context); repl.interpret("people.toDF.registerTempTable(\"people\")", context); } else { - repl.interpret("val people = z.sqlContext.applySchema(raw, schema)", + repl.interpret("val people = sqlContext.applySchema(raw, schema)", context); repl.interpret("people.registerTempTable(\"people\")", context); } diff --git a/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java similarity index 94% rename from spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java rename to spark/interpreter/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java index 2f1077da57c..2d40871712e 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java @@ -47,22 +47,22 @@ public class PySparkInterpreterMatplotlibTest { static InterpreterGroup intpGroup; static Logger LOGGER = LoggerFactory.getLogger(PySparkInterpreterTest.class); static InterpreterContext context; - + public static class AltPySparkInterpreter extends PySparkInterpreter { /** * Since pyspark output is sent to an outputstream rather than * being directly provided by interpret(), this subclass is created to * override interpret() to append the result from the outputStream - * for the sake of convenience in testing. + * for the sake of convenience in testing. */ public AltPySparkInterpreter(Properties property) { super(property); } /** - * This code is mainly copied from RemoteInterpreterServer.java which + * This code is mainly copied from RemoteInterpreterServer.java which * normally handles this in real use cases. - */ + */ @Override public InterpreterResult interpret(String st, InterpreterContext context) throws InterpreterException { context.out.clear(); @@ -82,7 +82,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) throws private static Properties getPySparkTestProperties() throws IOException { Properties p = new Properties(); - p.setProperty("master", "local[*]"); + p.setProperty("spark.master", "local[*]"); p.setProperty("spark.app.name", "Zeppelin Test"); p.setProperty("zeppelin.spark.useHiveContext", "true"); p.setProperty("zeppelin.spark.maxResult", "1000"); @@ -132,10 +132,19 @@ public static void setUp() throws Exception { pyspark.setInterpreterGroup(intpGroup); pyspark.open(); + context = new InterpreterContext("note", "id", null, "title", "text", + new AuthenticationInfo(), + new HashMap(), + new GUI(), + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null), + new LocalResourcePool("id"), + new LinkedList(), + new InterpreterOutput(null)); } @AfterClass - public static void tearDown() { + public static void tearDown() throws InterpreterException { pyspark.close(); sparkInterpreter.close(); } @@ -145,7 +154,7 @@ public void dependenciesAreInstalled() throws InterpreterException { // matplotlib InterpreterResult ret = pyspark.interpret("import matplotlib", context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); - + // inline backend ret = pyspark.interpret("import backend_zinline", context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); @@ -178,14 +187,14 @@ public void testClose() throws InterpreterException { ret = pyspark.interpret("z.configure_mpl(interactive=False, close=True, angular=False)", context); ret = pyspark.interpret("plt.plot([1, 2, 3])", context); ret1 = pyspark.interpret("plt.show()", context); - + // Second call to show() should print nothing, and Type should be TEXT. // This is because when close=True, there should be no living instances // of FigureManager, causing show() to return before setting the output // type to HTML. ret = pyspark.interpret("plt.show()", context); assertEquals(0, ret.message().size()); - + // Now test that new plot is drawn. It should be identical to the // previous one. ret = pyspark.interpret("plt.plot([1, 2, 3])", context); @@ -193,7 +202,7 @@ public void testClose() throws InterpreterException { assertEquals(ret1.message().get(0).getType(), ret2.message().get(0).getType()); assertEquals(ret1.message().get(0).getData(), ret2.message().get(0).getData()); } - + @Test // Test for when configuration is set to not auto-close figures after show(). public void testNoClose() throws InterpreterException { @@ -205,7 +214,7 @@ public void testNoClose() throws InterpreterException { ret = pyspark.interpret("z.configure_mpl(interactive=False, close=False, angular=False)", context); ret = pyspark.interpret("plt.plot([1, 2, 3])", context); ret1 = pyspark.interpret("plt.show()", context); - + // Second call to show() should print nothing, and Type should be HTML. // This is because when close=False, there should be living instances // of FigureManager, causing show() to set the output @@ -220,7 +229,7 @@ public void testNoClose() throws InterpreterException { ret2 = pyspark.interpret("plt.show()", context); assertNotSame(ret1.message().get(0).getData(), ret2.message().get(0).getData()); } - + @Test // Test angular mode public void testAngular() throws InterpreterException { @@ -229,7 +238,7 @@ public void testAngular() throws InterpreterException { ret = pyspark.interpret("plt.close()", context); ret = pyspark.interpret("z.configure_mpl(interactive=False, close=False, angular=True)", context); ret = pyspark.interpret("plt.plot([1, 2, 3])", context); - ret = pyspark.interpret("plt.show()", context); + ret = pyspark.interpret("plt.show()", context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); assertEquals(ret.message().toString(), Type.ANGULAR, ret.message().get(0).getType()); @@ -237,5 +246,5 @@ public void testAngular() throws InterpreterException { AngularObjectRegistry registry = context.getAngularObjectRegistry(); String figureData = registry.getAll("note", null).get(0).toString(); assertTrue(figureData.contains("data:image/png;base64")); - } + } } diff --git a/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java similarity index 97% rename from spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java rename to spark/interpreter/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java index 0db2bb1c49b..00972b42e01 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java @@ -26,8 +26,7 @@ import org.junit.*; import org.junit.rules.TemporaryFolder; import org.junit.runners.MethodSorters; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import java.io.IOException; import java.util.HashMap; import java.util.LinkedList; @@ -47,12 +46,11 @@ public class PySparkInterpreterTest { static SparkInterpreter sparkInterpreter; static PySparkInterpreter pySparkInterpreter; static InterpreterGroup intpGroup; - static Logger LOGGER = LoggerFactory.getLogger(PySparkInterpreterTest.class); static InterpreterContext context; private static Properties getPySparkTestProperties() throws IOException { Properties p = new Properties(); - p.setProperty("master", "local[*]"); + p.setProperty("spark.master", "local"); p.setProperty("spark.app.name", "Zeppelin Test"); p.setProperty("zeppelin.spark.useHiveContext", "true"); p.setProperty("zeppelin.spark.maxResult", "1000"); @@ -60,6 +58,7 @@ private static Properties getPySparkTestProperties() throws IOException { p.setProperty("zeppelin.pyspark.python", "python"); p.setProperty("zeppelin.dep.localrepo", tmpDir.newFolder().getAbsolutePath()); p.setProperty("zeppelin.pyspark.useIPython", "false"); + p.setProperty("zeppelin.spark.test", "true"); return p; } @@ -107,7 +106,7 @@ public static void setUp() throws Exception { } @AfterClass - public static void tearDown() { + public static void tearDown() throws InterpreterException { pySparkInterpreter.close(); sparkInterpreter.close(); } diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java new file mode 100644 index 00000000000..2d585f5387c --- /dev/null +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class SparkRInterpreterTest { + + private SparkRInterpreter sparkRInterpreter; + private SparkInterpreter sparkInterpreter; + + + @Test + public void testSparkRInterpreter() throws IOException, InterruptedException, InterpreterException { + Properties properties = new Properties(); + properties.setProperty("spark.master", "local"); + properties.setProperty("spark.app.name", "test"); + properties.setProperty("zeppelin.spark.maxResult", "100"); + properties.setProperty("zeppelin.spark.test", "true"); + properties.setProperty("zeppelin.spark.useNew", "true"); + properties.setProperty("zeppelin.R.knitr", "true"); + + sparkRInterpreter = new SparkRInterpreter(properties); + sparkInterpreter = new SparkInterpreter(properties); + + InterpreterGroup interpreterGroup = new InterpreterGroup(); + interpreterGroup.addInterpreterToSession(new LazyOpenInterpreter(sparkRInterpreter), "session_1"); + interpreterGroup.addInterpreterToSession(new LazyOpenInterpreter(sparkInterpreter), "session_1"); + sparkRInterpreter.setInterpreterGroup(interpreterGroup); + sparkInterpreter.setInterpreterGroup(interpreterGroup); + + sparkRInterpreter.open(); + + InterpreterResult result = sparkRInterpreter.interpret("1+1", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertTrue(result.message().get(0).getData().contains("2")); + + result = sparkRInterpreter.interpret("sparkR.version()", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + if (result.message().get(0).getData().contains("2.")) { + // spark 2.x + result = sparkRInterpreter.interpret("df <- as.DataFrame(faithful)\nhead(df)", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertTrue(result.message().get(0).getData().contains("eruptions waiting")); + } else { + // spark 1.x + result = sparkRInterpreter.interpret("df <- createDataFrame(sqlContext, faithful)\nhead(df)", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertTrue(result.message().get(0).getData().contains("eruptions waiting")); + } + } + + private InterpreterContext getInterpreterContext() { + return new InterpreterContext( + "noteId", + "paragraphId", + "replName", + "paragraphTitle", + "paragraphText", + new AuthenticationInfo(), + new HashMap(), + new GUI(), + new GUI(), + new AngularObjectRegistry("spark", null), + null, + null, + null); + } +} diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java similarity index 100% rename from spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java rename to spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java diff --git a/spark/src/test/resources/log4j.properties b/spark/interpreter/src/test/resources/log4j.properties similarity index 97% rename from spark/src/test/resources/log4j.properties rename to spark/interpreter/src/test/resources/log4j.properties index 3ee61ab864b..6958d4c30fb 100644 --- a/spark/src/test/resources/log4j.properties +++ b/spark/interpreter/src/test/resources/log4j.properties @@ -45,5 +45,8 @@ log4j.logger.org.hibernate.type=ALL log4j.logger.org.apache.zeppelin.interpreter=DEBUG log4j.logger.org.apache.zeppelin.spark=DEBUG + log4j.logger.org.apache.zeppelin.python.IPythonInterpreter=DEBUG log4j.logger.org.apache.zeppelin.python.IPythonClient=DEBUG +log4j.logger.org.apache.spark.repl.Main=INFO + diff --git a/spark/src/test/scala/org/apache/zeppelin/spark/utils/DisplayFunctionsTest.scala b/spark/interpreter/src/test/scala/org/apache/zeppelin/spark/utils/DisplayFunctionsTest.scala similarity index 100% rename from spark/src/test/scala/org/apache/zeppelin/spark/utils/DisplayFunctionsTest.scala rename to spark/interpreter/src/test/scala/org/apache/zeppelin/spark/utils/DisplayFunctionsTest.scala diff --git a/spark/pom.xml b/spark/pom.xml index 1972f26d383..06b7d9f74b1 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -16,680 +16,227 @@ ~ limitations under the License. --> - - 4.0.0 + + 4.0.0 + + + interpreter-parent + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../interpreter-parent/pom.xml + - - zeppelin org.apache.zeppelin + spark-parent + pom 0.9.0-SNAPSHOT - .. - - - org.apache.zeppelin - zeppelin-spark_2.10 - jar - 0.9.0-SNAPSHOT - Zeppelin: Spark - Zeppelin spark support - - - - 1.8.2 - 2.0.2 - 14.0.1 - 1.3 - 1.9 - 3.0 - 1.12 - 3.0.3 - 1.0 - - 3.2.9 - 3.2.6 - 3.2.10 - - - 2.3 - 2.15.2 - - - **/PySparkInterpreterMatplotlibTest.java - **/*Test.* - - - - - ${project.groupId} - zeppelin-display_${scala.binary.version} - ${project.version} - - - - ${project.groupId} - zeppelin-interpreter - ${project.version} - - - - ${project.groupId} - zeppelin-python - ${project.version} - - - net.sf.py4j - py4j - - - - - - ${project.groupId} - zeppelin-python - ${project.version} - tests - test - - - net.sf.py4j - py4j - - - - - - org.slf4j - slf4j-api - - - - org.slf4j - slf4j-log4j12 - - - - org.apache.spark - spark-repl_${scala.binary.version} - ${spark.version} - provided - - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark.version} - provided - - - - - org.apache.maven - maven-plugin-api - ${maven.plugin.api.version} - - - org.codehaus.plexus - plexus-utils - - - org.sonatype.sisu - sisu-inject-plexus - - - org.apache.maven - maven-model - - - - - - org.sonatype.aether - aether-api - ${aether.version} - - - - org.sonatype.aether - aether-util - ${aether.version} - - - - org.sonatype.aether - aether-impl - ${aether.version} - - - - org.apache.maven - maven-aether-provider - ${maven.aeither.provider.version} - - - org.sonatype.aether - aether-api - - - org.sonatype.aether - aether-spi - - - org.sonatype.aether - aether-util - - - org.sonatype.aether - aether-impl - - - org.codehaus.plexus - plexus-utils - - - - - - org.sonatype.aether - aether-connector-file - ${aether.version} - - - - org.sonatype.aether - aether-connector-wagon - ${aether.version} - - - org.apache.maven.wagon - wagon-provider-api - - - - - - org.apache.maven.wagon - wagon-provider-api - ${wagon.version} - - - org.codehaus.plexus - plexus-utils - - - - - - org.apache.maven.wagon - wagon-http-lightweight - ${wagon.version} - - - org.apache.maven.wagon - wagon-http-shared - - - - - - org.apache.maven.wagon - wagon-http - ${wagon.version} - - - - - - org.apache.commons - commons-exec - ${commons.exec.version} - - - - org.scala-lang - scala-library - ${scala.version} - provided - - - - org.scala-lang - scala-compiler - ${scala.version} - provided - - - - org.scala-lang - scala-reflect - ${scala.version} - provided - - - - commons-lang - commons-lang - provided - - - - org.apache.commons - commons-compress - ${commons.compress.version} - provided - - - - org.jsoup - jsoup - ${jsoup.version} - - - - - org.scalatest - scalatest_${scala.binary.version} - ${scalatest.version} - test - - - - junit - junit - test - - - - org.datanucleus - datanucleus-core - ${datanucleus.core.version} - test - - - - org.datanucleus - datanucleus-api-jdo - ${datanucleus.apijdo.version} - test - - - - org.datanucleus - datanucleus-rdbms - ${datanucleus.rdbms.version} - test - - - - org.mockito - mockito-core - test - - - - org.powermock - powermock-api-mockito - test - - - - org.powermock - powermock-module-junit4 - test - - - - - - - - maven-enforcer-plugin - - - enforce - none - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - 1 - false - -Xmx1024m -XX:MaxPermSize=256m - - **/SparkRInterpreterTest.java - ${pyspark.test.exclude} - - - - ../interpreter/spark/pyspark/pyspark.zip:../interpreter/spark/pyspark/py4j-${spark.py4j.version}-src.zip:../interpreter/lib/python - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${plugin.shade.version} - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - reference.conf - - - - - - - com.google - org.apache.zeppelin.com.google - - - - io.netty - org.apache.zeppelin.io.netty - - - - - - package - - shade - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - package - - copy - - - ${project.build.directory}/../../interpreter/spark - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - - - - - - org.scala-tools - maven-scala-plugin - ${plugin.scala.version} - - ${scala.version} - - **/ZeppelinR.scala - **/SparkRBackend.scala - - - - - compile - - compile - - compile - - - test-compile - - testCompile - - test-compile - - - process-resources - - compile - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - ${pyspark.test.exclude} - - - - - org.scala-tools - maven-scala-plugin - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - ${pyspark.test.exclude} - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - ${pyspark.test.exclude} - - - - - org.scala-tools - maven-scala-plugin - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - ${pyspark.test.exclude} - - - - - - maven-resources-plugin - - - copy-interpreter-setting - package - - resources - - - ${project.build.directory}/../../interpreter/spark - - - - - - - - - - spark-1.4 - - 1.4.1 - - - - - - - - spark-1.5 - - 1.5.2 - com.typesafe.akka - 2.3.11 - 2.5.0 - - - - - spark-1.6 - - 1.6.3 - 0.9 - com.typesafe.akka - 2.3.11 - 2.5.0 - - - - - spark-2.0 - - 2.0.2 - 2.5.0 - 0.10.3 - - + spark-parent + Zeppelin spark support + + + + 3.2.9 + 3.2.6 + 3.2.10 + + + 2.4.1 + 2.15.2 + + 2.2.0 + 0.10.4 + - - spark-2.1 - - 2.1.0 - 2.5.0 - 0.10.4 - 2.11.8 - - + - - spark-2.2 - - true - - - 2.2.0 - 2.5.0 - 0.10.4 - - + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + - - hadoop-0.23 - - + - org.apache.avro - avro + org.apache.zeppelin + zeppelin-display + ${project.version} + test - - - 0.23.10 - - - - hadoop-1 - - 1.0.4 - hadoop1 - 1.8.8 - org.spark-project.akka - - + + org.scalatest + scalatest_${scala.binary.version} + ${scalatest.version} + test + - - hadoop-2.2 - - 2.2.0 - 2.5.0 - hadoop2 - - + + junit + junit + test + - - hadoop-2.3 - - 2.3.0 - 2.5.0 - 0.9.3 - hadoop2 - - + + org.datanucleus + datanucleus-core + ${datanucleus.core.version} + test + - - hadoop-2.4 - - 2.4.0 - 2.5.0 - 0.9.3 - hadoop2 - - + + org.datanucleus + datanucleus-api-jdo + ${datanucleus.apijdo.version} + test + - - hadoop-2.6 - - 2.6.0 - 2.5.0 - 0.9.3 - hadoop2 - - + + org.datanucleus + datanucleus-rdbms + ${datanucleus.rdbms.version} + test + - - hadoop-2.7 - - 2.7.2 - 2.5.0 - 0.9.0 - hadoop2 - - - + + + + + + maven-enforcer-plugin + + + enforce + none + + + + + + org.scalatest + scalatest-maven-plugin + + ${project.build.directory}/surefire-reports + . + WDF TestSuite.txt + + + + test + + test + + + + + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + eclipse-add-source + + add-source + + + + scala-compile-first + process-resources + + compile + + + + scala-test-compile-first + process-test-resources + + testCompile + + + + + ${scala.compile.version} + + + + -unchecked + -deprecation + -feature + + + -Xms1024m + -Xmx1024m + -XX:PermSize=${PermGen} + -XX:MaxPermSize=${MaxPermGen} + + + -source + ${java.version} + -target + ${java.version} + -Xlint:all,-serial,-path,-options + + + + + + + + + + + spark-2.2 + + true + + + 2.2.0 + 0.10.4 + + + + + spark-2.1 + + 2.1.0 + 0.10.4 + + + + + spark-2.0 + + 2.0.2 + 0.10.3 + + + + + spark-1.6 + + 1.6.3 + 0.9 + + + + + spark-1.5 + + 1.5.2 + 0.8.2.1 + + + + + spark-1.4 + + 1.4.1 + 0.8.2.1 + + + + diff --git a/spark/scala-2.10/pom.xml b/spark/scala-2.10/pom.xml new file mode 100644 index 00000000000..e32e620bf30 --- /dev/null +++ b/spark/scala-2.10/pom.xml @@ -0,0 +1,41 @@ + + + + 4.0.0 + org.apache.zeppelin + spark-scala-2.10 + 0.9.0-SNAPSHOT + jar + Spark Interpreter: Scala_2.10 + + + org.apache.zeppelin + spark-scala-parent + 0.9.0-SNAPSHOT + ../spark-scala-parent/pom.xml + + + + 2.10.5 + 2.10 + ${scala.version} + + + diff --git a/spark/scala-2.10/spark-scala-parent b/spark/scala-2.10/spark-scala-parent new file mode 120000 index 00000000000..e5e899e58cf --- /dev/null +++ b/spark/scala-2.10/spark-scala-parent @@ -0,0 +1 @@ +../spark-scala-parent \ No newline at end of file diff --git a/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala b/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala new file mode 100644 index 00000000000..43aa8643639 --- /dev/null +++ b/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark + +import java.io.File +import java.nio.file.{Files, Paths} + +import org.apache.spark.SparkConf +import org.apache.spark.repl.SparkILoop +import org.apache.spark.repl.SparkILoop._ +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion +import org.apache.zeppelin.interpreter.util.InterpreterOutputStream +import org.apache.zeppelin.interpreter.{InterpreterContext, InterpreterResult} +import org.slf4j.{Logger, LoggerFactory} + +import scala.tools.nsc.Settings +import scala.tools.nsc.interpreter._ + +/** + * SparkInterpreter for scala-2.10 + */ +class SparkScala210Interpreter(override val conf: SparkConf, + override val depFiles: java.util.List[String]) + extends BaseSparkScalaInterpreter(conf, depFiles) { + + lazy override val LOGGER: Logger = LoggerFactory.getLogger(getClass) + + private var sparkILoop: SparkILoop = _ + + override val interpreterOutput = + new InterpreterOutputStream(LoggerFactory.getLogger(classOf[SparkScala210Interpreter])) + + override def open(): Unit = { + super.open() + // redirect the output of open to InterpreterOutputStream, so that user can have more + // diagnose info in frontend + if (InterpreterContext.get() != null) { + interpreterOutput.setInterpreterOutput(InterpreterContext.get().out) + } + val rootDir = conf.get("spark.repl.classdir", System.getProperty("java.io.tmpdir")) + val outputDir = Files.createTempDirectory(Paths.get(rootDir), "spark").toFile + outputDir.deleteOnExit() + conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath) + // Only Spark1 requires to create http server, Spark2 removes HttpServer class. + startHttpServer(outputDir).foreach { case (server, uri) => + sparkHttpServer = server + conf.set("spark.repl.class.uri", uri) + } + + val settings = new Settings() + settings.embeddedDefaults(Thread.currentThread().getContextClassLoader()) + settings.usejavacp.value = true + settings.classpath.value = getUserJars.mkString(File.pathSeparator) + Console.setOut(interpreterOutput) + sparkILoop = new SparkILoop(null, new JPrintWriter(Console.out, true)) + + setDeclaredField(sparkILoop, "settings", settings) + callMethod(sparkILoop, "createInterpreter") + sparkILoop.initializeSynchronous() + callMethod(sparkILoop, "postInitialization") + val reader = callMethod(sparkILoop, + "org$apache$spark$repl$SparkILoop$$chooseReader", + Array(settings.getClass), Array(settings)).asInstanceOf[InteractiveReader] + setDeclaredField(sparkILoop, "org$apache$spark$repl$SparkILoop$$in", reader) + scalaCompleter = reader.completion.completer() + + createSparkContext() + } + + override def close(): Unit = { + super.close() + if (sparkILoop != null) { + callMethod(sparkILoop, "org$apache$spark$repl$SparkILoop$$closeInterpreter") + } + } + + protected override def interpret(code: String, context: InterpreterContext): InterpreterResult = { + if (context != null) { + interpreterOutput.setInterpreterOutput(context.out) + context.out.clear() + } else { + interpreterOutput.setInterpreterOutput(null) + } + + Console.withOut(if (context != null) context.out else Console.out) { + interpreterOutput.ignoreLeadingNewLinesFromScalaReporter() + // add print("") at the end in case the last line is comment which lead to INCOMPLETE + val lines = code.split("\\n") ++ List("print(\"\")") + var incompleteCode = "" + var lastStatus: InterpreterResult.Code = null + for (line <- lines if !line.trim.isEmpty) { + val nextLine = if (incompleteCode != "") { + incompleteCode + "\n" + line + } else { + line + } + scalaInterpret(nextLine) match { + case scala.tools.nsc.interpreter.IR.Success => + // continue the next line + incompleteCode = "" + lastStatus = InterpreterResult.Code.SUCCESS + case error@scala.tools.nsc.interpreter.IR.Error => + return new InterpreterResult(InterpreterResult.Code.ERROR) + case scala.tools.nsc.interpreter.IR.Incomplete => + // put this line into inCompleteCode for the next execution. + incompleteCode = incompleteCode + "\n" + line + lastStatus = InterpreterResult.Code.INCOMPLETE + } + } + // flush all output before returning result to frontend + Console.flush() + interpreterOutput.setInterpreterOutput(null) + return new InterpreterResult(lastStatus) + } + } + + def scalaInterpret(code: String): scala.tools.nsc.interpreter.IR.Result = + sparkILoop.interpret(code) + + protected def bind(name: String, tpe: String, value: Object, modifier: List[String]): Unit = { + sparkILoop.beQuietDuring { + sparkILoop.bind(name, tpe, value, modifier) + } + } + +} diff --git a/spark/scala-2.11/pom.xml b/spark/scala-2.11/pom.xml new file mode 100644 index 00000000000..d9113d1075e --- /dev/null +++ b/spark/scala-2.11/pom.xml @@ -0,0 +1,41 @@ + + + + 4.0.0 + org.apache.zeppelin + spark-scala-2.11 + 0.9.0-SNAPSHOT + jar + Spark Interpreter: Scala_2.11 + + + org.apache.zeppelin + spark-scala-parent + 0.9.0-SNAPSHOT + ../spark-scala-parent/pom.xml + + + + 2.11.8 + 2.11 + ${scala.version} + + + diff --git a/spark/scala-2.11/spark-scala-parent b/spark/scala-2.11/spark-scala-parent new file mode 120000 index 00000000000..e5e899e58cf --- /dev/null +++ b/spark/scala-2.11/spark-scala-parent @@ -0,0 +1 @@ +../spark-scala-parent \ No newline at end of file diff --git a/spark/scala-2.11/src/main/resources/log4j.properties b/spark/scala-2.11/src/main/resources/log4j.properties new file mode 100644 index 00000000000..0c90b21ae00 --- /dev/null +++ b/spark/scala-2.11/src/main/resources/log4j.properties @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n +#log4j.appender.stdout.layout.ConversionPattern= +#%5p [%t] (%F:%L) - %m%n +#%-4r [%t] %-5p %c %x - %m%n +# + +# Root logger option +log4j.rootLogger=INFO, stdout + +#mute some noisy guys +log4j.logger.org.apache.hadoop.mapred=WARN +log4j.logger.org.apache.hadoop.hive.ql=WARN +log4j.logger.org.apache.hadoop.hive.metastore=WARN +log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN +log4j.logger.org.apache.zeppelin.scheduler=WARN + +log4j.logger.org.quartz=WARN +log4j.logger.DataNucleus=WARN +log4j.logger.DataNucleus.MetaData=ERROR +log4j.logger.DataNucleus.Datastore=ERROR + +# Log all JDBC parameters +log4j.logger.org.hibernate.type=ALL + +log4j.logger.org.apache.zeppelin.interpreter=DEBUG +log4j.logger.org.apache.zeppelin.spark=DEBUG + + +log4j.logger.org.apache.spark.repl.Main=INFO diff --git a/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala b/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala new file mode 100644 index 00000000000..e1452606c09 --- /dev/null +++ b/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark + +import java.io.{BufferedReader, File} +import java.net.URLClassLoader +import java.nio.file.{Files, Paths} + +import org.apache.spark.SparkConf +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion +import org.apache.zeppelin.interpreter.util.InterpreterOutputStream +import org.apache.zeppelin.interpreter.{InterpreterContext, InterpreterResult} +import org.slf4j.LoggerFactory +import org.slf4j.Logger + +import scala.tools.nsc.Settings +import scala.tools.nsc.interpreter._ + +/** + * SparkInterpreter for scala-2.11 + */ +class SparkScala211Interpreter(override val conf: SparkConf, + override val depFiles: java.util.List[String]) + extends BaseSparkScalaInterpreter(conf, depFiles) { + + lazy override val LOGGER: Logger = LoggerFactory.getLogger(getClass) + + private var sparkILoop: ILoop = _ + + override val interpreterOutput = new InterpreterOutputStream(LOGGER) + + override def open(): Unit = { + super.open() + if (conf.get("spark.master", "local") == "yarn-client") { + System.setProperty("SPARK_YARN_MODE", "true") + } + // Only Spark1 requires to create http server, Spark2 removes HttpServer class. + val rootDir = conf.get("spark.repl.classdir", System.getProperty("java.io.tmpdir")) + val outputDir = Files.createTempDirectory(Paths.get(rootDir), "spark").toFile + outputDir.deleteOnExit() + conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath) + startHttpServer(outputDir).foreach { case (server, uri) => + sparkHttpServer = server + conf.set("spark.repl.class.uri", uri) + } + + val settings = new Settings() + settings.processArguments(List("-Yrepl-class-based", + "-Yrepl-outdir", s"${outputDir.getAbsolutePath}"), true) + settings.embeddedDefaults(Thread.currentThread().getContextClassLoader()) + settings.usejavacp.value = true + settings.classpath.value = getUserJars.mkString(File.pathSeparator) + + val replOut = new JPrintWriter(interpreterOutput, true) + sparkILoop = new ILoop(None, replOut) + sparkILoop.settings = settings + sparkILoop.createInterpreter() + + val in0 = getField(sparkILoop, "scala$tools$nsc$interpreter$ILoop$$in0").asInstanceOf[Option[BufferedReader]] + val reader = in0.fold(sparkILoop.chooseReader(settings))(r => SimpleReader(r, replOut, interactive = true)) + + sparkILoop.in = reader + sparkILoop.initializeSynchronous() + callMethod(sparkILoop, "scala$tools$nsc$interpreter$ILoop$$loopPostInit") + this.scalaCompleter = reader.completion.completer() + + createSparkContext() + } + + protected def bind(name: String, tpe: String, value: Object, modifier: List[String]): Unit = { + sparkILoop.beQuietDuring { + sparkILoop.bind(name, tpe, value, modifier) + } + } + + + override def close(): Unit = { + super.close() + if (sparkILoop != null) { + sparkILoop.closeInterpreter() + } + } + + protected override def interpret(code: String, context: InterpreterContext): InterpreterResult = { + if (context != null) { + interpreterOutput.setInterpreterOutput(context.out) + context.out.clear() + } + + Console.withOut(if (context != null) context.out else Console.out) { + interpreterOutput.ignoreLeadingNewLinesFromScalaReporter() + // add print("") at the end in case the last line is comment which lead to INCOMPLETE + val lines = code.split("\\n") ++ List("print(\"\")") + var incompleteCode = "" + var lastStatus: InterpreterResult.Code = null + for (line <- lines if !line.trim.isEmpty) { + val nextLine = if (incompleteCode != "") { + incompleteCode + "\n" + line + } else { + line + } + scalaInterpret(nextLine) match { + case scala.tools.nsc.interpreter.IR.Success => + // continue the next line + incompleteCode = "" + lastStatus = InterpreterResult.Code.SUCCESS + case error@scala.tools.nsc.interpreter.IR.Error => + return new InterpreterResult(InterpreterResult.Code.ERROR) + case scala.tools.nsc.interpreter.IR.Incomplete => + // put this line into inCompleteCode for the next execution. + incompleteCode = incompleteCode + "\n" + line + lastStatus = InterpreterResult.Code.INCOMPLETE + } + } + // flush all output before returning result to frontend + Console.flush() + interpreterOutput.setInterpreterOutput(null) + return new InterpreterResult(lastStatus) + } + } + + def scalaInterpret(code: String): scala.tools.nsc.interpreter.IR.Result = + sparkILoop.interpret(code) + +} diff --git a/spark-dependencies/pom.xml b/spark/spark-dependencies/pom.xml similarity index 57% rename from spark-dependencies/pom.xml rename to spark/spark-dependencies/pom.xml index 15138cd593c..7643dc9d8f4 100644 --- a/spark-dependencies/pom.xml +++ b/spark/spark-dependencies/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - zeppelin + spark-parent org.apache.zeppelin 0.9.0-SNAPSHOT .. @@ -44,7 +44,6 @@ instead of changing spark.version in this section. --> - 1.4.1 2.3.0 ${hadoop.version} 1.7.7 @@ -62,7 +61,6 @@ http://d3kbcqa49mib13.cloudfront.net/${spark.archive}-bin-without-hadoop.tgz - 0.8.2.1 2.3 @@ -359,480 +357,6 @@ - - - spark-1.1 - - - - - 1.1.1 - 2.2.3-shaded-protobuf - - - - - cassandra-spark-1.1 - - - com.datastax.spark - spark-cassandra-connector_${scala.binary.version} - 1.1.1 - - - org.joda - joda-convert - - - - - - 1.1.1 - 2.2.3-shaded-protobuf - - - - - spark-1.2 - - - - 1.2.1 - - - - - cassandra-spark-1.2 - - 1.2.1 - - - - com.datastax.spark - spark-cassandra-connector_${scala.binary.version} - 1.2.1 - - - org.joda - joda-convert - - - - - - - - spark-1.3 - - - 1.3.1 - - - - - - - - - cassandra-spark-1.3 - - 1.3.0 - - - - - com.datastax.spark - spark-cassandra-connector_${scala.binary.version} - 1.3.1 - - - org.joda - joda-convert - - - - - - - - spark-1.4 - - 1.4.1 - - - - - - - - cassandra-spark-1.4 - - 1.4.1 - - - - - com.datastax.spark - spark-cassandra-connector_${scala.binary.version} - 1.4.0 - - - org.joda - joda-convert - - - - - - - - spark-1.5 - - 1.5.2 - com.typesafe.akka - 2.3.11 - 2.5.0 - - - - - - - - cassandra-spark-1.5 - - 1.5.1 - com.typesafe.akka - 2.3.11 - 2.5.0 - 16.0.1 - - - - - com.datastax.spark - spark-cassandra-connector_${scala.binary.version} - 1.5.0 - - - org.joda - joda-convert - - - - - - - - spark-1.6 - - 1.6.3 - 0.9 - com.typesafe.akka - 2.3.11 - 2.5.0 - - - - - spark-2.0 - - 2.0.2 - 2.5.0 - 0.10.3 - - - - - spark-2.1 - - 2.1.0 - 2.5.0 - 0.10.4 - 2.11.8 - - - - - spark-2.2 - - true - - - 2.2.0 - 2.5.0 - 0.10.4 - - - - - hadoop-0.23 - - - - org.apache.avro - avro - - - - 0.23.10 - - - - - hadoop-1 - - 1.0.4 - hadoop1 - 1.8.8 - org.spark-project.akka - - - - - hadoop-2.2 - - 2.2.0 - 2.5.0 - hadoop2 - - - - - hadoop-2.3 - - 2.3.0 - 2.5.0 - 0.9.3 - hadoop2 - - - - - hadoop-2.4 - - 2.4.0 - 2.5.0 - 0.9.3 - hadoop2 - - - - - hadoop-2.6 - - 2.6.0 - 2.5.0 - 0.9.3 - hadoop2 - - - - - hadoop-2.7 - - 2.7.2 - 2.5.0 - 0.9.0 - hadoop2 - - - - - mapr3 - - false - - - 1.0.3-mapr-3.0.3 - 2.3.0-mapr-4.0.0-FCS - 0.7.1 - - - - mapr-releases - http://repository.mapr.com/maven/ - - false - - - true - - - - - - - mapr40 - - false - - - 2.4.1-mapr-1503 - 2.4.1-mapr-1503 - 0.9.3 - - - - org.apache.curator - curator-recipes - 2.4.0 - - - org.apache.zookeeper - zookeeper - - - - - org.apache.zookeeper - zookeeper - 3.4.5-mapr-1503 - - - - - mapr-releases - http://repository.mapr.com/maven/ - - false - - - true - - - - - - - mapr41 - - false - - - 2.5.1-mapr-1503 - 2.5.1-mapr-1503 - 0.7.1 - - - - org.apache.curator - curator-recipes - 2.4.0 - - - org.apache.zookeeper - zookeeper - - - - - org.apache.zookeeper - zookeeper - 3.4.5-mapr-1503 - - - - - mapr-releases - http://repository.mapr.com/maven/ - - false - - - true - - - - - - - mapr50 - - false - - - 2.7.0-mapr-1506 - 2.7.0-mapr-1506 - 0.9.3 - - - - org.apache.curator - curator-recipes - 2.4.0 - - - org.apache.zookeeper - zookeeper - - - - - org.apache.zookeeper - zookeeper - 3.4.5-mapr-1503 - - - - - mapr-releases - http://repository.mapr.com/maven/ - - false - - - true - - - - - - - mapr51 - - false - - - 2.7.0-mapr-1602 - 2.7.0-mapr-1602 - 0.9.3 - - - - org.apache.curator - curator-recipes - 2.4.0 - - - org.apache.zookeeper - zookeeper - - - - - org.apache.zookeeper - zookeeper - 3.4.5-mapr-1503 - - - - - mapr-releases - http://repository.mapr.com/maven/ - - false - - - true - - - - - - - @@ -900,13 +424,24 @@ maven-dependency-plugin - copy-dependencies + copy-interpreter-dependencies + package + + copy-dependencies + + + true + + + + + copy-spark-interpreter-dependencies package copy-dependencies - ${project.build.directory}/../../interpreter/spark/dep + ${project.build.directory}/../../../interpreter/spark/dep false false true @@ -914,12 +449,13 @@ + copy-artifact package copy - ${project.build.directory}/../../interpreter/spark/dep + ${project.build.directory}/../../../interpreter/spark/dep false false true @@ -936,6 +472,19 @@ + + maven-resources-plugin + + + copy-interpreter-setting + none + + true + + + + + com.googlecode.maven-download-plugin @@ -981,10 +530,10 @@ - - - + + @@ -1025,7 +574,7 @@ copy-resources - ${project.build.directory}/../../interpreter/spark/R/lib + ${project.build.directory}/../../../interpreter/spark/R/lib diff --git a/spark/spark-scala-parent/pom.xml b/spark/spark-scala-parent/pom.xml new file mode 100644 index 00000000000..830fa59a684 --- /dev/null +++ b/spark/spark-scala-parent/pom.xml @@ -0,0 +1,172 @@ + + + + + + + spark-parent + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../pom.xml + + + 4.0.0 + org.apache.zeppelin + spark-scala-parent + 0.9.0-SNAPSHOT + pom + + + + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + + + + org.apache.spark + spark-repl_${scala.binary.version} + ${spark.version} + provided + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + provided + + + + org.apache.spark + spark-hive_${scala.binary.version} + ${spark.version} + provided + + + + org.scala-lang + scala-compiler + ${scala.version} + provided + + + + org.scala-lang + scala-library + ${scala.version} + provided + + + + org.scala-lang + scala-reflect + ${scala.version} + provided + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-scala-sources + generate-sources + + add-source + + + + ${project.basedir}/../spark-scala-parent/src/main/scala + + + + + add-scala-test-sources + generate-test-sources + + add-test-source + + + + ${project.basedir}/../spark-scala-parent/src/test/scala + + + + + add-resource + generate-resources + + add-resource + + + + + ${project.basedir}/../spark-scala-parent/src/main/resources + + + + + + add-test-resource + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/../spark-scala-parent/src/test/resources + + + + + + + + + maven-dependency-plugin + + true + + + + + maven-resources-plugin + + + copy-interpreter-setting + none + + true + + + + + + + + + \ No newline at end of file diff --git a/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala b/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala new file mode 100644 index 00000000000..3ef4fe71e00 --- /dev/null +++ b/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark + + +import java.io.File + +import org.apache.spark.sql.SQLContext +import org.apache.spark.{SparkConf, SparkContext} +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion +import org.apache.zeppelin.interpreter.util.InterpreterOutputStream +import org.apache.zeppelin.interpreter.{InterpreterContext, InterpreterResult} +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.JavaConverters._ +import scala.tools.nsc.interpreter.Completion.ScalaCompleter +import scala.util.control.NonFatal + +/** + * Base class for different scala versions of SparkInterpreter. It should be + * binary compatible between multiple scala versions. + * @param conf + * @param depFiles + */ +abstract class BaseSparkScalaInterpreter(val conf: SparkConf, + val depFiles: java.util.List[String]) { + + protected lazy val LOGGER: Logger = LoggerFactory.getLogger(getClass) + + private val isTest = conf.getBoolean("zeppelin.spark.test", false) + + protected var sc: SparkContext = _ + + protected var sqlContext: SQLContext = _ + + protected var sparkSession: Object = _ + + protected var sparkHttpServer: Object = _ + + protected var sparkUrl: String = _ + + protected var scalaCompleter: ScalaCompleter = _ + + protected val interpreterOutput: InterpreterOutputStream + + protected def open(): Unit = { + /* Required for scoped mode. + * In scoped mode multiple scala compiler (repl) generates class in the same directory. + * Class names is not randomly generated and look like '$line12.$read$$iw$$iw' + * Therefore it's possible to generated class conflict(overwrite) with other repl generated + * class. + * + * To prevent generated class name conflict, + * change prefix of generated class name from each scala compiler (repl) instance. + * + * In Spark 2.x, REPL generated wrapper class name should compatible with the pattern + * ^(\$line(?:\d+)\.\$read)(?:\$\$iw)+$ + * + * As hashCode() can return a negative integer value and the minus character '-' is invalid + * in a package name we change it to a numeric value '0' which still conforms to the regexp. + * + */ + System.setProperty("scala.repl.name.line", ("$line" + this.hashCode).replace('-', '0')) + } + + protected def interpret(code: String, context: InterpreterContext): InterpreterResult + + protected def interpret(code: String): InterpreterResult = interpret(code, null) + + protected def scalaInterpret(code: String): scala.tools.nsc.interpreter.IR.Result + + protected def completion(buf: String, + cursor: Int, + context: InterpreterContext): java.util.List[InterpreterCompletion] = { + val completions = scalaCompleter.complete(buf, cursor).candidates + .map(e => new InterpreterCompletion(e, e, null)) + scala.collection.JavaConversions.seqAsJavaList(completions) + } + + protected def getProgress(jobGroup: String, context: InterpreterContext): Int = { + val jobIds = sc.statusTracker.getJobIdsForGroup(jobGroup) + val jobs = jobIds.flatMap { id => sc.statusTracker.getJobInfo(id) } + val stages = jobs.flatMap { job => + job.stageIds().flatMap(sc.statusTracker.getStageInfo) + } + + val taskCount = stages.map(_.numTasks).sum + val completedTaskCount = stages.map(_.numCompletedTasks).sum + if (taskCount == 0) { + 0 + } else { + (100 * completedTaskCount.toDouble / taskCount).toInt + } + } + + protected def bind(name: String, tpe: String, value: Object, modifier: List[String]): Unit + + // for use in java side + protected def bind(name: String, + tpe: String, + value: Object, + modifier: java.util.List[String]): Unit = + bind(name, tpe, value, modifier.asScala.toList) + + protected def close(): Unit = { + if (sc != null) { + sc.stop() + } + if (sparkHttpServer != null) { + sparkHttpServer.getClass.getMethod("stop").invoke(sparkHttpServer) + } + sc = null + sqlContext = null + if (sparkSession != null) { + sparkSession.getClass.getMethod("stop").invoke(sparkSession) + sparkSession = null + } + + } + + protected def createSparkContext(): Unit = { + if (isSparkSessionPresent()) { + spark2CreateContext() + } else { + spark1CreateContext() + } + } + + private def spark1CreateContext(): Unit = { + this.sc = SparkContext.getOrCreate(conf) + if (!isTest) { + interpreterOutput.write("Created SparkContext.\n".getBytes()) + } + getUserFiles().foreach(file => sc.addFile(file)) + + sc.getClass.getMethod("ui").invoke(sc).asInstanceOf[Option[_]] match { + case Some(webui) => + sparkUrl = webui.getClass.getMethod("appUIAddress").invoke(webui).asInstanceOf[String] + case None => + } + + val hiveSiteExisted: Boolean = + Thread.currentThread().getContextClassLoader.getResource("hive-site.xml") != null + val hiveEnabled = conf.getBoolean("spark.useHiveContext", false) + if (hiveEnabled && hiveSiteExisted) { + sqlContext = Class.forName("org.apache.spark.sql.hive.HiveContext") + .getConstructor(classOf[SparkContext]).newInstance(sc).asInstanceOf[SQLContext] + if (!isTest) { + interpreterOutput.write("Created sql context (with Hive support).\n".getBytes()) + } + } else { + if (hiveEnabled && !hiveSiteExisted && !isTest) { + interpreterOutput.write(("spark.useHiveContext is set as true but no hive-site.xml" + + " is found in classpath, so zeppelin will fallback to SQLContext.\n").getBytes()) + } + sqlContext = Class.forName("org.apache.spark.sql.SQLContext") + .getConstructor(classOf[SparkContext]).newInstance(sc).asInstanceOf[SQLContext] + if (!isTest) { + interpreterOutput.write("Created sql context.\n".getBytes()) + } + } + + bind("sc", "org.apache.spark.SparkContext", sc, List("""@transient""")) + bind("sqlContext", sqlContext.getClass.getCanonicalName, sqlContext, List("""@transient""")) + + interpret("import org.apache.spark.SparkContext._") + interpret("import sqlContext.implicits._") + interpret("import sqlContext.sql") + interpret("import org.apache.spark.sql.functions._") + } + + private def spark2CreateContext(): Unit = { + val sparkClz = Class.forName("org.apache.spark.sql.SparkSession$") + val sparkObj = sparkClz.getField("MODULE$").get(null) + + val builderMethod = sparkClz.getMethod("builder") + val builder = builderMethod.invoke(sparkObj) + builder.getClass.getMethod("config", classOf[SparkConf]).invoke(builder, conf) + + if (conf.get("spark.sql.catalogImplementation", "in-memory").toLowerCase == "hive" + || conf.get("spark.useHiveContext", "false").toLowerCase == "true") { + val hiveSiteExisted: Boolean = + Thread.currentThread().getContextClassLoader.getResource("hive-site.xml") != null + val hiveClassesPresent = + sparkClz.getMethod("hiveClassesArePresent").invoke(sparkObj).asInstanceOf[Boolean] + if (hiveSiteExisted && hiveClassesPresent) { + builder.getClass.getMethod("enableHiveSupport").invoke(builder) + sparkSession = builder.getClass.getMethod("getOrCreate").invoke(builder) + if (!isTest) { + interpreterOutput.write("Created Spark session (with Hive support).\n".getBytes()) + } + } else { + if (!hiveClassesPresent && !isTest) { + interpreterOutput.write( + "Hive support can not be enabled because spark is not built with hive\n".getBytes) + } + if (!hiveSiteExisted && !isTest) { + interpreterOutput.write( + "Hive support can not be enabled because no hive-site.xml found\n".getBytes) + } + sparkSession = builder.getClass.getMethod("getOrCreate").invoke(builder) + if (!isTest) { + interpreterOutput.write("Created Spark session.\n".getBytes()) + } + } + } else { + sparkSession = builder.getClass.getMethod("getOrCreate").invoke(builder) + if (!isTest) { + interpreterOutput.write("Created Spark session.\n".getBytes()) + } + } + + sc = sparkSession.getClass.getMethod("sparkContext").invoke(sparkSession) + .asInstanceOf[SparkContext] + getUserFiles().foreach(file => sc.addFile(file)) + sqlContext = sparkSession.getClass.getMethod("sqlContext").invoke(sparkSession) + .asInstanceOf[SQLContext] + sc.getClass.getMethod("uiWebUrl").invoke(sc).asInstanceOf[Option[String]] match { + case Some(url) => sparkUrl = url + case None => + } + + bind("spark", sparkSession.getClass.getCanonicalName, sparkSession, List("""@transient""")) + bind("sc", "org.apache.spark.SparkContext", sc, List("""@transient""")) + bind("sqlContext", "org.apache.spark.sql.SQLContext", sqlContext, List("""@transient""")) + + interpret("import org.apache.spark.SparkContext._") + interpret("import spark.implicits._") + interpret("import spark.sql") + interpret("import org.apache.spark.sql.functions._") + } + + private def isSparkSessionPresent(): Boolean = { + try { + Class.forName("org.apache.spark.sql.SparkSession") + true + } catch { + case _: ClassNotFoundException | _: NoClassDefFoundError => false + } + } + + protected def getField(obj: Object, name: String): Object = { + val field = obj.getClass.getField(name) + field.setAccessible(true) + field.get(obj) + } + + protected def getDeclareField(obj: Object, name: String): Object = { + val field = obj.getClass.getDeclaredField(name) + field.setAccessible(true) + field.get(obj) + } + + protected def setDeclaredField(obj: Object, name: String, value: Object): Unit = { + val field = obj.getClass.getDeclaredField(name) + field.setAccessible(true) + field.set(obj, value) + } + + protected def callMethod(obj: Object, name: String): Object = { + callMethod(obj, name, Array.empty[Class[_]], Array.empty[Object]) + } + + protected def callMethod(obj: Object, name: String, + parameterTypes: Array[Class[_]], + parameters: Array[Object]): Object = { + val method = obj.getClass.getMethod(name, parameterTypes: _ *) + method.setAccessible(true) + method.invoke(obj, parameters: _ *) + } + + protected def startHttpServer(outputDir: File): Option[(Object, String)] = { + try { + val httpServerClass = Class.forName("org.apache.spark.HttpServer") + val securityManager = { + val constructor = Class.forName("org.apache.spark.SecurityManager") + .getConstructor(classOf[SparkConf]) + constructor.setAccessible(true) + constructor.newInstance(conf).asInstanceOf[Object] + } + val httpServerConstructor = httpServerClass + .getConstructor(classOf[SparkConf], + classOf[File], + Class.forName("org.apache.spark.SecurityManager"), + classOf[Int], + classOf[String]) + httpServerConstructor.setAccessible(true) + // Create Http Server + val port = conf.getInt("spark.replClassServer.port", 0) + val server = httpServerConstructor + .newInstance(conf, outputDir, securityManager, new Integer(port), "HTTP server") + .asInstanceOf[Object] + + // Start Http Server + val startMethod = server.getClass.getMethod("start") + startMethod.setAccessible(true) + startMethod.invoke(server) + + // Get uri of this Http Server + val uriMethod = server.getClass.getMethod("uri") + uriMethod.setAccessible(true) + val uri = uriMethod.invoke(server).asInstanceOf[String] + Some((server, uri)) + } catch { + // Spark 2.0+ removed HttpServer, so return null instead. + case NonFatal(e) => + None + } + } + + protected def getUserJars(): Seq[String] = { + val sparkJars = conf.getOption("spark.jars").map(_.split(",")) + .map(_.filter(_.nonEmpty)).toSeq.flatten + val depJars = depFiles.asScala.filter(_.endsWith(".jar")) + val result = sparkJars ++ depJars + conf.set("spark.jars", result.mkString(",")) + result + } + + protected def getUserFiles(): Seq[String] = { + depFiles.asScala.filter(!_.endsWith(".jar")) + } +} diff --git a/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java b/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java deleted file mode 100644 index b226a001d24..00000000000 --- a/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import static org.junit.Assert.assertEquals; - -import org.junit.Test; - -public class SparkDependencyResolverTest { - - @Test - public void testInferScalaVersion() { - String [] version = scala.util.Properties.versionNumberString().split("[.]"); - String scalaVersion = version[0] + "." + version[1]; - - assertEquals("groupId:artifactId:version", - SparkDependencyResolver.inferScalaVersion("groupId:artifactId:version")); - assertEquals("groupId:artifactId_" + scalaVersion + ":version", - SparkDependencyResolver.inferScalaVersion("groupId::artifactId:version")); - assertEquals("groupId:artifactId:version::test", - SparkDependencyResolver.inferScalaVersion("groupId:artifactId:version::test")); - assertEquals("*", - SparkDependencyResolver.inferScalaVersion("*")); - assertEquals("groupId:*", - SparkDependencyResolver.inferScalaVersion("groupId:*")); - assertEquals("groupId:artifactId*", - SparkDependencyResolver.inferScalaVersion("groupId:artifactId*")); - assertEquals("groupId:artifactId_" + scalaVersion, - SparkDependencyResolver.inferScalaVersion("groupId::artifactId")); - assertEquals("groupId:artifactId_" + scalaVersion + "*", - SparkDependencyResolver.inferScalaVersion("groupId::artifactId*")); - assertEquals("groupId:artifactId_" + scalaVersion + ":*", - SparkDependencyResolver.inferScalaVersion("groupId::artifactId:*")); - } - -} diff --git a/testing/install_external_dependencies.sh b/testing/install_external_dependencies.sh index e34296e3ab6..d6c07368b9d 100755 --- a/testing/install_external_dependencies.sh +++ b/testing/install_external_dependencies.sh @@ -44,6 +44,6 @@ if [[ -n "$PYTHON" ]] ; then conda update -q conda conda info -a conda config --add channels conda-forge - conda install -q matplotlib pandasql ipython=5.4.1 jupyter_client ipykernel matplotlib bokeh=0.12.6 - pip install -q grpcio ggplot + conda install -q matplotlib pandasql ipython=5.4.1 jupyter_client ipykernel matplotlib bokeh=0.12.10 + pip install -q grpcio ggplot bkzep==0.4.0 fi diff --git a/zeppelin-display/pom.xml b/zeppelin-display/pom.xml index c6edd95e42f..79a08a69a6d 100644 --- a/zeppelin-display/pom.xml +++ b/zeppelin-display/pom.xml @@ -27,7 +27,7 @@ org.apache.zeppelin - zeppelin-display_2.10 + zeppelin-display jar 0.9.0-SNAPSHOT Zeppelin: Display system apis @@ -45,18 +45,21 @@ org.scala-lang scala-library ${scala.version} + provided org.scala-lang scala-compiler ${scala.version} + provided org.scala-lang scalap ${scala.version} + provided @@ -84,13 +87,6 @@ test - - org.scala-lang - scala-library - ${scala.version} - provided - - org.scalatest scalatest_${scala.binary.version} diff --git a/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java b/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java index f7bb7768d8c..1804fc4cb24 100644 --- a/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java +++ b/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java @@ -184,7 +184,7 @@ public void testSqlSpark() throws Exception { } } - @Test +// @Test public void testDep() throws Exception { try { // restart spark interpreter before running %dep diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java index 65bb06fe143..e38a29f82b8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java @@ -237,6 +237,8 @@ public void show(Object o, int maxResult) { if (isSupportedObject(o)) { interpreterContext.out.write(showData(o)); } else { + interpreterContext.out.write("ZeppelinContext doesn't support to show type: " + + o.getClass().getCanonicalName() + "\n"); interpreterContext.out.write(o.toString()); } } catch (IOException e) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index fca84498aa4..37db1fce8ec 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -96,10 +96,10 @@ * Entry point for Interpreter process. * Accepting thrift connections from ZeppelinServer. */ -public class RemoteInterpreterServer - extends Thread +public class RemoteInterpreterServer extends Thread implements RemoteInterpreterService.Iface, AngularObjectRegistryListener { - Logger logger = LoggerFactory.getLogger(RemoteInterpreterServer.class); + + private static Logger logger = LoggerFactory.getLogger(RemoteInterpreterServer.class); InterpreterGroup interpreterGroup; AngularObjectRegistry angularObjectRegistry; @@ -255,6 +255,9 @@ public boolean isRunning() { public static void main(String[] args) throws TTransportException, InterruptedException, IOException { + Class klass = RemoteInterpreterServer.class; + URL location = klass.getResource('/' + klass.getName().replace('.', '/') + ".class"); + logger.info("URL:" + location); String callbackHost = null; int port = Constants.ZEPPELIN_INTERPRETER_DEFAUlT_PORT; String portRange = ":"; diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index a73cd9643f0..970f302fec0 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -261,6 +261,12 @@ scalatest_${scala.binary.version} ${scalatest.version} test + + + org.scala-lang.modules + scala-xml_${scala.binary.version} + + diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 7d4c21cb7d3..519342012ac 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -265,21 +265,21 @@ private static void start(boolean withAuth, String testClassName, boolean withKn // set spark master and other properties sparkProperties.put("master", new InterpreterProperty("master", "local[2]", InterpreterPropertyType.TEXTAREA.getValue())); + sparkProperties.put("spark.master", + new InterpreterProperty("spark.master", "local[2]", InterpreterPropertyType.TEXTAREA.getValue())); sparkProperties.put("spark.cores.max", new InterpreterProperty("spark.cores.max", "2", InterpreterPropertyType.TEXTAREA.getValue())); sparkProperties.put("zeppelin.spark.useHiveContext", new InterpreterProperty("zeppelin.spark.useHiveContext", false, InterpreterPropertyType.CHECKBOX.getValue())); - // set spark home for pyspark - sparkProperties.put("spark.home", - new InterpreterProperty("spark.home", getSparkHome(), InterpreterPropertyType.TEXTAREA.getValue())); sparkProperties.put("zeppelin.pyspark.useIPython", new InterpreterProperty("zeppelin.pyspark.useIPython", "false", InterpreterPropertyType.TEXTAREA.getValue())); - + sparkProperties.put("zeppelin.spark.test", new InterpreterProperty("zeppelin.spark.test", "true", InterpreterPropertyType.TEXTAREA.getValue())); sparkIntpSetting.setProperties(sparkProperties); pySpark = true; sparkR = true; ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId()); } else { String sparkHome = getSparkHome(); + LOG.info("SPARK HOME detected " + sparkHome); if (sparkHome != null) { if (System.getenv("SPARK_MASTER") != null) { sparkProperties.put("master", @@ -288,14 +288,14 @@ private static void start(boolean withAuth, String testClassName, boolean withKn sparkProperties.put("master", new InterpreterProperty("master", "local[2]", InterpreterPropertyType.TEXTAREA.getValue())); } + sparkProperties.put("spark.master", + new InterpreterProperty("spark.master", "local[2]", InterpreterPropertyType.TEXTAREA.getValue())); sparkProperties.put("spark.cores.max", new InterpreterProperty("spark.cores.max", "2", InterpreterPropertyType.TEXTAREA.getValue())); - // set spark home for pyspark - sparkProperties.put("spark.home", - new InterpreterProperty("spark.home", sparkHome, InterpreterPropertyType.TEXTAREA.getValue())); sparkProperties.put("zeppelin.spark.useHiveContext", new InterpreterProperty("zeppelin.spark.useHiveContext", false, InterpreterPropertyType.CHECKBOX.getValue())); sparkProperties.put("zeppelin.pyspark.useIPython", new InterpreterProperty("zeppelin.pyspark.useIPython", "false", InterpreterPropertyType.TEXTAREA.getValue())); + sparkProperties.put("zeppelin.spark.test", new InterpreterProperty("zeppelin.spark.test", "true", InterpreterPropertyType.TEXTAREA.getValue())); pySpark = true; sparkR = true; @@ -333,7 +333,6 @@ private static String getSparkHome() { return sparkHome; } sparkHome = getSparkHomeRecursively(new File(System.getProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName()))); - System.out.println("SPARK HOME detected " + sparkHome); return sparkHome; } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index 615675544ec..f3a70996414 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -167,8 +167,8 @@ public void sparkSQLTest() throws IOException { assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(1).getType()); assertEquals("_1\t_2\nhello\t20\n", p.getResult().message().get(1).getData()); } - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); } + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); } @Test @@ -470,7 +470,7 @@ public void pySparkDepLoaderTest() throws IOException, InterpreterException { p1.setText("%pyspark\n" + "from pyspark.sql import SQLContext\n" + "print(" + sqlContextName + ".read.format('com.databricks.spark.csv')" + - ".load('"+ tmpFile.getAbsolutePath() +"').count())"); + ".load('" + tmpFile.getAbsolutePath() +"').count())"); p1.setAuthenticationInfo(anonymous); note.run(p1.getId()); @@ -576,6 +576,7 @@ public void testPySparkZeppelinContextDynamicForms() throws IOException { @Test public void testConfInterpreter() throws IOException { + ZeppelinServer.notebook.getInterpreterSettingManager().close(); Note note = ZeppelinServer.notebook.createNote(AuthenticationInfo.ANONYMOUS); Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); Map config = p.getConfig(); diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index ac7536018b1..fade4dd6123 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -603,7 +603,7 @@ org.apache.zeppelin - zeppelin-spark_2.10 + spark-interpreter ${project.version} test From 6f4f0ff96c800a7f78df2b4401e04147400378f9 Mon Sep 17 00:00:00 2001 From: skymon Date: Wed, 31 Jan 2018 11:54:53 +0100 Subject: [PATCH 009/386] Update configuration.md Corrected spelling "ba" -> "be" ### What is this PR for? Minor spelling correction ### What type of PR is it? Documentation ### Todos - ### What is the Jira issue - ### How should this be tested? - ### Screenshots (if appropriate) ### Questions: - Author: skymon Closes #2754 from skymon/patch-1 and squashes the following commits: c4a7be75e [skymon] Update configuration.md --- docs/setup/operation/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md index 458affc9287..1f4c6a24231 100644 --- a/docs/setup/operation/configuration.md +++ b/docs/setup/operation/configuration.md @@ -27,7 +27,7 @@ limitations under the License. There are two locations you can configure Apache Zeppelin. * **Environment variables** can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows). -* **Java properties** can ba defined in `conf/zeppelin-site.xml`. +* **Java properties** can be defined in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority. > Mouse hover on each property and click then you can get a link for that. From cb174ffc6a4464e75342b809edc59273d3b4af6a Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Thu, 1 Feb 2018 15:53:29 +0100 Subject: [PATCH 010/386] ZEPPELIN-3202. Added missing test dependencies in the scio module ### What is this PR for? Added missing test dependencies for the **scio** module to prevent test failures when running `mvn clean install`. ### What type of PR is it? Bug Fix ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3202 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2759 from HorizonNet/ZEPPELIN-3202 and squashes the following commits: 2d9ffdf [Jan Hentschel] ZEPPELIN-3202. Added missing test dependencies in the scio module --- scio/pom.xml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scio/pom.xml b/scio/pom.xml index a62ff2b83e4..08d0696850a 100644 --- a/scio/pom.xml +++ b/scio/pom.xml @@ -43,6 +43,9 @@ 2.3 2.15.2 1.7.7 + + + 1.3 @@ -101,6 +104,25 @@ test + + org.hamcrest + hamcrest-all + ${hamcrest.all.version} + test + + + + com.google.code.gson + gson + test + + + + commons-lang + commons-lang + test + + From c6b73beae71f79b48d834dc751182ec45d244d69 Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Mon, 5 Feb 2018 23:10:23 +0100 Subject: [PATCH 011/386] ZEPPELIN-3145. Fixed Checkstyle errors and warnings in the shell module ### What is this PR for? Fixed all Checkstyle errors and warnings in the **shell** module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3145 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2767 from HorizonNet/ZEPPELIN-3145 and squashes the following commits: 206ffd0 [Jan Hentschel] ZEPPELIN-3145. Fixed Checkstyle errors and warnings in the shell module --- shell/pom.xml | 7 +++++ .../zeppelin/shell/ShellInterpreter.java | 31 ++++++++++--------- .../zeppelin/shell/ShellInterpreterTest.java | 7 +++-- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/shell/pom.xml b/shell/pom.xml index 6a7fda9ed01..9f51dcc5acd 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -88,6 +88,13 @@ maven-resources-plugin + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java index 970720554c1..9f6b11d3f0f 100644 --- a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java +++ b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java @@ -17,30 +17,31 @@ package org.apache.zeppelin.shell; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.OutputStream; -import java.io.File; import java.util.List; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.ExecuteException; -import org.apache.commons.exec.ExecuteWatchdog; -import org.apache.commons.exec.PumpStreamHandler; -import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.KerberosInterpreter; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.KerberosInterpreter; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Shell interpreter for Zeppelin. @@ -49,7 +50,7 @@ public class ShellInterpreter extends KerberosInterpreter { private static final Logger LOGGER = LoggerFactory.getLogger(ShellInterpreter.class); private static final String TIMEOUT_PROPERTY = "shell.command.timeout.millisecs"; - private String DEFAULT_TIMEOUT_PROPERTY = "60000"; + private String defaultTimeoutProperty = "60000"; private static final String DIRECTORY_USER_HOME = "shell.working.directory.user.home"; private final boolean isWindows = System.getProperty("os.name").startsWith("Windows"); @@ -100,10 +101,10 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr try { DefaultExecutor executor = new DefaultExecutor(); executor.setStreamHandler(new PumpStreamHandler( - contextInterpreter.out, contextInterpreter.out)); + contextInterpreter.out, contextInterpreter.out)); executor.setWatchdog(new ExecuteWatchdog( - Long.valueOf(getProperty(TIMEOUT_PROPERTY, DEFAULT_TIMEOUT_PROPERTY)))); + Long.valueOf(getProperty(TIMEOUT_PROPERTY, defaultTimeoutProperty)))); executors.put(contextInterpreter.getParagraphId(), executor); if (Boolean.valueOf(getProperty(DIRECTORY_USER_HOME))) { executor.setWorkingDirectory(new File(System.getProperty("user.home"))); @@ -111,7 +112,7 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr int exitVal = executor.execute(cmdLine); LOGGER.info("Paragraph " + contextInterpreter.getParagraphId() - + " return with exit value: " + exitVal); + + " return with exit value: " + exitVal); return new InterpreterResult(Code.SUCCESS, outStream.toString()); } catch (ExecuteException e) { int exitValue = e.getExitValue(); @@ -122,7 +123,7 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr code = Code.INCOMPLETE; message += "Paragraph received a SIGTERM\n"; LOGGER.info("The paragraph " + contextInterpreter.getParagraphId() - + " stopped executing: " + message); + + " stopped executing: " + message); } message += "ExitValue: " + exitValue; return new InterpreterResult(code, message); diff --git a/shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java b/shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java index b67170c14a2..1b76346659f 100644 --- a/shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java +++ b/shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java @@ -20,14 +20,15 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + import java.util.Properties; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; public class ShellInterpreterTest { From 7691b7f09c6a2bd59f310656771d7cb7916a5a94 Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Wed, 7 Feb 2018 13:15:18 +0100 Subject: [PATCH 012/386] ZEPPELIN-3141. Fixed Checkstyle errors and warnings in the pig module ### What is this PR for? Fixed the Checkstyle errors and warnings in the pig module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3141 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2773 from HorizonNet/ZEPPELIN-3141 and squashes the following commits: 080fad5 [Jan Hentschel] ZEPPELIN-3141. Fixed Checkstyle errors and warnings in the pig module --- pig/pom.xml | 7 +++++ .../zeppelin/pig/BasePigInterpreter.java | 12 ++++----- .../apache/zeppelin/pig/PigInterpreter.java | 15 ++++++----- .../zeppelin/pig/PigQueryInterpreter.java | 16 ++++++----- .../zeppelin/pig/PigScriptListener.java | 4 +-- .../org/apache/zeppelin/pig/PigUtils.java | 24 ++--------------- .../zeppelin/pig/PigInterpreterSparkTest.java | 17 ++++++------ .../zeppelin/pig/PigInterpreterTest.java | 23 ++++++++-------- .../zeppelin/pig/PigInterpreterTezTest.java | 21 ++++++++------- .../zeppelin/pig/PigQueryInterpreterTest.java | 27 ++++++++++--------- 10 files changed, 82 insertions(+), 84 deletions(-) diff --git a/pig/pom.xml b/pig/pom.xml index 4553b5cf5cd..571d198b449 100644 --- a/pig/pom.xml +++ b/pig/pom.xml @@ -190,6 +190,13 @@ always + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java index 1fb2a69c37e..9503aa7ba2e 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java @@ -23,10 +23,6 @@ import org.apache.pig.backend.BackendException; import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; import org.apache.pig.backend.hadoop.executionengine.Launcher; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,12 +31,16 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; + /** * */ public abstract class BasePigInterpreter extends Interpreter { - - private static Logger LOGGER = LoggerFactory.getLogger(BasePigInterpreter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BasePigInterpreter.class); protected ConcurrentHashMap listenerMap = new ConcurrentHashMap<>(); diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java index 0f2d59bb9e6..4fc0676f9e3 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java @@ -22,23 +22,26 @@ import org.apache.pig.PigServer; import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.tools.pigscript.parser.ParseException; -import org.apache.pig.tools.pigstats.*; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.pig.tools.pigstats.PigStats; +import org.apache.pig.tools.pigstats.ScriptState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.io.PrintStream; -import java.util.*; +import java.util.Map; +import java.util.Properties; + +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; /** * Pig interpreter for Zeppelin. */ public class PigInterpreter extends BasePigInterpreter { - private static Logger LOGGER = LoggerFactory.getLogger(PigInterpreter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(PigInterpreter.class); private PigServer pigServer; private boolean includeJobStats = false; diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java index da3d50e09bd..364d412faaa 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java @@ -15,10 +15,8 @@ * limitations under the License. */ - package org.apache.zeppelin.pig; - import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.pig.PigServer; @@ -28,8 +26,6 @@ import org.apache.pig.tools.pigscript.parser.ParseException; import org.apache.pig.tools.pigstats.PigStats; import org.apache.pig.tools.pigstats.ScriptState; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,12 +36,20 @@ import java.util.List; import java.util.Properties; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.apache.zeppelin.interpreter.ResultMessages; +import org.apache.zeppelin.interpreter.WrappedInterpreter; + /** * */ public class PigQueryInterpreter extends BasePigInterpreter { - - private static Logger LOGGER = LoggerFactory.getLogger(PigQueryInterpreter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(PigQueryInterpreter.class); private static final String MAX_RESULTS = "zeppelin.pig.maxResult"; private PigServer pigServer; private int maxResult; diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java b/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java index 1f88b2ee6cc..8ff1bf8986e 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java @@ -15,7 +15,6 @@ * limitations under the License. */ - package org.apache.zeppelin.pig; import org.apache.pig.impl.plan.OperatorPlan; @@ -32,8 +31,7 @@ * */ public class PigScriptListener implements PigProgressNotificationListener { - - private static Logger LOGGER = LoggerFactory.getLogger(PigScriptListener.class); + private static final Logger LOGGER = LoggerFactory.getLogger(PigScriptListener.class); private Set jobIds = new HashSet(); private int progress; diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java index 8fc69ed4013..1c48250434d 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java @@ -17,40 +17,21 @@ package org.apache.zeppelin.pig; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.pig.PigRunner; -import org.apache.pig.backend.hadoop.executionengine.spark.plan.SparkOperator; -import org.apache.pig.backend.hadoop.executionengine.tez.TezExecType; -import org.apache.pig.tools.pigstats.InputStats; -import org.apache.pig.tools.pigstats.JobStats; -import org.apache.pig.tools.pigstats.OutputStats; -import org.apache.pig.tools.pigstats.PigStats; -import org.apache.pig.tools.pigstats.mapreduce.MRJobStats; -import org.apache.pig.tools.pigstats.mapreduce.SimplePigStats; -import org.apache.pig.tools.pigstats.spark.SparkJobStats; -import org.apache.pig.tools.pigstats.spark.SparkPigStats; -import org.apache.pig.tools.pigstats.spark.SparkScriptState; -import org.apache.pig.tools.pigstats.tez.TezDAGStats; -import org.apache.pig.tools.pigstats.tez.TezPigScriptStats; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.lang.reflect.Field; -import java.text.SimpleDateFormat; -import java.util.*; +import java.util.List; /** * */ public class PigUtils { - - private static Logger LOGGER = LoggerFactory.getLogger(PigUtils.class); + private static final Logger LOGGER = LoggerFactory.getLogger(PigUtils.class); protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; @@ -66,5 +47,4 @@ public static File createTempPigScript(String content) throws IOException { public static File createTempPigScript(List lines) throws IOException { return createTempPigScript(StringUtils.join(lines, "\n")); } - } diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java index 43ade16990d..d25d6e25771 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java @@ -16,12 +16,12 @@ * limitations under the License. */ - package org.apache.zeppelin.pig; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import org.apache.commons.io.IOUtils; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterResult; import org.junit.After; import org.junit.Test; @@ -30,9 +30,8 @@ import java.io.IOException; import java.util.Properties; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; public class PigInterpreterSparkTest { private PigInterpreter pigInterpreter; @@ -86,7 +85,8 @@ public void testBasics() throws IOException { result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // syntax error pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" @@ -133,7 +133,8 @@ public void testIncludeJobStats() throws IOException { assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.ERROR, result.code()); // no job is launched, so no jobStats - assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // execution error pigscript = "a = load 'invalid_path';" diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java index ac1339068ad..59ea75365b2 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java @@ -16,16 +16,13 @@ * limitations under the License. */ - package org.apache.zeppelin.pig; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import org.apache.commons.io.IOUtils; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.InterpreterResult.Type; import org.junit.After; -import org.junit.Before; import org.junit.Test; import java.io.File; @@ -33,8 +30,10 @@ import java.io.IOException; import java.util.Properties; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterResult.Type; public class PigInterpreterTest { @@ -48,7 +47,7 @@ private void setUpLocal(boolean includeJobStats) { pigInterpreter = new PigInterpreter(properties); pigInterpreter.open(); context = new InterpreterContext(null, "paragraph_id", null, null, null, - null, null, null, null, null, null,null, null); + null, null, null, null, null, null, null, null); } @After @@ -89,7 +88,8 @@ public void testBasics() throws IOException { result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); - assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // execution error pigscript = "a = load 'invalid_path';" @@ -139,7 +139,8 @@ public void testIncludeJobStats() throws IOException { assertEquals(Code.ERROR, result.code()); // no job is launched, so no jobStats assertTrue(!result.message().get(0).getData().contains("Counters:")); - assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // execution error pigscript = "a = load 'invalid_path';" diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java index 48f07bf147a..409502b40a7 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java @@ -16,16 +16,13 @@ * limitations under the License. */ - package org.apache.zeppelin.pig; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import org.apache.commons.io.IOUtils; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.InterpreterResult.Type; import org.junit.After; -import org.junit.Before; import org.junit.Test; import java.io.File; @@ -33,8 +30,10 @@ import java.io.IOException; import java.util.Properties; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterResult.Type; public class PigInterpreterTezTest { @@ -94,7 +93,8 @@ public void testBasics() throws IOException { result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); - assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // syntax error pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" @@ -143,7 +143,8 @@ public void testIncludeJobStats() throws IOException { assertEquals(Code.ERROR, result.code()); // no job is launched, so no jobStats assertTrue(!result.message().get(0).getData().contains("Vertex Stats")); - assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // execution error pigscript = "a = load 'invalid_path';" diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java index ad395b5814d..2ca586bcdfd 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java @@ -18,15 +18,12 @@ package org.apache.zeppelin.pig; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import org.apache.commons.io.IOUtils; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterResult; import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import java.io.File; @@ -36,8 +33,11 @@ import java.util.List; import java.util.Properties; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterResult; /** * @@ -93,7 +93,8 @@ public void testBasics() throws IOException { InterpreterResult result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertTrue(result.message().get(0).getData().contains("(andy,male,10)\n(peter,male,20)\n(amy,female,14)")); + assertTrue(result.message().get(0).getData().contains( + "(andy,male,10)\n(peter,male,20)\n(amy,female,14)")); // run single line query in PigQueryInterpreter String query = "foreach a generate name, age;"; @@ -117,11 +118,13 @@ public void testBasics() throws IOException { assertEquals("group\tcol_1\nmale\t2\nfemale\t1\n", result.message().get(0).getData()); // syntax error in PigQueryInterpereter - query = "b = group a by invalid_column;\nforeach b generate group as gender, COUNT($1) as count;"; + query = "b = group a by invalid_column;\nforeach b generate group as gender, " + + "COUNT($1) as count;"; result = pigQueryInterpreter.interpret(query, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertTrue(result.message().get(0).getData().contains("Projected field [invalid_column] does not exist in schema")); + assertTrue(result.message().get(0).getData().contains( + "Projected field [invalid_column] does not exist in schema")); // execution error in PigQueryInterpreter query = "foreach a2 generate name, age;"; @@ -134,7 +137,7 @@ public void testBasics() throws IOException { @Test public void testMaxResult() throws IOException { StringBuilder content = new StringBuilder(); - for (int i=0;i<30;++i) { + for (int i = 0; i < 30; ++i) { content.append(i + "\tname_" + i + "\n"); } File tmpFile = File.createTempFile("zeppelin", "test"); From d07c70a6dc32d3d2668198b2e4c10c57602f8ab8 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 7 Feb 2018 14:47:46 +0800 Subject: [PATCH 013/386] ZEPPELIN-3171. Restart of interpreter in note also aborts running interpreter in another note ### What is this PR for? The root cause is that in isolated mode interpreters will share the same scheduler. That means when one interpreter is terminated, all the running jobs under the scheduler of this interpreter will be aborted too. This PR will create one scheduler for each session. So when one session is closed, only the running jobs under this session's scheduler is aborted. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3171 ### How should this be tested? * Unit test is added. * Also verify it manually. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2769 from zjffdu/ZEPPELIN-3171 and squashes the following commits: 3586f45 [Jeff Zhang] ZEPPELIN-3171. Restart of interpreter in note also aborts running interpreter in another note --- .../zeppelin/interpreter/remote/RemoteInterpreter.java | 7 ++++--- .../zeppelin/interpreter/remote/RemoteInterpreterTest.java | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index bda8010d93c..f38d037243e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -380,15 +380,16 @@ public String call(Client client) throws Exception { }); } - //TODO(zjffdu) Share the Scheduler in the same session or in the same InterpreterGroup ? + @Override public Scheduler getScheduler() { int maxConcurrency = Integer.parseInt( getProperty("zeppelin.interpreter.max.poolsize", ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE.getIntValue() + "")); - + // one session own one Scheduler, so that when one session is closed, all the jobs/paragraphs + // running under the scheduler of this session will be aborted. Scheduler s = new RemoteScheduler( - RemoteInterpreter.class.getName() + "-" + sessionId, + RemoteInterpreter.class.getName() + "-" + getInterpreterGroup().getId() + "-" + sessionId, SchedulerFactory.singleton().getExecutor(), sessionId, this, diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index 7f9978a5aa8..04b7a5bf1a2 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -93,6 +93,8 @@ public void testSharedMode() throws InterpreterException, IOException { assertTrue(interpreter2 instanceof RemoteInterpreter); RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2; + assertEquals(remoteInterpreter1.getScheduler(), remoteInterpreter2.getScheduler()); + InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl", "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap(), new GUI(), new GUI(), null, null, new ArrayList(), null); @@ -136,6 +138,8 @@ public void testScopedMode() throws InterpreterException, IOException { assertTrue(interpreter2 instanceof RemoteInterpreter); RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2; + assertNotEquals(interpreter1.getScheduler(), interpreter2.getScheduler()); + InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl", "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap(), new GUI(), new GUI(), null, null, new ArrayList(), null); @@ -182,6 +186,8 @@ public void testIsolatedMode() throws InterpreterException, IOException { assertTrue(interpreter2 instanceof RemoteInterpreter); RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2; + assertNotEquals(interpreter1.getScheduler(), interpreter2.getScheduler()); + InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl", "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap(), new GUI(), new GUI(), null, null, new ArrayList(), null); From 5b01477451a2c1cceddb5313714677e861280dc5 Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Sat, 3 Feb 2018 19:46:39 +0100 Subject: [PATCH 014/386] ZEPPELIN-3146. Fixed Checkstyle errors in alluxio module ### What is this PR for? Fix the Checkstyle errors and warning in the **alluxio** module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3146 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2764 from HorizonNet/ZEPPELIN-3146 and squashes the following commits: 8269db6 [Jan Hentschel] ZEPPELIN-3146. Fixed Checkstyle errors in alluxio module --- alluxio/pom.xml | 7 ++ .../zeppelin/alluxio/AlluxioInterpreter.java | 21 +++--- .../alluxio/AlluxioInterpreterTest.java | 71 +++++++++++-------- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/alluxio/pom.xml b/alluxio/pom.xml index 6ef7a01b2d0..af23c87b673 100644 --- a/alluxio/pom.xml +++ b/alluxio/pom.xml @@ -137,6 +137,13 @@ maven-resources-plugin + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java b/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java index 8eb152bae71..be912ecab5e 100644 --- a/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java +++ b/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java @@ -18,10 +18,20 @@ package org.apache.zeppelin.alluxio; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; -import java.io.ByteArrayOutputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import alluxio.Configuration; +import alluxio.shell.AlluxioShell; import org.apache.zeppelin.completer.CompletionType; import org.apache.zeppelin.interpreter.Interpreter; @@ -29,11 +39,6 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import alluxio.Configuration; -import alluxio.shell.AlluxioShell; /** * Alluxio interpreter for Zeppelin. @@ -71,7 +76,7 @@ public AlluxioInterpreter(Properties property) { @Override public void open() { logger.info("Starting Alluxio shell to connect to " + alluxioMasterHostname + - " on port " + alluxioMasterPort); + " on port " + alluxioMasterPort); System.setProperty(ALLUXIO_MASTER_HOSTNAME, alluxioMasterHostname); System.setProperty(ALLUXIO_MASTER_PORT, alluxioMasterPort); diff --git a/alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java b/alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java index e272a51e507..06711de3265 100644 --- a/alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java +++ b/alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java @@ -18,6 +18,11 @@ package org.apache.zeppelin.alluxio; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -27,28 +32,25 @@ import java.util.List; import java.util.Properties; -import alluxio.client.WriteType; -import alluxio.client.file.URIStatus; - -import org.apache.zeppelin.completer.CompletionType; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.junit.*; - -import alluxio.Constants; import alluxio.AlluxioURI; +import alluxio.Constants; import alluxio.client.FileSystemTestUtils; +import alluxio.client.WriteType; import alluxio.client.file.FileInStream; import alluxio.client.file.FileSystem; -import alluxio.exception.ExceptionMessage; +import alluxio.client.file.URIStatus; import alluxio.exception.AlluxioException; +import alluxio.exception.ExceptionMessage; import alluxio.master.LocalAlluxioCluster; import alluxio.shell.command.CommandUtils; import alluxio.util.FormatUtils; import alluxio.util.io.BufferUtils; import alluxio.util.io.PathUtils; +import org.apache.zeppelin.completer.CompletionType; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; public class AlluxioInterpreterTest { private AlluxioInterpreter alluxioInterpreter; @@ -80,21 +82,25 @@ public final void before() throws Exception { @Test public void testCompletion() { List expectedResultOne = Arrays.asList( - new InterpreterCompletion("cat", "cat", CompletionType.command.name()), - new InterpreterCompletion("chgrp", "chgrp", CompletionType.command.name()), - new InterpreterCompletion("chmod", "chmod", CompletionType.command.name()), - new InterpreterCompletion("chown", "chown", CompletionType.command.name()), - new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()), - new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()), - new InterpreterCompletion("count", "count", CompletionType.command.name()), - new InterpreterCompletion("createLineage", "createLineage", CompletionType.command.name())); + new InterpreterCompletion("cat", "cat", CompletionType.command.name()), + new InterpreterCompletion("chgrp", "chgrp", CompletionType.command.name()), + new InterpreterCompletion("chmod", "chmod", CompletionType.command.name()), + new InterpreterCompletion("chown", "chown", CompletionType.command.name()), + new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()), + new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()), + new InterpreterCompletion("count", "count", CompletionType.command.name()), + new InterpreterCompletion("createLineage", "createLineage", CompletionType.command.name())); List expectedResultTwo = Arrays.asList( - new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()), - new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()), - new InterpreterCompletion("count", "count", CompletionType.command.name())); + new InterpreterCompletion("copyFromLocal", "copyFromLocal", + CompletionType.command.name()), + new InterpreterCompletion("copyToLocal", "copyToLocal", + CompletionType.command.name()), + new InterpreterCompletion("count", "count", CompletionType.command.name())); List expectedResultThree = Arrays.asList( - new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()), - new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name())); + new InterpreterCompletion("copyFromLocal", "copyFromLocal", + CompletionType.command.name()), + new InterpreterCompletion("copyToLocal", "copyToLocal", + CompletionType.command.name())); List expectedResultNone = new ArrayList<>(); List resultOne = alluxioInterpreter.completion("c", 0, null); @@ -143,7 +149,8 @@ public void catTest() throws IOException { Assert.assertEquals(Code.SUCCESS, output.code()); Assert.assertArrayEquals(expected, - output.message().get(0).getData().substring(0, output.message().get(0).getData().length() - 1).getBytes()); + output.message().get(0).getData().substring(0, + output.message().get(0).getData().length() - 1).getBytes()); } @Test @@ -188,8 +195,10 @@ public void loadDirTest() throws IOException, AlluxioException { FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WriteType.CACHE_THROUGH, 10, 10); FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileB", WriteType.MUST_CACHE, 10, 10); - int memPercentageA = fs.getStatus(new AlluxioURI("/testRoot/testFileA")).getInMemoryPercentage(); - int memPercentageB = fs.getStatus(new AlluxioURI("/testRoot/testFileB")).getInMemoryPercentage(); + int memPercentageA = fs.getStatus( + new AlluxioURI("/testRoot/testFileA")).getInMemoryPercentage(); + int memPercentageB = fs.getStatus( + new AlluxioURI("/testRoot/testFileB")).getInMemoryPercentage(); Assert.assertFalse(memPercentageA == 0); Assert.assertTrue(memPercentageB == 100); @@ -359,7 +368,8 @@ public void lsTest() throws IOException, AlluxioException { String expected = ""; String format = "%-10s%-25s%-15s%-5s\n"; expected += String.format(format, FormatUtils.getSizeFromBytes(10), - CommandUtils.convertMsToDate(files[0].getCreationTimeMs()), "In Memory", "/testRoot/testFileA"); + CommandUtils.convertMsToDate(files[0].getCreationTimeMs()), "In Memory", + "/testRoot/testFileA"); expected += String.format(format, FormatUtils.getSizeFromBytes(0), CommandUtils.convertMsToDate(files[1].getCreationTimeMs()), "", "/testRoot/testDir"); expected += String.format(format, FormatUtils.getSizeFromBytes(30), @@ -397,7 +407,8 @@ public void lsRecursiveTest() throws IOException, AlluxioException { "/testRoot/testFileA"); expected += String.format(format, FormatUtils.getSizeFromBytes(0), - CommandUtils.convertMsToDate(files[1].getCreationTimeMs()), "", "/testRoot/testDir"); + CommandUtils.convertMsToDate(files[1].getCreationTimeMs()), "", + "/testRoot/testDir"); expected += String.format(format, FormatUtils.getSizeFromBytes(20), CommandUtils.convertMsToDate(files[2].getCreationTimeMs()), "In Memory", @@ -478,4 +489,4 @@ private void fileReadTest(String fileName, int size) throws IOException { fis.close(); Assert.assertTrue(BufferUtils.equalIncreasingByteArray(size, read)); } -} \ No newline at end of file +} From f307392b043ff3f21b612063c652ec921f3d423c Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Sun, 4 Feb 2018 21:10:24 +0100 Subject: [PATCH 015/386] ZEPPELIN-3147. Added Checkstyle to the angular module ### What is this PR for? Added Checkstyle to the **angular** module. Did not have to fix any errors or warnings. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3147 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2766 from HorizonNet/ZEPPELIN-3147 and squashes the following commits: b766924 [Jan Hentschel] ZEPPELIN-3147. Added Checkstyle to the angular module --- angular/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/angular/pom.xml b/angular/pom.xml index f9d74481a1c..47ffbf35976 100644 --- a/angular/pom.xml +++ b/angular/pom.xml @@ -72,6 +72,13 @@ maven-resources-plugin + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + From b3ba458bd22f4320162cbb4baf3bab48a6844b30 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 7 Feb 2018 18:25:46 +0800 Subject: [PATCH 016/386] ZEPPELIN-3208. Use interpreter setting's name as its id ### What is this PR for? Interpreter setting's name is unique, so it is not necessary to use id to identify it. For now we use a random string to represent such id which is not easy to read especially when you read log for diagnose. But for backward compatibility, I will keep id and set it as the same as interpreter setting's name ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3208 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2778 from zjffdu/ZEPPELIN-3208 and squashes the following commits: 56a33c2 [Jeff Zhang] ZEPPELIN-3208. Use interpreter setting's name as its id --- .../org/apache/zeppelin/interpreter/InterpreterSetting.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 397ae108417..bb447375860 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -268,6 +268,7 @@ public InterpreterSetting() { void postProcessing() { this.status = Status.READY; + this.id = this.name; if (this.lifecycleManager == null) { this.lifecycleManager = new NullLifecycleManager(conf); } @@ -287,7 +288,7 @@ void postProcessing() { */ public InterpreterSetting(InterpreterSetting o) { this(); - this.id = generateId(); + this.id = o.name; this.name = o.name; this.group = o.group; this.properties = convertInterpreterProperties( From ad6e691812957e240d37918bbeae4a3c537fcccf Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 23 Aug 2017 20:34:15 +0800 Subject: [PATCH 017/386] [ZEPPELIN-2909]. Support shared SparkContext across language in livy interpreter ### What is this PR for? LIVY-194 implement the shared SparkContext across languages, this ticket is trying to integrate this feature. ### What type of PR is it? [ Feature ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2909 ### How should this be tested? Tested is added ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2587 from zjffdu/ZEPPELIN-2909 and squashes the following commits: d6e38e6 [Jeff Zhang] [ZEPPELIN-2909]. Support shared SparkContext across language in livy interpreter --- .travis.yml | 8 +- docs/interpreter/livy.md | 4 + livy/pom.xml | 148 +++++++++++++++++- .../zeppelin/livy/BaseLivyInterpreter.java | 88 +++++++++-- .../zeppelin/livy/LivySharedInterpreter.java | 108 +++++++++++++ .../zeppelin/livy/LivySparkInterpreter.java | 1 + .../livy/LivySparkSQLInterpreter.java | 36 ++++- .../org/apache/zeppelin/livy/LivyVersion.java | 5 + .../main/resources/interpreter-setting.json | 16 ++ .../zeppelin/livy/LivyInterpreterIT.java | 133 ++++++++++++++-- 10 files changed, 508 insertions(+), 39 deletions(-) create mode 100644 livy/src/main/java/org/apache/zeppelin/livy/LivySharedInterpreter.java diff --git a/.travis.yml b/.travis.yml index ce935b2c70e..24da368a3c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -107,17 +107,17 @@ matrix: dist: trusty env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test python/pyspark with python 2, livy 0.2 + # Test python/pyspark with python 2, livy 0.5 - sudo: required dist: trusty jdk: "openjdk7" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Plivy-0.2 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" - # Test python/pyspark with python 3, livy 0.3 + # Test python/pyspark with python 3, livy 0.5 - sudo: required dist: trusty jdk: "openjdk7" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" before_install: # check files included in commit range, clear bower_components if a bower.json file has changed. diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index d53672a94b2..e4784d4513d 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -216,6 +216,10 @@ select * from products where ${product_id=1} And creating dynamic formst programmatically is not feasible in livy interpreter, because ZeppelinContext is not available in livy interpreter. +## Shared SparkContext +Starting from livy 0.5 which is supported by Zeppelin 0.8.0, SparkContext is shared between scala, python, r and sql. +That means you can query the table via `%livy.sql` when this table is registered in `%livy.spark`, `%livy.pyspark`, `$livy.sparkr`. + ## FAQ Livy debugging: If you see any of these in error console diff --git a/livy/pom.xml b/livy/pom.xml index 1c9d8fb8efb..eddeb83301d 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -42,8 +42,9 @@ 1.0.1.RELEASE - 0.4.0-incubating + 0.5.0-incubating 2.1.0 + 2.6.0 2.16 1.8 @@ -105,6 +106,30 @@ org.apache.spark spark-yarn_${scala.binary.version} + + org.apache.hadoop + hadoop-auth + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-hdfs + + + org.apache.hadoop + hadoop-yarn-client + + + org.apache.hadoop + hadoop-client + + + org.apache.hadoop + hadoop-yarn-server-tests + @@ -188,6 +213,127 @@ test + + org.apache.hadoop + hadoop-auth + ${hadoop.version} + test + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + test + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-common + tests + ${hadoop.version} + test + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + test + + + io.netty + netty + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-hdfs + tests + ${hadoop.version} + test + + + io.netty + netty + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + test + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-yarn-client + ${hadoop.version} + test + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop.version} + test + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-yarn-server-tests + tests + ${hadoop.version} + test + + + com.google.guava + guava + + + diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index f3b75792e1e..724a4b36c7c 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -57,6 +57,8 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.InterpreterUtils; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,6 +97,9 @@ public abstract class BaseLivyInterpreter extends Interpreter { private RestTemplate restTemplate; private Map customHeaders = new HashMap<>(); + // delegate to sharedInterpreter when it is available + protected LivySharedInterpreter sharedInterpreter; + Set paragraphsToCancel = Collections.newSetFromMap( new ConcurrentHashMap()); private ConcurrentHashMap paragraphId2StmtProgressMap = @@ -144,7 +149,13 @@ Map getCustomHeaders() { @Override public void open() throws InterpreterException { try { - initLivySession(); + this.livyVersion = getLivyVersion(); + if (this.livyVersion.isSharedSupported()) { + sharedInterpreter = getLivySharedInterpreter(); + } + if (sharedInterpreter == null || !sharedInterpreter.isSupported()) { + initLivySession(); + } } catch (LivyException e) { String msg = "Fail to create session, please check livy interpreter log and " + "livy server log"; @@ -152,8 +163,32 @@ public void open() throws InterpreterException { } } + protected LivySharedInterpreter getLivySharedInterpreter() throws InterpreterException { + LazyOpenInterpreter lazy = null; + LivySharedInterpreter sharedInterpreter = null; + Interpreter p = getInterpreterInTheSameSessionByClassName( + LivySharedInterpreter.class.getName()); + + while (p instanceof WrappedInterpreter) { + if (p instanceof LazyOpenInterpreter) { + lazy = (LazyOpenInterpreter) p; + } + p = ((WrappedInterpreter) p).getInnerInterpreter(); + } + sharedInterpreter = (LivySharedInterpreter) p; + + if (lazy != null) { + lazy.open(); + } + return sharedInterpreter; + } + @Override public void close() { + if (sharedInterpreter != null && sharedInterpreter.isSupported()) { + sharedInterpreter.close(); + return; + } if (sessionInfo != null) { closeSession(sessionInfo.id); // reset sessionInfo to null so that we won't close it twice. @@ -181,14 +216,6 @@ protected void initLivySession() throws LivyException { } else { LOGGER.info("Create livy session successfully with sessionId: {}", this.sessionInfo.id); } - // check livy version - try { - this.livyVersion = getLivyVersion(); - LOGGER.info("Use livy " + livyVersion); - } catch (APINotFoundException e) { - this.livyVersion = new LivyVersion("0.2.0"); - LOGGER.info("Use livy 0.2.0"); - } } protected abstract String extractAppId() throws LivyException; @@ -196,17 +223,30 @@ protected void initLivySession() throws LivyException { protected abstract String extractWebUIAddress() throws LivyException; public SessionInfo getSessionInfo() { + if (sharedInterpreter != null && sharedInterpreter.isSupported()) { + return sharedInterpreter.getSessionInfo(); + } return sessionInfo; } + public String getCodeType() { + if (getSessionKind().equalsIgnoreCase("pyspark3")) { + return "pyspark"; + } + return getSessionKind(); + } + @Override public InterpreterResult interpret(String st, InterpreterContext context) { + if (sharedInterpreter != null && sharedInterpreter.isSupported()) { + return sharedInterpreter.interpret(st, getCodeType(), context); + } if (StringUtils.isEmpty(st)) { return new InterpreterResult(InterpreterResult.Code.SUCCESS, ""); } try { - return interpret(st, context.getParagraphId(), this.displayAppInfo, true); + return interpret(st, null, context.getParagraphId(), this.displayAppInfo, true); } catch (LivyException e) { LOGGER.error("Fail to interpret:" + st, e); return new InterpreterResult(InterpreterResult.Code.ERROR, @@ -245,6 +285,10 @@ private List callCompletion(CompletionRequest req) throws @Override public void cancel(InterpreterContext context) { + if (sharedInterpreter != null && sharedInterpreter.isSupported()) { + sharedInterpreter.cancel(context); + return; + } paragraphsToCancel.add(context.getParagraphId()); LOGGER.info("Added paragraph " + context.getParagraphId() + " for cancellation."); } @@ -256,6 +300,10 @@ public FormType getFormType() { @Override public int getProgress(InterpreterContext context) { + if (sharedInterpreter != null && sharedInterpreter.isSupported()) { + return sharedInterpreter.getProgress(context); + } + if (livyVersion.isGetProgressSupported()) { String paraId = context.getParagraphId(); Integer progress = paragraphId2StmtProgressMap.get(paraId); @@ -312,11 +360,20 @@ public InterpreterResult interpret(String code, String paragraphId, boolean displayAppInfo, boolean appendSessionExpired) throws LivyException { + return interpret(code, sharedInterpreter.isSupported() ? getSessionKind() : null, + paragraphId, displayAppInfo, appendSessionExpired); + } + + public InterpreterResult interpret(String code, + String codeType, + String paragraphId, + boolean displayAppInfo, + boolean appendSessionExpired) throws LivyException { StatementInfo stmtInfo = null; boolean sessionExpired = false; try { try { - stmtInfo = executeStatement(new ExecuteRequest(code)); + stmtInfo = executeStatement(new ExecuteRequest(code, codeType)); } catch (SessionNotFoundException e) { LOGGER.warn("Livy session {} is expired, new session will be created.", sessionInfo.id); sessionExpired = true; @@ -328,7 +385,7 @@ public InterpreterResult interpret(String code, initLivySession(); } } - stmtInfo = executeStatement(new ExecuteRequest(code)); + stmtInfo = executeStatement(new ExecuteRequest(code, codeType)); } // pull the statement status @@ -731,11 +788,12 @@ public static SessionInfo fromJson(String json) { } } - private static class ExecuteRequest { + static class ExecuteRequest { public final String code; - - public ExecuteRequest(String code) { + public final String kind; + public ExecuteRequest(String code, String kind) { this.code = code; + this.kind = kind; } public String toJson() { diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySharedInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySharedInterpreter.java new file mode 100644 index 00000000000..77e288bfa65 --- /dev/null +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySharedInterpreter.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.livy; + +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +/** + * Livy Interpreter for shared kind which share SparkContext across spark/pyspark/r + */ +public class LivySharedInterpreter extends BaseLivyInterpreter { + + private static final Logger LOGGER = LoggerFactory.getLogger(LivySharedInterpreter.class); + + private boolean isSupported = false; + + public LivySharedInterpreter(Properties property) { + super(property); + } + + @Override + public void open() throws InterpreterException { + try { + // check livy version + try { + this.livyVersion = getLivyVersion(); + LOGGER.info("Use livy " + livyVersion); + } catch (APINotFoundException e) { + // assume it is livy 0.2.0 when livy doesn't support rest api of fetching version. + this.livyVersion = new LivyVersion("0.2.0"); + LOGGER.info("Use livy 0.2.0"); + } + + if (livyVersion.isSharedSupported()) { + LOGGER.info("LivySharedInterpreter is supported."); + isSupported = true; + initLivySession(); + } else { + LOGGER.info("LivySharedInterpreter is not supported."); + isSupported = false; + } + } catch (LivyException e) { + String msg = "Fail to create session, please check livy interpreter log and " + + "livy server log"; + throw new InterpreterException(msg, e); + } + } + + public boolean isSupported() { + return isSupported; + } + + public InterpreterResult interpret(String st, String codeType, InterpreterContext context) { + if (StringUtils.isEmpty(st)) { + return new InterpreterResult(InterpreterResult.Code.SUCCESS, ""); + } + + try { + return interpret(st, codeType, context.getParagraphId(), this.displayAppInfo, true); + } catch (LivyException e) { + LOGGER.error("Fail to interpret:" + st, e); + return new InterpreterResult(InterpreterResult.Code.ERROR, + InterpreterUtils.getMostRelevantMessage(e)); + } + } + + @Override + public String getSessionKind() { + return "shared"; + } + + @Override + protected String extractAppId() throws LivyException { + return null; + } + + @Override + protected String extractWebUIAddress() throws LivyException { + return null; + } + + public static void main(String[] args) { + ExecuteRequest request = new ExecuteRequest("1+1", null); + System.out.println(request.toJson()); + } +} diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java index 606ef64a8a4..066d0da8cc9 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java @@ -44,6 +44,7 @@ protected String extractAppId() throws LivyException { protected String extractWebUIAddress() throws LivyException { interpret( "val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get", + null, null, false, false); return extractStatementResult( interpret( diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index 7b2d7d66684..2faa350bfb1 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -19,11 +19,14 @@ import org.apache.commons.lang.StringUtils; import static org.apache.commons.lang.StringEscapeUtils.escapeJavaScript; +import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.user.AuthenticationInfo; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Properties; @@ -39,6 +42,7 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter { "zeppelin.livy.spark.sql.maxResult"; private LivySparkInterpreter sparkInterpreter; + private String codeType = null; private boolean isSpark2 = false; private int maxResult = 1000; @@ -64,7 +68,21 @@ public void open() throws InterpreterException { // As we don't know whether livyserver use spark2 or spark1, so we will detect SparkSession // to judge whether it is using spark2. try { - InterpreterResult result = sparkInterpreter.interpret("spark", null, false, false); + InterpreterContext context = new InterpreterContext( + "noteId", + "paragraphId", + "replName", + "paragraphTitle", + "paragraphText", + new AuthenticationInfo(), + new HashMap(), + new GUI(), + new GUI(), + null, + null, + null, + new InterpreterOutput(null)); + InterpreterResult result = sparkInterpreter.interpret("spark", context); if (result.code() == InterpreterResult.Code.SUCCESS && result.message().get(0).getData().contains("org.apache.spark.sql.SparkSession")) { LOGGER.info("SparkSession is detected so we are using spark 2.x for session {}", @@ -72,7 +90,7 @@ public void open() throws InterpreterException { isSpark2 = true; } else { // spark 1.x - result = sparkInterpreter.interpret("sqlContext", null, false, false); + result = sparkInterpreter.interpret("sqlContext", context); if (result.code() == InterpreterResult.Code.SUCCESS) { LOGGER.info("sqlContext is detected."); } else if (result.code() == InterpreterResult.Code.ERROR) { @@ -81,7 +99,7 @@ public void open() throws InterpreterException { LOGGER.info("sqlContext is not detected, try to create SQLContext by ourselves"); result = sparkInterpreter.interpret( "val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n" - + "import sqlContext.implicits._", null, false, false); + + "import sqlContext.implicits._", context); if (result.code() == InterpreterResult.Code.ERROR) { throw new LivyException("Fail to create SQLContext," + result.message().get(0).getData()); @@ -128,9 +146,7 @@ public InterpreterResult interpret(String line, InterpreterContext context) { sqlQuery = "sqlContext.sql(\"\"\"" + line + "\"\"\").show(" + maxResult + ", " + truncate + ")"; } - InterpreterResult result = sparkInterpreter.interpret(sqlQuery, context.getParagraphId(), - this.displayAppInfo, true); - + InterpreterResult result = sparkInterpreter.interpret(sqlQuery, context); if (result.code() == InterpreterResult.Code.SUCCESS) { InterpreterResult result2 = new InterpreterResult(InterpreterResult.Code.SUCCESS); for (InterpreterResultMessage message : result.message()) { @@ -248,12 +264,16 @@ public Scheduler getScheduler() { @Override public void cancel(InterpreterContext context) { - sparkInterpreter.cancel(context); + if (this.sparkInterpreter != null) { + sparkInterpreter.cancel(context); + } } @Override public void close() { - this.sparkInterpreter.close(); + if (this.sparkInterpreter != null) { + this.sparkInterpreter.close(); + } } @Override diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyVersion.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyVersion.java index 7cfecfb197c..81bb8d4956f 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivyVersion.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyVersion.java @@ -29,6 +29,7 @@ public class LivyVersion { protected static final LivyVersion LIVY_0_2_0 = LivyVersion.fromVersionString("0.2.0"); protected static final LivyVersion LIVY_0_3_0 = LivyVersion.fromVersionString("0.3.0"); protected static final LivyVersion LIVY_0_4_0 = LivyVersion.fromVersionString("0.4.0"); + protected static final LivyVersion LIVY_0_5_0 = LivyVersion.fromVersionString("0.5.0"); private int version; private String versionString; @@ -79,6 +80,10 @@ public boolean isGetProgressSupported() { return this.newerThanEquals(LIVY_0_4_0); } + public boolean isSharedSupported() { + return this.newerThanEquals(LIVY_0_5_0); + } + public boolean equals(Object versionToCompare) { return version == ((LivyVersion) versionToCompare).version; } diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json index 2d724875659..cecacac2e48 100644 --- a/livy/src/main/resources/interpreter-setting.json +++ b/livy/src/main/resources/interpreter-setting.json @@ -227,5 +227,21 @@ "editOnDblClick": false, "completionKey": "TAB" } + }, + { + "group": "livy", + "name": "shared", + "className": "org.apache.zeppelin.livy.LivySharedInterpreter", + "properties": { + }, + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "scoped", + "isExistingProcess": false, + "setPermission": false, + "users": [] + } } ] diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index ef3eabe66ec..3dfeb363f8f 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -34,6 +34,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; public class LivyInterpreterIT { @@ -76,7 +77,7 @@ public static boolean checkPreCondition() { } -// @Test + @Test public void testSparkInterpreterRDD() throws InterpreterException { if (!checkPreCondition()) { return; @@ -197,7 +198,7 @@ public void run() { } -// @Test + @Test public void testSparkInterpreterDataFrame() throws InterpreterException { if (!checkPreCondition()) { return; @@ -285,7 +286,7 @@ public void testSparkInterpreterDataFrame() throws InterpreterException { } } -// @Test + @Test public void testSparkSQLInterpreter() throws InterpreterException { if (!checkPreCondition()) { return; @@ -320,7 +321,7 @@ public void testSparkSQLInterpreter() throws InterpreterException { } -// @Test + @Test public void testSparkSQLCancellation() throws InterpreterException { if (!checkPreCondition()) { return; @@ -401,7 +402,7 @@ public void run() { } } -// @Test + @Test public void testStringWithTruncation() throws InterpreterException { if (!checkPreCondition()) { return; @@ -462,7 +463,7 @@ public void testStringWithTruncation() throws InterpreterException { } -// @Test + @Test public void testStringWithoutTruncation() throws InterpreterException { if (!checkPreCondition()) { return; @@ -534,6 +535,7 @@ public void testPySparkInterpreter() throws LivyException, InterpreterException } final LivyPySparkInterpreter pysparkInterpreter = new LivyPySparkInterpreter(properties); + pysparkInterpreter.setInterpreterGroup(mock(InterpreterGroup.class)); AuthenticationInfo authInfo = new AuthenticationInfo("user1"); MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener(); InterpreterOutput output = new InterpreterOutput(outputListener); @@ -647,7 +649,7 @@ public void run() { } } -// @Test + @Test public void testSparkInterpreterWithDisplayAppInfo() throws InterpreterException { if (!checkPreCondition()) { return; @@ -686,13 +688,15 @@ public void testSparkInterpreterWithDisplayAppInfo() throws InterpreterException } } -// @Test + @Test public void testSparkRInterpreter() throws LivyException, InterpreterException { if (!checkPreCondition()) { return; } final LivySparkRInterpreter sparkRInterpreter = new LivySparkRInterpreter(properties); + sparkRInterpreter.setInterpreterGroup(mock(InterpreterGroup.class)); + try { sparkRInterpreter.getLivyVersion(); } catch (APINotFoundException e) { @@ -749,8 +753,7 @@ public void run() { // error result = sparkRInterpreter.interpret("cat(a)", context); - //TODO @zjffdu, it should be ERROR, it is due to bug of LIVY-313 - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(InterpreterResult.Code.ERROR, result.code()); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertTrue(result.message().get(0).getData().contains("object 'a' not found")); } finally { @@ -758,7 +761,7 @@ public void run() { } } -// @Test + @Test public void testLivyTutorialNote() throws IOException, InterpreterException { if (!checkPreCondition()) { return; @@ -796,6 +799,114 @@ public void testLivyTutorialNote() throws IOException, InterpreterException { } } + @Test + public void testSharedInterpreter() throws InterpreterException { + if (!checkPreCondition()) { + return; + } + InterpreterGroup interpreterGroup = new InterpreterGroup("group_1"); + interpreterGroup.put("session_1", new ArrayList()); + LazyOpenInterpreter sparkInterpreter = new LazyOpenInterpreter( + new LivySparkInterpreter(properties)); + sparkInterpreter.setInterpreterGroup(interpreterGroup); + interpreterGroup.get("session_1").add(sparkInterpreter); + + LazyOpenInterpreter sqlInterpreter = new LazyOpenInterpreter( + new LivySparkSQLInterpreter(properties)); + interpreterGroup.get("session_1").add(sqlInterpreter); + sqlInterpreter.setInterpreterGroup(interpreterGroup); + + LazyOpenInterpreter pysparkInterpreter = new LazyOpenInterpreter( + new LivyPySparkInterpreter(properties)); + interpreterGroup.get("session_1").add(pysparkInterpreter); + pysparkInterpreter.setInterpreterGroup(interpreterGroup); + + LazyOpenInterpreter sparkRInterpreter = new LazyOpenInterpreter( + new LivySparkRInterpreter(properties)); + interpreterGroup.get("session_1").add(sparkRInterpreter); + sparkRInterpreter.setInterpreterGroup(interpreterGroup); + + LazyOpenInterpreter sharedInterpreter = new LazyOpenInterpreter( + new LivySharedInterpreter(properties)); + interpreterGroup.get("session_1").add(sharedInterpreter); + sharedInterpreter.setInterpreterGroup(interpreterGroup); + + sparkInterpreter.open(); + sqlInterpreter.open(); + pysparkInterpreter.open(); + sparkRInterpreter.open(); + + try { + AuthenticationInfo authInfo = new AuthenticationInfo("user1"); + MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener(); + InterpreterOutput output = new InterpreterOutput(outputListener); + InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.sql", + "title", "text", authInfo, null, null, null, null, null, null, output); + // detect spark version + InterpreterResult result = sparkInterpreter.interpret("sc.version", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + boolean isSpark2 = isSpark2((BaseLivyInterpreter)sparkInterpreter.getInnerInterpreter(), context); + + if (!isSpark2) { + result = sparkInterpreter.interpret( + "val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n" + + "df.collect()", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData() + .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])")); + sparkInterpreter.interpret("df.registerTempTable(\"df\")", context); + + // access table from pyspark + result = pysparkInterpreter.interpret("sqlContext.sql(\"select * from df\").show()", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData() + .contains("+-----+-----+\n" + + "|col_1|col_2|\n" + + "+-----+-----+\n" + + "|hello| 20|\n" + + "+-----+-----+")); + + // access table from sparkr + result = sparkRInterpreter.interpret("head(sql(sqlContext, \"select * from df\"))", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData().contains("col_1 col_2\n1 hello 20")); + } else { + result = sparkInterpreter.interpret( + "val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n" + + "df.collect()", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData() + .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])")); + sparkInterpreter.interpret("df.registerTempTable(\"df\")", context); + + // access table from pyspark + result = pysparkInterpreter.interpret("spark.sql(\"select * from df\").show()", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData() + .contains("+-----+-----+\n" + + "|col_1|col_2|\n" + + "+-----+-----+\n" + + "|hello| 20|\n" + + "+-----+-----+")); + + // access table from sparkr + result = sparkRInterpreter.interpret("head(sql(\"select * from df\"))", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData().contains("col_1 col_2\n1 hello 20")); + } + } finally { + sparkInterpreter.close(); + sqlInterpreter.close(); + } + } private boolean isSpark2(BaseLivyInterpreter interpreter, InterpreterContext context) { InterpreterResult result = null; From f8d3488cbbb3a2d20c5736a8587ef0c41ae1eebc Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Thu, 8 Feb 2018 23:00:55 +0800 Subject: [PATCH 018/386] [HOTFIX] Fix zeppelin-display build ### What is this PR for? This is for fix the build of zeppelin-display module ### What type of PR is it? [ Hot Fix ] ### Todos * [ ] - Task ### How should this be tested? * Travis pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? NO Author: Jeff Zhang Closes #2781 from zjffdu/HOTFIX_BUILD and squashes the following commits: dd2c36d [Jeff Zhang] [HOTFIX] Fix zeppelin-display build --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 2c230cb8e46..4cda650b6dc 100644 --- a/pom.xml +++ b/pom.xml @@ -742,6 +742,9 @@ scala-2.10 + + true + 2.10.5 2.10 @@ -750,9 +753,6 @@ scala-2.11 - - true - 2.11.8 2.11 From 73d15704bc73f02884f5d6f3348c1c1ceb952a5a Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Thu, 8 Feb 2018 12:44:11 +0530 Subject: [PATCH 019/386] [ZEPPELIN-3213] Support for KNOXSSO logout url as API KNOXSSO logout URL can be an API or it can be a redirect URL, Zeppelin should support both. [Improvement] * [ZEPPELIN-3213](https://issues.apache.org/jira/browse/ZEPPELIN-3213) * Does the licenses files need update? N/A * Is there breaking changes for older versions? N/A * Does this needs documentation? N/A Author: Prabhjyot Singh Closes #2779 from prabhjyotsingh/ZEPPELIN-3213 and squashes the following commits: 6c89f2427 [Prabhjyot Singh] [ZEPPELIN-3213] Support for KNOXSSO logout url as API Change-Id: I44e3bb13dc3de9330751236c0b3703a8177200e7 --- conf/shiro.ini.template | 1 + .../org/apache/zeppelin/realm/jwt/KnoxJwtRealm.java | 9 +++++++++ .../java/org/apache/zeppelin/rest/LoginRestApi.java | 1 + .../src/components/navbar/navbar.controller.js | 10 +++++++++- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/conf/shiro.ini.template b/conf/shiro.ini.template index 81b31a2b9d8..9397025ca18 100644 --- a/conf/shiro.ini.template +++ b/conf/shiro.ini.template @@ -61,6 +61,7 @@ user3 = password4, role2 #knoxJwtRealm.providerUrl = https://domain.example.com/ #knoxJwtRealm.login = gateway/knoxsso/knoxauth/login.html #knoxJwtRealm.logout = gateway/knoxssout/api/v1/webssout +#knoxJwtRealm.logoutAPI = true #knoxJwtRealm.redirectParam = originalUrl #knoxJwtRealm.cookieName = hadoop-jwt #knoxJwtRealm.publicKeyPath = /etc/zeppelin/conf/knox-sso.pem diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/KnoxJwtRealm.java b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/KnoxJwtRealm.java index c3e9b77cc88..a903e6ec4bb 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/KnoxJwtRealm.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/KnoxJwtRealm.java @@ -60,6 +60,7 @@ public class KnoxJwtRealm extends AuthorizingRealm { private String publicKeyPath; private String login; private String logout; + private Boolean logoutAPI; private String principalMapping; private String groupPrincipalMapping; @@ -270,6 +271,14 @@ public void setLogout(String logout) { this.logout = logout; } + public Boolean getLogoutAPI() { + return logoutAPI; + } + + public void setLogoutAPI(Boolean logoutAPI) { + this.logoutAPI = logoutAPI; + } + public String getPrincipalMapping() { return principalMapping; } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/LoginRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/LoginRestApi.java index 3a084cf9e4b..b590bdb006d 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/LoginRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/LoginRestApi.java @@ -211,6 +211,7 @@ public Response logout() { KnoxJwtRealm knoxJwtRealm = getJTWRealm(); Map data = new HashMap<>(); data.put("redirectURL", constructKnoxUrl(knoxJwtRealm, knoxJwtRealm.getLogout())); + data.put("isLogoutAPI", knoxJwtRealm.getLogoutAPI().toString()); response = new JsonResponse(Status.UNAUTHORIZED, "", data); } else { response = new JsonResponse(Status.UNAUTHORIZED, "", ""); diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 7d7fada1d45..139328e11d7 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -93,7 +93,15 @@ function NavCtrl ($scope, $rootScope, $http, $routeParams, $location, if (response.data) { let res = angular.fromJson(response.data).body if (res['redirectURL']) { - window.location.href = res['redirectURL'] + window.location.href + if (res['isLogoutAPI'] === 'true') { + $http.get(res['redirectURL']).then(function () { + }, function () { + window.location = baseUrlSrv.getBase() + }) + } else { + window.location.href = res['redirectURL'] + window.location.href + } + return undefined } } From 29b9b10f392f8a36251d25dc4c8d006a07f5d4cb Mon Sep 17 00:00:00 2001 From: Savalek Date: Wed, 7 Feb 2018 20:16:27 +0300 Subject: [PATCH 020/386] [ZEPPELIN-3212] delete extra ">" in notebook-actionBar.html ### What is this PR for? Delete extra ">" in notebook-actionBar.html after `` ### What type of PR is it? [Improvement] ### What is the Jira issue? [ZEPPELIN-3212](https://issues.apache.org/jira/browse/ZEPPELIN-3212) ### Screenshots (if appropriate) ![1](https://user-images.githubusercontent.com/30798933/35930861-1ba8369e-0c44-11e8-8ebf-9328cb21a6e5.PNG) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Savalek Closes #2774 from Savalek/ZEPPELIN-3212 and squashes the following commits: 12352b772 [Savalek] [ZEPPELIN-3212] delete extra ">" in notebook-actionBar.html --- zeppelin-web/src/app/notebook/notebook-actionBar.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-web/src/app/notebook/notebook-actionBar.html b/zeppelin-web/src/app/notebook/notebook-actionBar.html index 573be001ebb..9b50e819f79 100644 --- a/zeppelin-web/src/app/notebook/notebook-actionBar.html +++ b/zeppelin-web/src/app/notebook/notebook-actionBar.html @@ -24,7 +24,7 @@

tooltip-placement="bottom" uib-tooltip={{noteName(note)}} ng-click="input.showEditor = !revisionView; input.value = note.name" - ng-show="!input.showEditor">{{noteName(note)}}>

+ ng-show="!input.showEditor">{{noteName(note)}}

From d9faef1085e4ade496ff7f3d7f8472a28678f8e7 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 14 Nov 2017 15:29:58 +0800 Subject: [PATCH 021/386] ZEPPELIN-3108. Support Spark 2.3 ### What is this PR for? Spark 2.3 remove `JobProgressListener` which cause zeppelin unable to run spark 2.3. This PR try to make Zeppelin support spark 2.3 via using `sc.statusTracker`, see `JobProgressUtil.scala` ### What type of PR is it? [Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3108 ### How should this be tested? * Verified manually. ### Screenshots (if appropriate) ![screen shot 2018-01-30 at 9 45 01 pm](https://user-images.githubusercontent.com/164491/35569317-dce6f348-0606-11e8-9b18-74a847d64ac9.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2750 from zjffdu/ZEPPELIN-3108 and squashes the following commits: 43ae78a [Jeff Zhang] ZEPPELIN-3108. Support Spark 2.3 --- spark/interpreter/pom.xml | 10 +- .../zeppelin/spark/OldSparkInterpreter.java | 153 ++++++++++++------ spark/pom.xml | 19 ++- spark/spark-dependencies/pom.xml | 10 +- .../spark/BaseSparkScalaInterpreter.scala | 16 +- .../zeppelin/spark/JobProgressUtil.scala | 37 +++++ 6 files changed, 164 insertions(+), 81 deletions(-) create mode 100644 spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/JobProgressUtil.scala diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index 449646242dc..758f6970e8e 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -53,15 +53,7 @@ **/PySparkInterpreterMatplotlibTest.java **/*Test.* - - spark-${spark.version} - - http://d3kbcqa49mib13.cloudfront.net/${spark.archive}.tgz - - - http://d3kbcqa49mib13.cloudfront.net/spark-${spark.version}-bin-without-hadoop.tgz - - + diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java index 6a54c3b37b9..da332fe5ef0 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java @@ -35,6 +35,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.spark.JobProgressUtil; import org.apache.spark.SecurityManager; import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; @@ -44,10 +45,26 @@ import org.apache.spark.scheduler.ActiveJob; import org.apache.spark.scheduler.DAGScheduler; import org.apache.spark.scheduler.Pool; +import org.apache.spark.scheduler.SparkListenerApplicationEnd; +import org.apache.spark.scheduler.SparkListenerApplicationStart; +import org.apache.spark.scheduler.SparkListenerBlockManagerAdded; +import org.apache.spark.scheduler.SparkListenerBlockManagerRemoved; +import org.apache.spark.scheduler.SparkListenerBlockUpdated; +import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate; +import org.apache.spark.scheduler.SparkListenerExecutorAdded; +import org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate; +import org.apache.spark.scheduler.SparkListenerExecutorRemoved; +import org.apache.spark.scheduler.SparkListenerJobEnd; import org.apache.spark.scheduler.SparkListenerJobStart; +import org.apache.spark.scheduler.SparkListenerStageCompleted; +import org.apache.spark.scheduler.SparkListenerStageSubmitted; +import org.apache.spark.scheduler.SparkListenerTaskEnd; +import org.apache.spark.scheduler.SparkListenerTaskGettingResult; +import org.apache.spark.scheduler.SparkListenerTaskStart; +import org.apache.spark.scheduler.SparkListenerUnpersistRDD; import org.apache.spark.sql.SQLContext; import org.apache.spark.ui.SparkUI; -import org.apache.spark.ui.jobs.JobProgressListener; +import org.apache.spark.scheduler.SparkListener; import org.apache.zeppelin.interpreter.BaseZeppelinContext; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; @@ -113,7 +130,7 @@ public class OldSparkInterpreter extends AbstractSparkInterpreter { private static InterpreterHookRegistry hooks; private static SparkEnv env; private static Object sparkSession; // spark 2.x - private static JobProgressListener sparkListener; + private static SparkListener sparkListener; private static AbstractFile classOutputDir; private static Integer sharedInterpreterLock = new Integer(0); private static AtomicInteger numReferenceOfSparkContext = new AtomicInteger(0); @@ -173,11 +190,10 @@ public boolean isSparkContextInitialized() { } } - static JobProgressListener setupListeners(SparkContext context) { - JobProgressListener pl = new JobProgressListener(context.getConf()) { + static SparkListener setupListeners(SparkContext context) { + SparkListener pl = new SparkListener() { @Override public synchronized void onJobStart(SparkListenerJobStart jobStart) { - super.onJobStart(jobStart); int jobId = jobStart.jobId(); String jobGroupId = jobStart.properties().getProperty("spark.jobGroup.id"); String uiEnabled = jobStart.properties().getProperty("spark.ui.enabled"); @@ -207,6 +223,85 @@ private String getJobUrl(int jobId) { return jobUrl; } + @Override + public void onBlockUpdated(SparkListenerBlockUpdated blockUpdated) { + + } + + @Override + public void onExecutorRemoved(SparkListenerExecutorRemoved executorRemoved) { + + } + + @Override + public void onExecutorAdded(SparkListenerExecutorAdded executorAdded) { + + } + + @Override + public void onExecutorMetricsUpdate(SparkListenerExecutorMetricsUpdate executorMetricsUpdate) { + + } + + @Override + public void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) { + + } + + @Override + public void onApplicationStart(SparkListenerApplicationStart applicationStart) { + + } + + @Override + public void onUnpersistRDD(SparkListenerUnpersistRDD unpersistRDD) { + + } + + @Override + public void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) { + + } + + @Override + public void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) { + + } + + @Override + public void onEnvironmentUpdate(SparkListenerEnvironmentUpdate environmentUpdate) { + + } + + @Override + public void onJobEnd(SparkListenerJobEnd jobEnd) { + + } + + @Override + public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { + + } + + @Override + public void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) { + + } + + @Override + public void onTaskEnd(SparkListenerTaskEnd taskEnd) { + + } + + @Override + public void onTaskGettingResult(SparkListenerTaskGettingResult taskGettingResult) { + + } + + @Override + public void onTaskStart(SparkListenerTaskStart taskStart) { + + } }; try { Object listenerBus = context.getClass().getMethod("listenerBus").invoke(context); @@ -224,7 +319,7 @@ private String getJobUrl(int jobId) { continue; } - if (!parameterTypes[0].isAssignableFrom(JobProgressListener.class)) { + if (!parameterTypes[0].isAssignableFrom(SparkListener.class)) { continue; } @@ -1274,48 +1369,10 @@ public void cancel(InterpreterContext context) { @Override public int getProgress(InterpreterContext context) { String jobGroup = Utils.buildJobGroupId(context); - int completedTasks = 0; - int totalTasks = 0; - - DAGScheduler scheduler = sc.dagScheduler(); - if (scheduler == null) { - return 0; - } - HashSet jobs = scheduler.activeJobs(); - if (jobs == null || jobs.size() == 0) { - return 0; - } - Iterator it = jobs.iterator(); - while (it.hasNext()) { - ActiveJob job = it.next(); - String g = (String) job.properties().get("spark.jobGroup.id"); - if (jobGroup.equals(g)) { - int[] progressInfo = null; - try { - Object finalStage = job.getClass().getMethod("finalStage").invoke(job); - if (sparkVersion.getProgress1_0()) { - progressInfo = getProgressFromStage_1_0x(sparkListener, finalStage); - } else { - progressInfo = getProgressFromStage_1_1x(sparkListener, finalStage); - } - } catch (IllegalAccessException | IllegalArgumentException - | InvocationTargetException | NoSuchMethodException - | SecurityException e) { - logger.error("Can't get progress info", e); - return 0; - } - totalTasks += progressInfo[0]; - completedTasks += progressInfo[1]; - } - } - - if (totalTasks == 0) { - return 0; - } - return completedTasks * 100 / totalTasks; + return JobProgressUtil.progress(sc, jobGroup); } - private int[] getProgressFromStage_1_0x(JobProgressListener sparkListener, Object stage) + private int[] getProgressFromStage_1_0x(SparkListener sparkListener, Object stage) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); @@ -1345,7 +1402,7 @@ private int[] getProgressFromStage_1_0x(JobProgressListener sparkListener, Objec return new int[] {numTasks, completedTasks}; } - private int[] getProgressFromStage_1_1x(JobProgressListener sparkListener, Object stage) + private int[] getProgressFromStage_1_1x(SparkListener sparkListener, Object stage) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); @@ -1421,7 +1478,7 @@ public FormType getFormType() { return FormType.NATIVE; } - public JobProgressListener getJobProgressListener() { + public SparkListener getJobProgressListener() { return sparkListener; } diff --git a/spark/pom.xml b/spark/pom.xml index 06b7d9f74b1..c55e453d210 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -47,6 +47,14 @@ 2.2.0 0.10.4 + + spark-${spark.version} + + http://d3kbcqa49mib13.cloudfront.net/${spark.archive}.tgz + + + http://d3kbcqa49mib13.cloudfront.net/${spark.archive}-bin-without-hadoop.tgz + @@ -57,7 +65,6 @@ ${project.version} - org.apache.zeppelin zeppelin-display @@ -187,6 +194,16 @@ + + + spark-2.3 + + 2.3.0 + 2.5.0 + 0.10.6 + + + spark-2.2 diff --git a/spark/spark-dependencies/pom.xml b/spark/spark-dependencies/pom.xml index 7643dc9d8f4..58977b4db77 100644 --- a/spark/spark-dependencies/pom.xml +++ b/spark/spark-dependencies/pom.xml @@ -28,7 +28,7 @@ org.apache.zeppelin - zeppelin-spark-dependencies_2.10 + zeppelin-spark-dependencies jar 0.9.0-SNAPSHOT Zeppelin: Spark dependencies @@ -54,14 +54,6 @@ org.spark-project.akka 2.3.4-spark - spark-${spark.version} - - http://d3kbcqa49mib13.cloudfront.net/${spark.archive}.tgz - - - http://d3kbcqa49mib13.cloudfront.net/${spark.archive}-bin-without-hadoop.tgz - - 2.3 diff --git a/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala b/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala index 3ef4fe71e00..883beb02255 100644 --- a/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala +++ b/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala @@ -21,7 +21,7 @@ package org.apache.zeppelin.spark import java.io.File import org.apache.spark.sql.SQLContext -import org.apache.spark.{SparkConf, SparkContext} +import org.apache.spark.{JobProgressUtil, SparkConf, SparkContext} import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion import org.apache.zeppelin.interpreter.util.InterpreterOutputStream import org.apache.zeppelin.interpreter.{InterpreterContext, InterpreterResult} @@ -93,19 +93,7 @@ abstract class BaseSparkScalaInterpreter(val conf: SparkConf, } protected def getProgress(jobGroup: String, context: InterpreterContext): Int = { - val jobIds = sc.statusTracker.getJobIdsForGroup(jobGroup) - val jobs = jobIds.flatMap { id => sc.statusTracker.getJobInfo(id) } - val stages = jobs.flatMap { job => - job.stageIds().flatMap(sc.statusTracker.getStageInfo) - } - - val taskCount = stages.map(_.numTasks).sum - val completedTaskCount = stages.map(_.numCompletedTasks).sum - if (taskCount == 0) { - 0 - } else { - (100 * completedTaskCount.toDouble / taskCount).toInt - } + JobProgressUtil.progress(sc, jobGroup) } protected def bind(name: String, tpe: String, value: Object, modifier: List[String]): Unit diff --git a/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/JobProgressUtil.scala b/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/JobProgressUtil.scala new file mode 100644 index 00000000000..517bed0cc93 --- /dev/null +++ b/spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/JobProgressUtil.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark + +object JobProgressUtil { + + def progress(sc: SparkContext, jobGroup : String):Int = { + val jobIds = sc.statusTracker.getJobIdsForGroup(jobGroup) + val jobs = jobIds.flatMap { id => sc.statusTracker.getJobInfo(id) } + val stages = jobs.flatMap { job => + job.stageIds().flatMap(sc.statusTracker.getStageInfo) + } + + val taskCount = stages.map(_.numTasks).sum + val completedTaskCount = stages.map(_.numCompletedTasks).sum + if (taskCount == 0) { + 0 + } else { + (100 * completedTaskCount.toDouble / taskCount).toInt + } + } +} From 7fcbb9a0296b306896a82d4f3c89c4550327c674 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Thu, 8 Feb 2018 15:27:12 +0800 Subject: [PATCH 022/386] ZEPPELIN-3214. Restructure of spark interpreter module ### What is this PR for? Just restructure of spark interpreter module. spark module now is a sub module with all the following child module. * interpreter * scala-2.10 * scala-2.11 * spark-dependencies * spark-scala-parent ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3214 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2788 from zjffdu/ZEPPELIN-3214 and squashes the following commits: 8dcdbbd [Jeff Zhang] ZEPPELIN-3214. Restructure of spark interpreter module --- .travis.yml | 20 +++++++++---------- pom.xml | 5 +---- .../spark/IPySparkInterpreterTest.java | 2 +- spark/pom.xml | 19 ++++++++++++++++-- spark/scala-2.10/pom.xml | 14 ++++++------- spark/scala-2.11/pom.xml | 14 ++++++------- spark/spark-scala-parent/pom.xml | 1 + 7 files changed, 44 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index 24da368a3c4..c31694a4dbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,14 +68,14 @@ matrix: dist: trusty addons: firefox: "31.0" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk8" dist: trusty addons: firefox: "31.0" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false" # Test interpreter modules - jdk: "openjdk7" @@ -85,39 +85,39 @@ matrix: # Test spark module for 2.2.0 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 2.1.0 with scala 2.11 - jdk: "openjdk7" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 2.0.2 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test spark module for 1.6.3 with scala 2.11 + # Test spark module for 1.6.3 with scala 2.10 - jdk: "openjdk7" dist: trusty - env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 1.6.3 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test python/pyspark with python 2, livy 0.5 - sudo: required dist: trusty jdk: "openjdk7" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="-Dpyspark.test.exclude='' -DfailIfNoTests=false" # Test python/pyspark with python 3, livy 0.5 - sudo: required dist: trusty jdk: "openjdk7" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark/interpreter,spark/scala-2.10,spark/scala-2.11,spark/spark-dependencies,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="-Dpyspark.test.exclude='' -DfailIfNoTests=false" before_install: # check files included in commit range, clear bower_components if a bower.json file has changed. diff --git a/pom.xml b/pom.xml index 4cda650b6dc..6ce20aa6beb 100644 --- a/pom.xml +++ b/pom.xml @@ -57,10 +57,7 @@ zeppelin-zengine zeppelin-display groovy - spark/scala-2.10 - spark/scala-2.11 - spark/interpreter - spark/spark-dependencies + spark markdown angular shell diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java index 765237c3855..10d87a63e0a 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java @@ -182,7 +182,7 @@ public void run() { assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.getInterpreterResultMessages(); assertEquals(1, interpreterResultMessages.size()); -// assertTrue(interpreterResultMessages.get(0).getData().contains("(0, 100)")); + assertTrue(interpreterResultMessages.get(0).getData().contains("(0, 100)")); } private InterpreterContext getInterpreterContext() { diff --git a/spark/pom.xml b/spark/pom.xml index c55e453d210..7a0c7c2ac66 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -32,8 +32,8 @@ spark-parent pom 0.9.0-SNAPSHOT - spark-parent - Zeppelin spark support + Zeppelin: Spark Parent + Zeppelin Spark Support @@ -57,6 +57,14 @@ + + interpreter + spark-scala-parent + scala-2.10 + scala-2.11 + spark-dependencies + + @@ -120,6 +128,13 @@ + + org.apache.maven.plugins + maven-clean-plugin + ${plugin.clean.version} + + + org.scalatest scalatest-maven-plugin diff --git a/spark/scala-2.10/pom.xml b/spark/scala-2.10/pom.xml index e32e620bf30..3d34f4f4447 100644 --- a/spark/scala-2.10/pom.xml +++ b/spark/scala-2.10/pom.xml @@ -18,13 +18,6 @@ - 4.0.0 - org.apache.zeppelin - spark-scala-2.10 - 0.9.0-SNAPSHOT - jar - Spark Interpreter: Scala_2.10 - org.apache.zeppelin spark-scala-parent @@ -32,6 +25,13 @@ ../spark-scala-parent/pom.xml + 4.0.0 + org.apache.zeppelin + spark-scala-2.10 + 0.9.0-SNAPSHOT + jar + Zeppelin: Spark Interpreter Scala_2.10 + 2.10.5 2.10 diff --git a/spark/scala-2.11/pom.xml b/spark/scala-2.11/pom.xml index d9113d1075e..fc55afd8b94 100644 --- a/spark/scala-2.11/pom.xml +++ b/spark/scala-2.11/pom.xml @@ -18,13 +18,6 @@ - 4.0.0 - org.apache.zeppelin - spark-scala-2.11 - 0.9.0-SNAPSHOT - jar - Spark Interpreter: Scala_2.11 - org.apache.zeppelin spark-scala-parent @@ -32,6 +25,13 @@ ../spark-scala-parent/pom.xml + 4.0.0 + org.apache.zeppelin + spark-scala-2.11 + 0.9.0-SNAPSHOT + jar + Zeppelin: Spark Interpreter Scala_2.11 + 2.11.8 2.11 diff --git a/spark/spark-scala-parent/pom.xml b/spark/spark-scala-parent/pom.xml index 830fa59a684..91359c681e3 100644 --- a/spark/spark-scala-parent/pom.xml +++ b/spark/spark-scala-parent/pom.xml @@ -32,6 +32,7 @@ spark-scala-parent 0.9.0-SNAPSHOT pom + Zeppelin: Spark Scala Parent From aa13a0a57c5f08c82ba90f37da6c8d7a503697f3 Mon Sep 17 00:00:00 2001 From: guevara Date: Fri, 9 Feb 2018 16:09:31 -0500 Subject: [PATCH 023/386] Add exclusion to hadoop-aws dependency. ### What is this PR for? Fix dependency error caused by different versions of same library ### What type of PR is it? Bug Fix ### Todos * [ ZEPPELIN-3217] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3217 ### How should this be tested? * Travis CI ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: guevara Closes #2784 from wilsonr990/ZEPPELIN-3217 and squashes the following commits: 9f3f7e3 [guevara] Add exclusion to hadoop-aws dependency. --- zeppelin-zengine/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index fade4dd6123..a864fdf122c 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -669,6 +669,10 @@ hadoop-aws ${hadoop.version} + + com.amazonaws + aws-java-sdk + com.fasterxml.jackson.core jackson-annotations @@ -789,6 +793,10 @@ hadoop-aws ${hadoop.version} + + com.amazonaws + aws-java-sdk + com.fasterxml.jackson.core jackson-annotations From c6afe8c632295d3eab23745ddce59bb4286b4202 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 12 Feb 2018 14:10:25 +0800 Subject: [PATCH 024/386] ZEPPELIN-3222. Shade libfb303 in SparkInterpreter ### What is this PR for? Trivial change for shading libfb303, otherwise it would conflict with libfb303 of CDH. ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3222 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2790 from zjffdu/ZEPPELIN-3222 and squashes the following commits: 8e8528d [Jeff Zhang] ZEPPELIN-3222. Shade libfb303 in SparkInterpreter --- spark/interpreter/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index 758f6970e8e..e8d57a23f1e 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -478,6 +478,10 @@ py4j. org.apache.zeppelin.py4j. + + com.facebook.fb303 + org.apache.zeppelin.com.facebook.fb303 + From d1293c6bc476378c57db47d48dd8c5355370bb8a Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 12 Feb 2018 14:28:45 +0800 Subject: [PATCH 025/386] ZEPPELIN-3221. Create LocalConfigStorage to keep behavior consistent with previous version ### What is this PR for? Due to ZEPPELIN-2742, config will be stored on hdfs if user add HADOOP_CONF_DIR in zeppelin-env.sh, this is not consistent with the previous behavior (0.7) This PR just add LocalConfigStorage which would be the default storage for config which is the same behavior of 0.7 ### What type of PR is it? [Bug Fix | Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3221 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2791 from zjffdu/ZEPPELIN-3221 and squashes the following commits: b442808 [Jeff Zhang] ZEPPELIN-3221. Create LocalConfigStorage to keep behavior consistent with previous version --- .../zeppelin/spark/OldSparkInterpreter.java | 3 +- .../zeppelin/conf/ZeppelinConfiguration.java | 4 +- .../zeppelin/storage/ConfigStorage.java | 27 +++++ .../storage/FileSystemConfigStorage.java | 22 +--- .../zeppelin/storage/LocalConfigStorage.java | 110 ++++++++++++++++++ .../notebook/repo/NotebookRepoSyncTest.java | 5 + 6 files changed, 150 insertions(+), 21 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java index da332fe5ef0..ff3a2caa565 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java @@ -239,7 +239,8 @@ public void onExecutorAdded(SparkListenerExecutorAdded executorAdded) { } @Override - public void onExecutorMetricsUpdate(SparkListenerExecutorMetricsUpdate executorMetricsUpdate) { + public void onExecutorMetricsUpdate( + SparkListenerExecutorMetricsUpdate executorMetricsUpdate) { } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index a10732023fd..6bce468acfa 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -518,7 +518,7 @@ public String getConfigFSDir() { if (StringUtils.isBlank(fsConfigDir)) { LOG.warn(ConfVars.ZEPPELIN_CONFIG_FS_DIR.varName + " is not specified, fall back to local " + "conf directory " + ConfVars.ZEPPELIN_CONF_DIR.varName); - return "file://" + getConfDir(); + return getConfDir(); } return fsConfigDir; } @@ -709,7 +709,7 @@ public enum ConfVars { ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), ZEPPELIN_CONFIG_FS_DIR("zeppelin.config.fs.dir", ""), ZEPPELIN_CONFIG_STORAGE_CLASS("zeppelin.config.storage.class", - "org.apache.zeppelin.storage.FileSystemConfigStorage"), + "org.apache.zeppelin.storage.LocalConfigStorage"), ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"), ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL), ZEPPELIN_HELIUM_NODE_INSTALLER_URL("zeppelin.helium.node.installer.url", diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/ConfigStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/ConfigStorage.java index 3dc935fd0c7..b3175e59f3c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/ConfigStorage.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/ConfigStorage.java @@ -18,9 +18,13 @@ package org.apache.zeppelin.storage; +import com.google.common.annotations.VisibleForTesting; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.helium.HeliumConf; import org.apache.zeppelin.interpreter.InterpreterInfoSaving; +import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.NotebookAuthorizationInfoSaving; import org.apache.zeppelin.user.Credentials; import org.apache.zeppelin.user.CredentialsInfoSaving; @@ -75,4 +79,27 @@ public abstract void save(NotebookAuthorizationInfoSaving authorizationInfoSavin public abstract String loadCredentials() throws IOException; public abstract void saveCredentials(String credentials) throws IOException; + + protected InterpreterInfoSaving buildInterpreterInfoSaving(String json) { + //TODO(zjffdu) This kind of post processing is ugly. + JsonParser jsonParser = new JsonParser(); + JsonObject jsonObject = jsonParser.parse(json).getAsJsonObject(); + InterpreterInfoSaving infoSaving = InterpreterInfoSaving.fromJson(json); + for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) { + // Always use separate interpreter process + // While we decided to turn this feature on always (without providing + // enable/disable option on GUI). + // previously created setting should turn this feature on here. + interpreterSetting.getOption(); + interpreterSetting.convertPermissionsFromUsersToOwners( + jsonObject.getAsJsonObject("interpreterSettings") + .getAsJsonObject(interpreterSetting.getId())); + } + return infoSaving; + } + + @VisibleForTesting + public static void reset() { + instance = null; + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/FileSystemConfigStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/FileSystemConfigStorage.java index 4df8163470d..20c19b65cb7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/FileSystemConfigStorage.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/FileSystemConfigStorage.java @@ -74,21 +74,7 @@ public InterpreterInfoSaving loadInterpreterSettings() throws IOException { } LOGGER.info("Load Interpreter Setting from file: " + interpreterSettingPath); String json = fs.readFile(interpreterSettingPath); - //TODO(zjffdu) This kind of post processing is ugly. - JsonParser jsonParser = new JsonParser(); - JsonObject jsonObject = jsonParser.parse(json).getAsJsonObject(); - InterpreterInfoSaving infoSaving = InterpreterInfoSaving.fromJson(json); - for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) { - // Always use separate interpreter process - // While we decided to turn this feature on always (without providing - // enable/disable option on GUI). - // previously created setting should turn this feature on here. - interpreterSetting.getOption(); - interpreterSetting.convertPermissionsFromUsersToOwners( - jsonObject.getAsJsonObject("interpreterSettings") - .getAsJsonObject(interpreterSetting.getId())); - } - return infoSaving; + return buildInterpreterInfoSaving(json); } public void save(NotebookAuthorizationInfoSaving authorizationInfoSaving) throws IOException { @@ -99,7 +85,7 @@ public void save(NotebookAuthorizationInfoSaving authorizationInfoSaving) throws @Override public NotebookAuthorizationInfoSaving loadNotebookAuthorization() throws IOException { if (!fs.exists(authorizationPath)) { - LOGGER.warn("Interpreter Setting file {} is not existed", authorizationPath); + LOGGER.warn("Notebook Authorization file {} is not existed", authorizationPath); return null; } LOGGER.info("Load notebook authorization from file: " + authorizationPath); @@ -110,10 +96,10 @@ public NotebookAuthorizationInfoSaving loadNotebookAuthorization() throws IOExce @Override public String loadCredentials() throws IOException { if (!fs.exists(credentialPath)) { - LOGGER.warn("Credential file {} is not existed", authorizationPath); + LOGGER.warn("Credential file {} is not existed", credentialPath); return null; } - LOGGER.info("Load Credential from file: " + authorizationPath); + LOGGER.info("Load Credential from file: " + credentialPath); return this.fs.readFile(credentialPath); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java new file mode 100644 index 00000000000..c1edbb51dd4 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.storage; + +import org.apache.commons.io.IOUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.InterpreterInfoSaving; +import org.apache.zeppelin.notebook.NotebookAuthorizationInfoSaving; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + + +/** + * Storing config in local file system + */ +public class LocalConfigStorage extends ConfigStorage { + + private static Logger LOGGER = LoggerFactory.getLogger(LocalConfigStorage.class); + + private File interpreterSettingPath; + private File authorizationPath; + private File credentialPath; + + public LocalConfigStorage(ZeppelinConfiguration zConf) { + super(zConf); + this.interpreterSettingPath = new File(zConf.getInterpreterSettingPath()); + this.authorizationPath = new File(zConf.getNotebookAuthorizationPath()); + this.credentialPath = new File(zConf.getCredentialsPath()); + } + + @Override + public void save(InterpreterInfoSaving settingInfos) throws IOException { + writeToFile(settingInfos.toJson(), interpreterSettingPath); + } + + @Override + public InterpreterInfoSaving loadInterpreterSettings() throws IOException { + if (!interpreterSettingPath.exists()) { + LOGGER.warn("Interpreter Setting file {} is not existed", interpreterSettingPath); + return null; + } + LOGGER.info("Load Interpreter Setting from file: " + interpreterSettingPath); + String json = readFromFile(interpreterSettingPath); + return buildInterpreterInfoSaving(json); + } + + @Override + public void save(NotebookAuthorizationInfoSaving authorizationInfoSaving) throws IOException { + LOGGER.info("Save notebook authorization to file: " + authorizationPath); + writeToFile(authorizationInfoSaving.toJson(), authorizationPath); + } + + @Override + public NotebookAuthorizationInfoSaving loadNotebookAuthorization() throws IOException { + if (!authorizationPath.exists()) { + LOGGER.warn("NotebookAuthorization file {} is not existed", authorizationPath); + return null; + } + LOGGER.info("Load notebook authorization from file: " + authorizationPath); + String json = readFromFile(authorizationPath); + return NotebookAuthorizationInfoSaving.fromJson(json); + } + + @Override + public String loadCredentials() throws IOException { + if (!credentialPath.exists()) { + LOGGER.warn("Credential file {} is not existed", credentialPath); + return null; + } + LOGGER.info("Load Credential from file: " + credentialPath); + return readFromFile(credentialPath); + } + + @Override + public void saveCredentials(String credentials) throws IOException { + LOGGER.info("Save Credentials to file: " + credentialPath); + writeToFile(credentials, credentialPath); + } + + private String readFromFile(File file) throws IOException { + return IOUtils.toString(new FileInputStream(file)); + } + + private void writeToFile(String content, File file) throws IOException { + FileOutputStream out = new FileOutputStream(file); + IOUtils.write(content, out); + out.close(); + } + +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 22366545ee7..8904239310b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -44,6 +44,7 @@ import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.search.SearchService; +import org.apache.zeppelin.storage.ConfigStorage; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; import org.junit.After; @@ -89,10 +90,14 @@ public void setUp() throws Exception { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "false"); + System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), mainZepDir.getAbsolutePath() + "/conf"); + LOG.info("main Note dir : " + mainNotePath); LOG.info("secondary note dir : " + secNotePath); conf = ZeppelinConfiguration.create(); + ConfigStorage.reset(); + this.schedulerFactory = SchedulerFactory.singleton(); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); From 2be8f350658076c33d9d905b9e9907aa3d3a8792 Mon Sep 17 00:00:00 2001 From: Mohamed Magdy Date: Wed, 24 Jan 2018 10:11:15 +0100 Subject: [PATCH 026/386] [ZEPPELIN-3092] GitHub Integration ### What is this PR for? GitHub integration as a storage for notebooks. ### What type of PR is it? Feature ### What is the Jira issue? [ZEPPELIN-3092](https://issues.apache.org/jira/browse/ZEPPELIN-3092) ### How should this be tested? 1. Change the configuration in `zeppelin-site.xml` to enable GitHub integration (add GitHub url, username, access token and origin) as described in https://github.com/apache/zeppelin/compare/master...mohamagdy:zeppelin-3092-remote-github-integration?expand=1#diff-89104d48f0358450399a6f679bba9c4f 2. Start the Zeppelin server 3. Open an existing notebook or create a new notebook 4. Do some changes to the notebook, for example add a new paragraph 5. Click on the versioning button on the top menu to commit and save changes 6. Checkout the changes in the GitHub repository. The changes should be reflected ### Questions: * **Does the licenses files need update?** No * **Is there breaking changes for older versions?** No * **Does this needs documentation?** Yes. Documentation is updated as part of the pull request. Author: Mohamed Magdy Author: Mohamed Magdy Author: Mohamed Magdy Closes #2700 from mohamagdy/zeppelin-3092-remote-github-integration and squashes the following commits: b445960 [Mohamed Magdy] [ZEPPELIN-3092] Optimize imports for `Notebook` class afa5de1 [Mohamed Magdy] Merge branch 'master' of github.com:apache/zeppelin into zeppelin-3092-remote-github-integration 548c423 [Mohamed Magdy] [ZEPPELIN-3092] Add `zeppelin-site.xml` to `.gitignore` e98d1b0 [Mohamed Magdy] [ZEPPELIN-3092] Remove `zeppelin-site.xml` from Zeppelin Server resources 7a02855 [Mohamed Magdy] [ZEPPELIN-3092] Add Apache Software Foundation header 9101e58 [Mohamed Magdy] [ZEPPELIN-3092] Replace `printStackTrace()` with error logging db94d55 [Mohamed Magdy] [ZEPPELIN-3092] Remove loading notebook from repository when requested af952a0 [Mohamed Magdy] [ZEPPELIN-3029] Change authentication to `anonymous` instead of `empty` b5fbc1e [Mohamed Magdy] [ZEPPELIN-3092] Break long line to smaller ones 4d6cc76 [Mohamed Magdy] [ZEPPELIN-3092] Load notebook from repository when requested d1d43eb [Mohamed Magdy] Merge branch 'zeppelin-3092-remote-github-integration' of github.com:mohamagdy/zeppelin into zeppelin-3092-remote-github-integration 579bd6f [Mohamed Magdy] [ZEPPELIN-3092] Load note from memory when reloading 2f1b8bc [Mohamed Magdy] [ZEPPELIN-3092] Load note from memory when reloading d545e81 [Mohamed Magdy] [ZEPPELIN-3092] Remove duplicated dependency from `pom.xml` fc13fa6 [Mohamed Magdy] Revert "[ZEPPELIN-3029] Increase Paragraph and Browser timeouts" be2c278 [Mohamed Magdy] Revert "[ZEPPELIN-3092] Set browser timeout to 180 seconds" f362dcb [Mohamed Magdy] [ZEPPELIN-3029] Use jGit version 4.5.4 instead of 4.3.1 8bd23d0 [Mohamed Magdy] [ZEPPELIN-3092] Set browser timeout to 180 seconds 30f2ab4 [Mohamed Magdy] [ZEPPELIN-3029] Increase Paragraph and Browser timeouts 13a0014 [Mohamed Magdy] [ZEPPELIN-3092] Disable GitHub configuration for Zeppelin server 14cb024 [Mohamed Magdy] [ZEPPELIN-3092] Fix notebook path for Git and GitHub tests 0e9db3f [Mohamed Magdy] [ZEPPELIN-3092] Remove test GitHub repository URL and access token 90de14c [Mohamed Magdy] Merge branch 'master' into zeppelin-3092-remote-github-integration 2c1cf74 [Mohamed Magdy] [ZEPPELIN-3029] Fix remote origin key name 6ba67ca [Mohamed Magdy] [ZEPPELIN-3092] Add Javadoc to `GitHubNotebookRepo` and fix line length to 100 264565b [Mohamed Magdy] [ZEPPELIN-3092] Fix line length to be 100 0174bbd [Mohamed Magdy] [ZEPPELIN-3092] Add documentation how to enabled `GitHubNotebookRepo` 81969e1 [Mohamed Magdy] [ZEPPELIN-3092] Add documentation for loading notebooks from repo 3009abd [Mohamed Magdy] [ZEPPELIN-3092] Reset `GitNotebookRepo` to `master` 6aa4ba7 [Mohamed Magdy] [ZEPPELIN-3092] Revert back `GitNotebookRepo` to `master` b77a2d3 [Mohamed Magdy] [ZEPPELIN-3092] Fix identation in `pom.xml` aadd9b5 [Mohamed Magdy] [ZEPPELIN-3092] Revert back ZeppelinServer changes 0dacbf1 [Mohamed Magdy] [ZEPPELIN-3092] Fix encoding in the documenation 2b093b2 [Mohamed Magdy] [ZEPPELIN-3092] Add documentation about GitHub integration 843e42a [Mohamed Magdy] [ZEPPELIN-3092] Cleanup GitHub repository tests 5236176 [Mohamed Magdy] [ZEPPELIN-3092] Move GitHub notebook repostiory to separte file 2dbf116 [Mohamed Magdy] [ZEPPELIN-3092] Add GitHub configuration to `zeppelin-site.xml` template bb0afe2 [Mohamed Magdy] [ZEPPELIN-3092] Add GitHub remote to configurations 33ae24a [Mohamed Magdy] [ZEPPELIN-3092] Add remote Github repository synchronzing 32f6764 [Mohamed Magdy] [ZEPPELIN-3092] Fix GitNotebook test eeb485a [Mohamed Magdy] [ZEPPELIN-3092] Add Github configuration reader 0bde310 [Mohamed Magdy] [ZEPPELIN-3092] Add `zeppelin-site.xml` to `zeppelin-server` resources 9467503 [Mohamed Magdy] [ZEPPELIN-3092] Add `zepplein-server/local-repo` to `.gitignore` --- .gitignore | 3 + conf/zeppelin-site.xml.template | 24 ++ .../contribution/how_to_contribute_code.md | 12 + docs/setup/operation/configuration.md | 30 +- docs/setup/storage/storage.md | 42 ++ .../zeppelin/conf/ZeppelinConfiguration.java | 22 +- .../zeppelin/socket/NotebookServer.java | 2 +- .../src/test/resources/2A94M5J1Z/note.json | 376 ++++++++++++++++++ .../src/test/resources/2A94M5J2Z/note.json | 376 ++++++++++++++++++ .../apache/zeppelin/notebook/Notebook.java | 38 +- .../notebook/repo/GitHubNotebookRepo.java | 126 ++++++ .../notebook/repo/GitNotebookRepo.java | 5 +- .../notebook/repo/GitHubNotebookRepoTest.java | 207 ++++++++++ .../notebook/repo/GitNotebookRepoTest.java | 16 +- 14 files changed, 1243 insertions(+), 36 deletions(-) create mode 100644 zeppelin-server/src/test/resources/2A94M5J1Z/note.json create mode 100644 zeppelin-server/src/test/resources/2A94M5J2Z/note.json create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java diff --git a/.gitignore b/.gitignore index 773edc80676..4086a4bb43e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ spark-1.*-bin-hadoop* lens/lens-cli-hist.log +# Zeppelin server +zeppelin-server/local-repo +zeppelin-server/src/main/resources/zeppelin-site.xml # conf file conf/zeppelin-env.sh diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 33aa8acf6da..9e9898bb4db 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -499,5 +499,29 @@ --> + diff --git a/docs/development/contribution/how_to_contribute_code.md b/docs/development/contribution/how_to_contribute_code.md index b172aa193be..92b69b5c267 100644 --- a/docs/development/contribution/how_to_contribute_code.md +++ b/docs/development/contribution/how_to_contribute_code.md @@ -89,11 +89,17 @@ For the further ### Run Zeppelin server in development mode +#### Option 1 - Command Line + +1. Copy the `conf/zeppelin-site.xml.template` to `zeppelin-server/src/main/resources/zeppelin-site.xml` and change the configurations in this file if required +2. Run the following command ``` cd zeppelin-server HADOOP_HOME=YOUR_HADOOP_HOME JAVA_HOME=YOUR_JAVA_HOME mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args="" ``` +#### Option 2 - Daemon Script + > **Note:** Make sure you first run ```mvn clean install -DskipTests``` on your zeppelin root directory, otherwise your server build will fail to find the required dependencies in the local repro. or use daemon script @@ -104,6 +110,12 @@ bin/zeppelin-daemon start Server will be run on [http://localhost:8080](http://localhost:8080). +#### Option 3 - IDE + +1. Copy the `conf/zeppelin-site.xml.template` to `zeppelin-server/src/main/resources/zeppelin-site.xml` and change the configurations in this file if required +2. `ZeppelinServer.java` Main class + + ### Generating Thrift Code Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this. But if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch. diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md index 1f4c6a24231..ed4e1f26ac0 100644 --- a/docs/setup/operation/configuration.md +++ b/docs/setup/operation/configuration.md @@ -329,6 +329,30 @@ If both are defined, then the **environment variables** will take priority. false Enable directory listings on server. + +
ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL
+
zeppelin.notebook.git.remote.url
+ + GitHub's repository URL. It could be either the HTTP URL or the SSH URL. For example git@github.com:apache/zeppelin.git + + +
ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME
+
zeppelin.notebook.git.remote.username
+ token + GitHub username. By default it is `token` to use GitHub's API + + +
ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN
+
zeppelin.notebook.git.remote.access-token
+ token + GitHub access token to use GitHub's API. If username/password combination is used and not GitHub API, then this value is the password + + +
ZEPPELIN_NOTEBOOK_GIT_REMOTE_ORIGIN
+
zeppelin.notebook.git.remote.origin
+ token + GitHub remote name. Default is `origin` + @@ -431,7 +455,7 @@ The following properties needs to be updated in the `zeppelin-site.xml` in order ### Storing user credentials -In order to avoid having to re-enter credentials every time you restart/redeploy Zeppelin, you can store the user credentials. Zeppelin supports this via the ZEPPELIN_CREDENTIALS_PERSIST configuration. +In order to avoid having to re-enter credentials every time you restart/redeploy Zeppelin, you can store the user credentials. Zeppelin supports this via the ZEPPELIN_CREDENTIALS_PERSIST configuration. Please notice that passwords will be stored in *plain text* by default. To encrypt the passwords, use the ZEPPELIN_CREDENTIALS_ENCRYPT_KEY config variable. This will encrypt passwords using the AES-128 algorithm. @@ -473,5 +497,9 @@ update your configuration with the obfuscated password : ``` +### Create GitHub Access Token + +When using GitHub to track notebooks, one can use GitHub's API for authentication. To create an access token, please use the following link https://github.com/settings/tokens. +The value of the access token generated is set in the `zeppelin.notebook.git.remote.access-token` property. **Note:** After updating these configurations, Zeppelin server needs to be restarted. diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md index f6b8b5c08d1..f34fc2cfc65 100644 --- a/docs/setup/storage/storage.md +++ b/docs/setup/storage/storage.md @@ -34,6 +34,7 @@ There are few notebook storage systems available for a use out of the box: * storage using Amazon S3 service - `S3NotebookRepo` * storage using Azure service - `AzureNotebookRepo` * storage using MongoDB - `MongoNotebookRepo` + * storage using GitHub - `GitHubNotebookRepo` Multiple storage systems can be used at the same time by providing a comma-separated list of the class-names in the configuration. By default, only first two of them will be automatically kept in sync by Zeppelin. @@ -361,3 +362,44 @@ export ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT=true #### Import your local notes automatically By setting `ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT` as `true` (default `false`), you can import your local notes automatically when Zeppelin daemon starts up. This feature is for easy migration from local file system storage to MongoDB storage. A note with ID already existing in the collection will not be imported. + +## Notebook Storage in GitHub + +To enable GitHub tracking, uncomment the following properties in `zeppelin-site.xml` + +```sh + + zeppelin.notebook.git.remote.url + + remote Git repository URL + + + + zeppelin.notebook.git.remote.username + token + remote Git repository username + + + + zeppelin.notebook.git.remote.access-token + + remote Git repository password + + + + zeppelin.notebook.git.remote.origin + origin + Git repository remote + +``` + +And set the `zeppelin.notebook.storage` propery to `org.apache.zeppelin.notebook.repo.GitHubNotebookRepo` + +```sh + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.GitHubNotebookRepo + +``` + +The access token could be obtained by following the steps on this link https://github.com/settings/tokens. \ No newline at end of file diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 6bce468acfa..f7b3d7b09f5 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -561,6 +561,22 @@ public String getLifecycleManagerClass() { return getString(ConfVars.ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_CLASS); } + public String getZeppelinNotebookGitURL() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL); + } + + public String getZeppelinNotebookGitUsername() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME); + } + + public String getZeppelinNotebookGitAccessToken() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN); + } + + public String getZeppelinNotebookGitRemoteOrigin() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ORIGIN); + } + public Map dumpConfigurations(ZeppelinConfiguration conf, ConfigurationKeyPredicate predicate) { Map configurations = new HashMap<>(); @@ -745,8 +761,12 @@ public enum ConfVars { ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_TIMEOUT_THRESHOLD( "zeppelin.interpreter.lifecyclemanager.timeout.threshold", 3600000L), - ZEPPELIN_OWNER_ROLE("zeppelin.notebook.default.owner.username", ""); + ZEPPELIN_OWNER_ROLE("zeppelin.notebook.default.owner.username", ""), + ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL("zeppelin.notebook.git.remote.url", ""), + ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME("zeppelin.notebook.git.remote.username", "token"), + ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN("zeppelin.notebook.git.remote.access-token", ""), + ZEPPELIN_NOTEBOOK_GIT_REMOTE_ORIGIN("zeppelin.notebook.git.remote.origin", "origin"); private String varName; @SuppressWarnings("rawtypes") diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 56aa50a3eb5..20d5ba9cc90 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -824,8 +824,8 @@ private void sendNote(NotebookSocket conn, HashSet userAndRoles, Noteboo String user = fromMessage.principal; Note note = notebook.getNote(noteId); - if (note != null) { + if (note != null) { if (!hasParagraphReaderPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "read")) { return; diff --git a/zeppelin-server/src/test/resources/2A94M5J1Z/note.json b/zeppelin-server/src/test/resources/2A94M5J1Z/note.json new file mode 100644 index 00000000000..6e8e06fe296 --- /dev/null +++ b/zeppelin-server/src/test/resources/2A94M5J1Z/note.json @@ -0,0 +1,376 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)", + "user": "anonymous", + "dateUpdated": "Dec 17, 2016 3:32:15 PM", + "config": { + "colWidth": 12.0, + "editorHide": true, + "results": [ + { + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + } + ], + "enabled": true, + "editorSetting": { + "language": "markdown", + "editOnDblClick": true + }, + "editorMode": "ace/mode/markdown", + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n\u003c/div\u003e" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423836981412_-1007008116", + "id": "20150213-231621_168813393", + "dateCreated": "Feb 13, 2015 11:16:21 PM", + "dateStarted": "Dec 17, 2016 3:32:15 PM", + "dateFinished": "Dec 17, 2016 3:32:18 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "title": "Load data into table", + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")", + "user": "anonymous", + "dateUpdated": "Dec 17, 2016 3:30:09 PM", + "config": { + "colWidth": 12.0, + "title": true, + "enabled": true, + "editorMode": "ace/mode/scala", + "results": [ + { + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false + } + } + ], + "editorSetting": { + "language": "scala", + "editOnDblClick": false + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TEXT", + "data": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[36] at parallelize at \u003cconsole\u003e:43\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string ... 3 more fields]\nwarning: there were 1 deprecation warning(s); re-run with -deprecation for details\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423500779206_-1502780787", + "id": "20150210-015259_1403135953", + "dateCreated": "Feb 10, 2015 1:52:59 AM", + "dateStarted": "Dec 17, 2016 3:30:09 PM", + "dateFinished": "Dec 17, 2016 3:30:58 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age", + "user": "anonymous", + "dateUpdated": "Mar 17, 2017 12:18:02 PM", + "config": { + "colWidth": 4.0, + "results": [ + { + "graph": { + "mode": "multiBarChart", + "height": 366.0, + "optionOpen": false + }, + "helium": {} + } + ], + "enabled": true, + "editorSetting": { + "language": "sql", + "editOnDblClick": false + }, + "editorMode": "ace/mode/sql" + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423500782552_-1439281894", + "id": "20150210-015302_1492795503", + "dateCreated": "Feb 10, 2015 1:53:02 AM", + "dateStarted": "Dec 17, 2016 3:30:13 PM", + "dateFinished": "Dec 17, 2016 3:31:04 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age", + "user": "anonymous", + "dateUpdated": "Mar 17, 2017 12:17:39 PM", + "config": { + "colWidth": 4.0, + "results": [ + { + "graph": { + "mode": "multiBarChart", + "height": 294.0, + "optionOpen": false + }, + "helium": {} + } + ], + "enabled": true, + "editorSetting": { + "language": "sql", + "editOnDblClick": false + }, + "editorMode": "ace/mode/sql" + }, + "settings": { + "params": { + "maxAge": "35" + }, + "forms": { + "maxAge": { + "name": "maxAge", + "defaultValue": "30", + "hidden": false + } + } + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423720444030_-1424110477", + "id": "20150212-145404_867439529", + "dateCreated": "Feb 12, 2015 2:54:04 PM", + "dateStarted": "Dec 17, 2016 3:30:58 PM", + "dateFinished": "Dec 17, 2016 3:31:07 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age", + "user": "anonymous", + "dateUpdated": "Mar 17, 2017 12:18:18 PM", + "config": { + "colWidth": 4.0, + "results": [ + { + "graph": { + "mode": "stackedAreaChart", + "height": 280.0, + "optionOpen": false + }, + "helium": {} + } + ], + "enabled": true, + "editorSetting": { + "language": "sql", + "editOnDblClick": false + }, + "editorMode": "ace/mode/sql" + }, + "settings": { + "params": { + "marital": "single" + }, + "forms": { + "marital": { + "name": "marital", + "defaultValue": "single", + "options": [ + { + "value": "single" + }, + { + "value": "divorced" + }, + { + "value": "married" + } + ], + "hidden": false + } + } + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423836262027_-210588283", + "id": "20150213-230422_1600658137", + "dateCreated": "Feb 13, 2015 11:04:22 PM", + "dateStarted": "Dec 17, 2016 3:31:05 PM", + "dateFinished": "Dec 17, 2016 3:31:09 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!", + "user": "anonymous", + "dateUpdated": "Dec 17, 2016 3:30:24 PM", + "config": { + "colWidth": 12.0, + "editorHide": true, + "results": [ + { + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false + } + } + ], + "enabled": true, + "editorSetting": { + "language": "markdown", + "editOnDblClick": true + }, + "editorMode": "ace/mode/markdown", + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eCongratulations, it\u0026rsquo;s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0026lsquo;Notebook\u0026rsquo; menu. Good luck!\u003c/h5\u003e\n\u003c/div\u003e" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423836268492_216498320", + "id": "20150213-230428_1231780373", + "dateCreated": "Feb 13, 2015 11:04:28 PM", + "dateStarted": "Dec 17, 2016 3:30:24 PM", + "dateFinished": "Dec 17, 2016 3:30:29 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```", + "user": "anonymous", + "dateUpdated": "Dec 17, 2016 3:30:34 PM", + "config": { + "colWidth": 12.0, + "editorHide": true, + "results": [ + { + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false + } + } + ], + "enabled": true, + "editorSetting": { + "language": "markdown", + "editOnDblClick": true + }, + "editorMode": "ace/mode/markdown", + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u0026#39;2011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n\u003c/div\u003e" + } + ] + }, + "apps": [], + "jobName": "paragraph_1427420818407_872443482", + "id": "20150326-214658_12335843", + "dateCreated": "Mar 26, 2015 9:46:58 PM", + "dateStarted": "Dec 17, 2016 3:30:34 PM", + "dateFinished": "Dec 17, 2016 3:30:34 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "apps": [], + "jobName": "paragraph_1435955447812_-158639899", + "id": "20150703-133047_853701097", + "dateCreated": "Jul 3, 2015 1:30:47 PM", + "status": "READY", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Zeppelin Tutorial/Basic Features (Spark)", + "id": "2A94M5J1Z", + "angularObjects": { + "2C73DY9P9:shared_process": [] + }, + "config": { + "looknfeel": "default" + }, + "info": {} +} \ No newline at end of file diff --git a/zeppelin-server/src/test/resources/2A94M5J2Z/note.json b/zeppelin-server/src/test/resources/2A94M5J2Z/note.json new file mode 100644 index 00000000000..dd9a74df9f7 --- /dev/null +++ b/zeppelin-server/src/test/resources/2A94M5J2Z/note.json @@ -0,0 +1,376 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)", + "user": "anonymous", + "dateUpdated": "Dec 17, 2016 3:32:15 PM", + "config": { + "colWidth": 12.0, + "editorHide": true, + "results": [ + { + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + } + ], + "enabled": true, + "editorSetting": { + "language": "markdown", + "editOnDblClick": true + }, + "editorMode": "ace/mode/markdown", + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n\u003c/div\u003e" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423836981412_-1007008116", + "id": "20150213-231621_168813393", + "dateCreated": "Feb 13, 2015 11:16:21 PM", + "dateStarted": "Dec 17, 2016 3:32:15 PM", + "dateFinished": "Dec 17, 2016 3:32:18 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "title": "Load data into table", + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")", + "user": "anonymous", + "dateUpdated": "Dec 17, 2016 3:30:09 PM", + "config": { + "colWidth": 12.0, + "title": true, + "enabled": true, + "editorMode": "ace/mode/scala", + "results": [ + { + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false + } + } + ], + "editorSetting": { + "language": "scala", + "editOnDblClick": false + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TEXT", + "data": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[36] at parallelize at \u003cconsole\u003e:43\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string ... 3 more fields]\nwarning: there were 1 deprecation warning(s); re-run with -deprecation for details\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423500779206_-1502780787", + "id": "20150210-015259_1403135953", + "dateCreated": "Feb 10, 2015 1:52:59 AM", + "dateStarted": "Dec 17, 2016 3:30:09 PM", + "dateFinished": "Dec 17, 2016 3:30:58 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age", + "user": "anonymous", + "dateUpdated": "Mar 17, 2017 12:18:02 PM", + "config": { + "colWidth": 4.0, + "results": [ + { + "graph": { + "mode": "multiBarChart", + "height": 366.0, + "optionOpen": false + }, + "helium": {} + } + ], + "enabled": true, + "editorSetting": { + "language": "sql", + "editOnDblClick": false + }, + "editorMode": "ace/mode/sql" + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423500782552_-1439281894", + "id": "20150210-015302_1492795503", + "dateCreated": "Feb 10, 2015 1:53:02 AM", + "dateStarted": "Dec 17, 2016 3:30:13 PM", + "dateFinished": "Dec 17, 2016 3:31:04 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age", + "user": "anonymous", + "dateUpdated": "Mar 17, 2017 12:17:39 PM", + "config": { + "colWidth": 4.0, + "results": [ + { + "graph": { + "mode": "multiBarChart", + "height": 294.0, + "optionOpen": false + }, + "helium": {} + } + ], + "enabled": true, + "editorSetting": { + "language": "sql", + "editOnDblClick": false + }, + "editorMode": "ace/mode/sql" + }, + "settings": { + "params": { + "maxAge": "35" + }, + "forms": { + "maxAge": { + "name": "maxAge", + "defaultValue": "30", + "hidden": false + } + } + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423720444030_-1424110477", + "id": "20150212-145404_867439529", + "dateCreated": "Feb 12, 2015 2:54:04 PM", + "dateStarted": "Dec 17, 2016 3:30:58 PM", + "dateFinished": "Dec 17, 2016 3:31:07 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age", + "user": "anonymous", + "dateUpdated": "Mar 17, 2017 12:18:18 PM", + "config": { + "colWidth": 4.0, + "results": [ + { + "graph": { + "mode": "stackedAreaChart", + "height": 280.0, + "optionOpen": false + }, + "helium": {} + } + ], + "enabled": true, + "editorSetting": { + "language": "sql", + "editOnDblClick": false + }, + "editorMode": "ace/mode/sql" + }, + "settings": { + "params": { + "marital": "single" + }, + "forms": { + "marital": { + "name": "marital", + "defaultValue": "single", + "options": [ + { + "value": "single" + }, + { + "value": "divorced" + }, + { + "value": "married" + } + ], + "hidden": false + } + } + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423836262027_-210588283", + "id": "20150213-230422_1600658137", + "dateCreated": "Feb 13, 2015 11:04:22 PM", + "dateStarted": "Dec 17, 2016 3:31:05 PM", + "dateFinished": "Dec 17, 2016 3:31:09 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!", + "user": "anonymous", + "dateUpdated": "Dec 17, 2016 3:30:24 PM", + "config": { + "colWidth": 12.0, + "editorHide": true, + "results": [ + { + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false + } + } + ], + "enabled": true, + "editorSetting": { + "language": "markdown", + "editOnDblClick": true + }, + "editorMode": "ace/mode/markdown", + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eCongratulations, it\u0026rsquo;s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0026lsquo;Notebook\u0026rsquo; menu. Good luck!\u003c/h5\u003e\n\u003c/div\u003e" + } + ] + }, + "apps": [], + "jobName": "paragraph_1423836268492_216498320", + "id": "20150213-230428_1231780373", + "dateCreated": "Feb 13, 2015 11:04:28 PM", + "dateStarted": "Dec 17, 2016 3:30:24 PM", + "dateFinished": "Dec 17, 2016 3:30:29 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```", + "user": "anonymous", + "dateUpdated": "Dec 17, 2016 3:30:34 PM", + "config": { + "colWidth": 12.0, + "editorHide": true, + "results": [ + { + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false + } + } + ], + "enabled": true, + "editorSetting": { + "language": "markdown", + "editOnDblClick": true + }, + "editorMode": "ace/mode/markdown", + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u0026#39;2011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n\u003c/div\u003e" + } + ] + }, + "apps": [], + "jobName": "paragraph_1427420818407_872443482", + "id": "20150326-214658_12335843", + "dateCreated": "Mar 26, 2015 9:46:58 PM", + "dateStarted": "Dec 17, 2016 3:30:34 PM", + "dateFinished": "Dec 17, 2016 3:30:34 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "apps": [], + "jobName": "paragraph_1435955447812_-158639899", + "id": "20150703-133047_853701097", + "dateCreated": "Jul 3, 2015 1:30:47 PM", + "status": "READY", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Zeppelin Tutorial/Basic Features (Spark)", + "id": "2A94M5J2Z", + "angularObjects": { + "2C73DY9P9:shared_process": [] + }, + "config": { + "looknfeel": "default" + }, + "info": {} +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index ff0ac62b03e..72ea2acca7b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -17,42 +17,16 @@ package org.apache.zeppelin.notebook; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; import com.google.common.collect.Sets; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; -import org.quartz.CronScheduleBuilder; -import org.quartz.CronTrigger; -import org.quartz.JobBuilder; -import org.quartz.JobDetail; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; -import org.quartz.JobKey; -import org.quartz.SchedulerException; -import org.quartz.TriggerBuilder; -import org.quartz.impl.StdSchedulerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision; import org.apache.zeppelin.notebook.repo.NotebookRepoSync; @@ -61,6 +35,14 @@ import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; +import org.quartz.*; +import org.quartz.impl.StdSchedulerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.TimeUnit; /** * Collection of Notes. diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java new file mode 100644 index 00000000000..6052e5fd756 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.PullCommand; +import org.eclipse.jgit.api.PushCommand; +import org.eclipse.jgit.api.RemoteAddCommand; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URISyntaxException; + +/** + * GitHub integration to store notebooks in a GitHub repository. + * It uses the same simple logic implemented in @see + * {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo} + * + * The logic for updating the local repository from the remote repository is the following: + * - When the GitHubNotebookRepo is initialized + * - When pushing the changes to the remote repository + * + * The logic for updating the remote repository on GitHub from local repository is the following: + * - When commit the changes (saving the notebook) + */ +public class GitHubNotebookRepo extends GitNotebookRepo { + private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class); + private ZeppelinConfiguration zeppelinConfiguration; + private Git git; + + public GitHubNotebookRepo(ZeppelinConfiguration conf) throws IOException { + super(conf); + + this.git = super.getGit(); + this.zeppelinConfiguration = conf; + + configureRemoteStream(); + pullFromRemoteStream(); + } + + @Override + public Revision checkpoint(String pattern, String commitMessage, AuthenticationInfo subject) { + Revision revision = super.checkpoint(pattern, commitMessage, subject); + + updateRemoteStream(); + + return revision; + } + + private void configureRemoteStream() { + try { + LOG.debug("Setting up remote stream"); + RemoteAddCommand remoteAddCommand = git.remoteAdd(); + remoteAddCommand.setName(zeppelinConfiguration.getZeppelinNotebookGitRemoteOrigin()); + remoteAddCommand.setUri(new URIish(zeppelinConfiguration.getZeppelinNotebookGitURL())); + remoteAddCommand.call(); + } catch (GitAPIException e) { + LOG.error("Error configuring GitHub", e); + } catch (URISyntaxException e) { + LOG.error("Error in GitHub URL provided", e); + } + } + + private void updateRemoteStream() { + LOG.debug("Updating remote stream"); + + pullFromRemoteStream(); + pushToRemoteSteam(); + } + + private void pullFromRemoteStream() { + try { + LOG.debug("Pull latest changed from remote stream"); + PullCommand pullCommand = git.pull(); + pullCommand.setCredentialsProvider( + new UsernamePasswordCredentialsProvider( + zeppelinConfiguration.getZeppelinNotebookGitUsername(), + zeppelinConfiguration.getZeppelinNotebookGitAccessToken() + ) + ); + + pullCommand.call(); + + } catch (GitAPIException e) { + LOG.error("Error when pulling latest changes from remote repository", e); + } + } + + private void pushToRemoteSteam() { + try { + LOG.debug("Push latest changed from remote stream"); + PushCommand pushCommand = git.push(); + pushCommand.setCredentialsProvider( + new UsernamePasswordCredentialsProvider( + zeppelinConfiguration.getZeppelinNotebookGitUsername(), + zeppelinConfiguration.getZeppelinNotebookGitAccessToken() + ) + ); + + pushCommand.call(); + } catch (GitAPIException e) { + LOG.error("Error when pushing latest changes from remote repository", e); + } + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java index 21183da3d0d..2ac4c725456 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java @@ -47,7 +47,8 @@ * * This impl intended to be simple and straightforward: * - does not handle branches - * - only basic local git file repo, no remote Github push\pull yet + * - only basic local git file repo, no remote Github push\pull. GitHub integration is + * implemented in @see {@link org.apache.zeppelin.notebook.repo.GitHubNotebookRepo} * * TODO(bzz): add default .gitignore */ @@ -177,7 +178,7 @@ public void close() { } //DI replacements for Tests - Git getGit() { + protected Git getGit() { return git; } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java new file mode 100644 index 00000000000..49a5cbde621 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + + +import com.google.common.base.Joiner; +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import static org.mockito.Mockito.mock; + +/** + * This tests the remote Git tracking for notebooks. The tests uses two local Git repositories created locally + * to handle the tracking of Git actions (pushes and pulls). The repositories are: + * 1. The first repository is considered as a remote that mimics a remote GitHub directory + * 2. The second repository is considered as the local notebook repository + */ +public class GitHubNotebookRepoTest { + private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepoTest.class); + + private static final String TEST_NOTE_ID = "2A94M5J1Z"; + + private File remoteZeppelinDir; + private File localZeppelinDir; + private String localNotebooksDir; + private String remoteNotebooksDir; + private ZeppelinConfiguration conf; + private GitHubNotebookRepo gitHubNotebookRepo; + private RevCommit firstCommitRevision; + private Git remoteGit; + + @Before + public void setUp() throws Exception { + conf = ZeppelinConfiguration.create(); + + String remoteRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTestRemote_" + + System.currentTimeMillis(); + String localRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + + System.currentTimeMillis(); + + // Create a fake remote notebook Git repository locally in another directory + remoteZeppelinDir = new File(remoteRepositoryPath); + remoteZeppelinDir.mkdirs(); + + // Create a local repository for notebooks + localZeppelinDir = new File(localRepositoryPath); + localZeppelinDir.mkdirs(); + + // Notebooks directory (for both the remote and local directories) + localNotebooksDir = Joiner.on(File.separator).join(localRepositoryPath, "notebook"); + remoteNotebooksDir = Joiner.on(File.separator).join(remoteRepositoryPath, "notebook"); + + File notebookDir = new File(localNotebooksDir); + notebookDir.mkdirs(); + + // Copy the test notebook directory from the test/resources/2A94M5J1Z folder to the fake remote Git directory + String remoteTestNoteDir = Joiner.on(File.separator).join(remoteNotebooksDir, TEST_NOTE_ID); + FileUtils.copyDirectory( + new File( + GitHubNotebookRepoTest.class.getResource( + Joiner.on(File.separator).join("", TEST_NOTE_ID) + ).getFile() + ), new File(remoteTestNoteDir) + ); + + // Create the fake remote Git repository + Repository remoteRepository = new FileRepository(Joiner.on(File.separator).join(remoteNotebooksDir, ".git")); + remoteRepository.create(); + + remoteGit = new Git(remoteRepository); + remoteGit.add().addFilepattern(".").call(); + firstCommitRevision = remoteGit.commit().setMessage("First commit from remote repository").call(); + + // Set the Git and Git configurations + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), remoteZeppelinDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); + + // Set the GitHub configurations + System.setProperty( + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), + "org.apache.zeppelin.notebook.repo.GitHubNotebookRepo"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL.getVarName(), + remoteNotebooksDir + File.separator + ".git"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME.getVarName(), "token"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN.getVarName(), + "access-token"); + + // Create the Notebook repository (configured for the local repository) + gitHubNotebookRepo = new GitHubNotebookRepo(conf); + } + + @After + public void tearDown() throws Exception { + // Cleanup the temporary folders uses as Git repositories + File[] temporaryFolders = { remoteZeppelinDir, localZeppelinDir }; + + for(File temporaryFolder : temporaryFolders) { + if (!FileUtils.deleteQuietly(temporaryFolder)) + LOG.error("Failed to delete {} ", temporaryFolder.getName()); + } + } + + @Test + /** + * Test the case when the Notebook repository is created, it pulls the latest changes from the remote repository + */ + public void pullChangesFromRemoteRepositoryOnLoadingNotebook() throws IOException, GitAPIException { + NotebookRepo.Revision firstHistoryRevision = gitHubNotebookRepo.revisionHistory(TEST_NOTE_ID, null).get(0); + + assert(this.firstCommitRevision.getName().equals(firstHistoryRevision.id)); + } + + @Test + /** + * Test the case when the check-pointing (add new files and commit) it also pulls the latest changes from the + * remote repository + */ + public void pullChangesFromRemoteRepositoryOnCheckpointing() throws GitAPIException, IOException { + // Create a new commit in the remote repository + RevCommit secondCommitRevision = remoteGit.commit().setMessage("Second commit from remote repository").call(); + + // Add a new paragraph to the local repository + addParagraphToNotebook(TEST_NOTE_ID); + + // Commit and push the changes to remote repository + NotebookRepo.Revision thirdCommitRevision = gitHubNotebookRepo.checkpoint( + TEST_NOTE_ID, "Third commit from local repository", null); + + // Check all the commits as seen from the local repository. The commits are ordered chronologically. The last + // commit is the first in the commit logs. + Iterator revisions = gitHubNotebookRepo.getGit().log().all().call().iterator(); + + revisions.next(); // The Merge `master` commit after pushing to the remote repository + + assert(thirdCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph + + // The second commit done on the remote repository + assert(secondCommitRevision.getName().equals(revisions.next().getName())); + + // The first commit done on the remote repository + assert(firstCommitRevision.getName().equals(revisions.next().getName())); + } + + @Test + /** + * Test the case when the check-pointing (add new files and commit) it pushes the local commits to the remote + * repository + */ + public void pushLocalChangesToRemoteRepositoryOnCheckpointing() throws IOException, GitAPIException { + // Add a new paragraph to the local repository + addParagraphToNotebook(TEST_NOTE_ID); + + // Commit and push the changes to remote repository + NotebookRepo.Revision secondCommitRevision = gitHubNotebookRepo.checkpoint( + TEST_NOTE_ID, "Second commit from local repository", null); + + // Check all the commits as seen from the remote repository. The commits are ordered chronologically. The last + // commit is the first in the commit logs. + Iterator revisions = remoteGit.log().all().call().iterator(); + + assert(secondCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph + + // The first commit done on the remote repository + assert(firstCommitRevision.getName().equals(revisions.next().getName())); + } + + private void addParagraphToNotebook(String noteId) throws IOException { + Note note = gitHubNotebookRepo.get(TEST_NOTE_ID, null); + note.setInterpreterFactory(mock(InterpreterFactory.class)); + Paragraph paragraph = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + paragraph.setText("%md text"); + gitHubNotebookRepo.save(note, null); + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java index 2276c25dfe2..72ea4395416 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java @@ -70,9 +70,19 @@ public void setUp() throws Exception { String testNoteDir = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID); String testNoteDir2 = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID2); - FileUtils.copyDirectory(new File(Joiner.on(File.separator).join("src", "test", "resources", TEST_NOTE_ID)), - new File(testNoteDir)); - FileUtils.copyDirectory(new File(Joiner.on(File.separator).join("src", "test", "resources", TEST_NOTE_ID2)), + FileUtils.copyDirectory( + new File( + GitHubNotebookRepoTest.class.getResource( + Joiner.on(File.separator).join("", TEST_NOTE_ID) + ).getFile() + ), + new File(testNoteDir)); + FileUtils.copyDirectory( + new File( + GitHubNotebookRepoTest.class.getResource( + Joiner.on(File.separator).join("", TEST_NOTE_ID2) + ).getFile() + ), new File(testNoteDir2) ); From a13010f33cf7bb851e50a739a1dbd435a55be17d Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 13 Feb 2018 09:13:23 +0800 Subject: [PATCH 027/386] ZEPPELIN-3226. Fail to launch IPySparkInterpreter in embedded mode ### What is this PR for? Trivial PR for fixing this issue. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3226 ### How should this be tested? * CI Pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2793 from zjffdu/ZEPPELIN-3226 and squashes the following commits: 4f6668b [Jeff Zhang] ZEPPELIN-3226. Fail to launch IPySparkInterpreter in embedded mode --- .../java/org/apache/zeppelin/spark/IPySparkInterpreter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java index c7253fb40c9..37896f982a5 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java @@ -51,9 +51,10 @@ public void open() throws InterpreterException { PySparkInterpreter.getPythonExec(getProperties())); sparkInterpreter = getSparkInterpreter(); SparkConf conf = sparkInterpreter.getSparkContext().getConf(); - // only set PYTHONPATH in local or yarn-client mode. + // only set PYTHONPATH in embedded, local or yarn-client mode. // yarn-cluster will setup PYTHONPATH automatically. - if (!conf.get("spark.submit.deployMode").equals("cluster")) { + if (!conf.contains("spark.submit.deployMode") || + !conf.get("spark.submit.deployMode").equals("cluster")) { setAdditionalPythonPath(PythonUtils.sparkPythonPath()); setAddBulitinPy4j(false); } From 3418055cca28aa16459d254999f580fd2785f5cc Mon Sep 17 00:00:00 2001 From: "Cardenas, Jhon" Date: Mon, 12 Feb 2018 17:47:10 -0500 Subject: [PATCH 028/386] [ZEPPELIN-3228] Currently interpreter dependencies are not downloaded on zeppelin start - regression issue Currently interpreter dependencies are not downloaded on zeppelin start. This was solved in [ZEPPELIN-3228], but it is happening again. ### What is this PR for? When zeppelin is started/restarted, server should try and download interpreter dependencies. ### What type of PR is it? [Bug Fix] ### What is the Jira issue? * [ZEPPELIN-3228] ### How should this be tested? * Put a dependency (say "org.apache.commons:commons-csv:1.1") in any of the interpreter. * From command line delete local-repo directory * Restart zeppelin server Expectation is local-repo should be recreated with all the dependencies that were mentioned in any of the interpreters. ### Questions: * Does the licenses files need update? n/a * Is there breaking changes for older versions? n/a * Does this needs documentation? n/a Author: Cardenas, Jhon Closes #2792 from jhonderson/ZEPPELIN-1143 and squashes the following commits: 3fcefb2 [Cardenas, Jhon] [ZEPPELIN-1143] When zeppelin starts it does the interpreter dependencies loading. --- .../zeppelin/interpreter/InterpreterSettingManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index bda1be60a40..04d409289a3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -266,6 +266,12 @@ private void loadFromFile() throws IOException { this.interpreterRepositories.add(repo); } } + + // force interpreter dependencies loading once the + // repositories have been loaded. + for (InterpreterSetting setting : interpreterSettings.values()) { + setting.setDependencies(setting.getDependencies()); + } } } From e89f1027875fbcec6482f24c3e316025eb16ee8e Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Sat, 10 Feb 2018 14:46:03 +0100 Subject: [PATCH 029/386] ZEPPELIN-3153. Fixed Checkstyle errors and warnings in the file module ### What is this PR for? Fixed the Checkstyle errors and warnings in the file module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3153 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2787 from HorizonNet/ZEPPELIN-3153 and squashes the following commits: 9b2c6fb [Jan Hentschel] ZEPPELIN-3153. Fixed Checkstyle errors and warnings in the file module --- file/pom.xml | 7 + .../apache/zeppelin/file/FileInterpreter.java | 38 +- .../org/apache/zeppelin/file/HDFSCommand.java | 23 +- .../zeppelin/file/HDFSFileInterpreter.java | 97 +++-- .../file/HDFSFileInterpreterTest.java | 400 ++++++++++-------- 5 files changed, 312 insertions(+), 253 deletions(-) diff --git a/file/pom.xml b/file/pom.xml index e649991ecf6..ed0ef3fa934 100644 --- a/file/pom.xml +++ b/file/pom.xml @@ -91,6 +91,13 @@ maven-resources-plugin + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java b/file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java index cf836727345..eea5650f00d 100644 --- a/file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java +++ b/file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java @@ -18,6 +18,17 @@ package org.apache.zeppelin.file; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.StringTokenizer; + import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -27,11 +38,6 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; /** * File interpreter for Zeppelin. @@ -48,7 +54,7 @@ public FileInterpreter(Properties property) { } /** - * Handling the arguments of the command + * Handling the arguments of the command. */ public class CommandArgs { public String input = null; @@ -74,25 +80,25 @@ private void parseArg(String arg) { } public void parseArgs() { - if (input == null) + if (input == null) { return; + } StringTokenizer st = new StringTokenizer(input); if (st.hasMoreTokens()) { command = st.nextToken(); - while (st.hasMoreTokens()) + while (st.hasMoreTokens()) { parseArg(st.nextToken()); + } } } } // Functions that each file system implementation must override - public abstract String listAll(String path) throws InterpreterException; public abstract boolean isDirectory(String path); // Combine paths, takes care of arguments such as .. - protected String getNewPath(String argument){ Path arg = Paths.get(argument); Path ret = arg.isAbsolute() ? arg : Paths.get(currentDir, argument); @@ -100,7 +106,6 @@ protected String getNewPath(String argument){ } // Handle the command handling uniformly across all file systems - @Override public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) { logger.info("Run File command '" + cmd + "'"); @@ -114,18 +119,15 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr } // Simple parsing of the command - if (args.command.equals("cd")) { - String newPath = !args.args.isEmpty() ? getNewPath(args.args.get(0)) : currentDir; - if (!isDirectory(newPath)) + if (!isDirectory(newPath)) { return new InterpreterResult(Code.ERROR, Type.TEXT, newPath + ": No such directory"); + } currentDir = newPath; return new InterpreterResult(Code.SUCCESS, Type.TEXT, "OK"); - } else if (args.command.equals("ls")) { - String newPath = !args.args.isEmpty() ? getNewPath(args.args.get(0)) : currentDir; try { String results = listAll(newPath); @@ -136,13 +138,9 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr } } else if (args.command.equals("pwd")) { - return new InterpreterResult(Code.SUCCESS, Type.TEXT, currentDir); - } else { - return new InterpreterResult(Code.ERROR, Type.TEXT, "Unknown command"); - } } diff --git a/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java b/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java index a097b889838..6b3dc4be917 100644 --- a/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java +++ b/file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java @@ -18,21 +18,21 @@ package org.apache.zeppelin.file; -import java.net.URL; -import java.net.HttpURLConnection; +import org.slf4j.Logger; + import java.io.BufferedReader; import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; + import javax.ws.rs.core.UriBuilder; -import org.slf4j.Logger; /** - * Definition and HTTP invocation methods for all WebHDFS commands - * + * Definition and HTTP invocation methods for all WebHDFS commands. */ public class HDFSCommand { - /** - * Type of HTTP request + * Type of HTTP request. */ public enum HttpType { GET, @@ -40,7 +40,7 @@ public enum HttpType { } /** - * Definition of WebHDFS operator + * Definition of WebHDFS operator. */ public class Op { public String op; @@ -55,7 +55,7 @@ public Op(String op, HttpType cmd, int minArgs) { } /** - * Definition of argument to an operator + * Definition of argument to an operator. */ public class Arg { public String key; @@ -90,8 +90,7 @@ public String checkArgs(Op op, String path, Arg[] args) throws Exception { path == null || (op.minArgs > 0 && (args == null || - args.length != op.minArgs))) - { + args.length != op.minArgs))) { String a = ""; a = (op != null) ? a + op.op + "\n" : a; a = (path != null) ? a + path + "\n" : a; @@ -101,10 +100,8 @@ public String checkArgs(Op op, String path, Arg[] args) throws Exception { return null; } - // The operator that runs all commands public String runCommand(Op op, String path, Arg[] args) throws Exception { - // Check arguments String error = checkArgs(op, path, args); if (error != null) { diff --git a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java b/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java index d715ed93a8a..b27dcb626c5 100644 --- a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java +++ b/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java @@ -18,11 +18,17 @@ package org.apache.zeppelin.file; -import java.text.SimpleDateFormat; -import java.util.*; - import com.google.gson.Gson; + +import com.google.gson.annotations.SerializedName; import org.apache.commons.lang.StringUtils; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Properties; + import org.apache.zeppelin.completer.CompletionType; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -30,7 +36,6 @@ /** * HDFS implementation of File interpreter for Zeppelin. - * */ public class HDFSFileInterpreter extends FileInterpreter { static final String HDFS_URL = "hdfs.url"; @@ -55,7 +60,7 @@ public HDFSFileInterpreter(Properties property){ } /** - * Status of one file + * Status of one file. * * matches returned JSON */ @@ -73,6 +78,7 @@ public class OneFileStatus { public int replication; public int storagePolicy; public String type; + public String toString() { StringBuilder sb = new StringBuilder(); sb.append("\nAccessTime = ").append(accessTime); @@ -93,38 +99,41 @@ public String toString() { } /** - * Status of one file + * Status of one file. * * matches returned JSON */ public class SingleFileStatus { - public OneFileStatus FileStatus; + @SerializedName("FileStatus") + public OneFileStatus fileStatus; } /** - * Status of all files in a directory + * Status of all files in a directory. * * matches returned JSON */ public class MultiFileStatus { - public OneFileStatus[] FileStatus; + @SerializedName("FileStatus") + public OneFileStatus[] fileStatus; } /** - * Status of all files in a directory + * Status of all files in a directory. * * matches returned JSON */ public class AllFileStatus { - public MultiFileStatus FileStatuses; + @SerializedName("FileStatuses") + public MultiFileStatus fileStatuses; } // tests whether we're able to connect to HDFS - private void testConnection() { try { - if (isDirectory("/")) + if (isDirectory("/")) { logger.info("Successfully created WebHDFS connection"); + } } catch (Exception e) { logger.error("testConnection: Cannot open WebHDFS connection. Bad URL: " + "/", e); exceptionOnConnect = e; @@ -159,9 +168,11 @@ private String listPermission(OneFileStatus fs){ sb.append(((p & 0x1) == 0) ? '-' : 'x'); return sb.toString(); } + private String listDate(OneFileStatus fs) { return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(new Date(fs.modificationTime)); } + private String listOne(String path, OneFileStatus fs) { if (args.flags.contains(new Character('l'))) { StringBuilder sb = new StringBuilder(); @@ -183,7 +194,11 @@ private String listOne(String path, OneFileStatus fs) { private String humanReadableByteCount(long bytes) { int unit = 1024; - if (bytes < unit) return bytes + " B"; + + if (bytes < unit) { + return bytes + " B"; + } + int exp = (int) (Math.log(bytes) / Math.log(unit)); String pre = "KMGTPE".charAt(exp - 1) + ""; return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); @@ -194,7 +209,7 @@ public String listFile(String filePath) { String str = cmd.runCommand(cmd.getFileStatus, filePath, null); SingleFileStatus sfs = gson.fromJson(str, SingleFileStatus.class); if (sfs != null) { - return listOne(filePath, sfs.FileStatus); + return listOne(filePath, sfs.fileStatus); } } catch (Exception e) { logger.error("listFile: " + filePath, e); @@ -204,8 +219,10 @@ public String listFile(String filePath) { public String listAll(String path) throws InterpreterException { String all = ""; - if (exceptionOnConnect != null) + if (exceptionOnConnect != null) { return "Error connecting to provided endpoint."; + } + try { //see if directory. if (isDirectory(path)) { @@ -214,13 +231,12 @@ public String listAll(String path) throws InterpreterException { AllFileStatus allFiles = gson.fromJson(sfs, AllFileStatus.class); if (allFiles != null && - allFiles.FileStatuses != null && - allFiles.FileStatuses.FileStatus != null) - { - int length = cmd.maxLength < allFiles.FileStatuses.FileStatus.length ? cmd.maxLength : - allFiles.FileStatuses.FileStatus.length; + allFiles.fileStatuses != null && + allFiles.fileStatuses.fileStatus != null) { + int length = cmd.maxLength < allFiles.fileStatuses.fileStatus.length ? cmd.maxLength : + allFiles.fileStatuses.fileStatus.length; for (int index = 0; index < length; index++) { - OneFileStatus fs = allFiles.FileStatuses.FileStatus[index]; + OneFileStatus fs = allFiles.fileStatuses.fileStatus[index]; all = all + listOne(path, fs) + '\n'; } } @@ -237,13 +253,16 @@ public String listAll(String path) throws InterpreterException { public boolean isDirectory(String path) { boolean ret = false; - if (exceptionOnConnect != null) + if (exceptionOnConnect != null) { return ret; + } + try { String str = cmd.runCommand(cmd.getFileStatus, path, null); SingleFileStatus sfs = gson.fromJson(str, SingleFileStatus.class); - if (sfs != null) - return sfs.FileStatus.type.equals("DIRECTORY"); + if (sfs != null) { + return sfs.fileStatus.type.equals("DIRECTORY"); + } } catch (Exception e) { logger.error("IsDirectory: " + path, e); return false; @@ -251,7 +270,6 @@ public boolean isDirectory(String path) { return ret; } - @Override public List completion(String buf, int cursor, InterpreterContext interpreterContext) { @@ -266,17 +284,22 @@ public List completion(String buf, int cursor, //part of a command == no spaces if (buf.split(" ").length == 1){ - if ("cd".contains(buf)) suggestions.add(new InterpreterCompletion("cd", "cd", - CompletionType.command.name())); - if ("ls".contains(buf)) suggestions.add(new InterpreterCompletion("ls", "ls", - CompletionType.command.name())); - if ("pwd".contains(buf)) suggestions.add(new InterpreterCompletion("pwd", "pwd", - CompletionType.command.name())); + if ("cd".contains(buf)) { + suggestions.add(new InterpreterCompletion("cd", "cd", + CompletionType.command.name())); + } + if ("ls".contains(buf)) { + suggestions.add(new InterpreterCompletion("ls", "ls", + CompletionType.command.name())); + } + if ("pwd".contains(buf)) { + suggestions.add(new InterpreterCompletion("pwd", "pwd", + CompletionType.command.name())); + } return suggestions; } - // last word will contain the path we're working with. String lastToken = buf.substring(buf.lastIndexOf(" ") + 1); if (lastToken.startsWith("-")) { //flag not path @@ -298,12 +321,10 @@ public List completion(String buf, int cursor, AllFileStatus allFiles = gson.fromJson(fileStatusString, AllFileStatus.class); if (allFiles != null && - allFiles.FileStatuses != null && - allFiles.FileStatuses.FileStatus != null) - { - for (OneFileStatus fs : allFiles.FileStatuses.FileStatus) { + allFiles.fileStatuses != null && + allFiles.fileStatuses.fileStatus != null) { + for (OneFileStatus fs : allFiles.fileStatuses.fileStatus) { if (fs.pathSuffix.contains(unfinished)) { - //only suggest the text after the last . String beforeLastPeriod = unfinished.substring(0, unfinished.lastIndexOf('.') + 1); //beforeLastPeriod should be the start of fs.pathSuffix, so take the end of it. diff --git a/file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java b/file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java index adc9bd6b55f..aa698866f24 100644 --- a/file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java +++ b/file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java @@ -18,13 +18,12 @@ package org.apache.zeppelin.file; +import static org.junit.Assert.assertNull; + import com.google.gson.Gson; + import junit.framework.TestCase; -import static org.junit.Assert.*; -import org.apache.zeppelin.completer.CompletionType; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.junit.Test; import org.slf4j.Logger; @@ -32,223 +31,260 @@ import java.util.HashMap; import java.util.List; import java.util.Properties; -import java.lang.Override; -import java.lang.String; +import org.apache.zeppelin.completer.CompletionType; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; /** - * Tests Interpreter by running pre-determined commands against mock file system - * + * Tests Interpreter by running pre-determined commands against mock file system. */ public class HDFSFileInterpreterTest extends TestCase { + @Test + public void testMaxLength() { + HDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties()); + t.open(); + InterpreterResult result = t.interpret("ls -l /", null); + String lineSeparator = "\n"; + int fileStatusLength = MockFileSystem.FILE_STATUSES.split(lineSeparator).length; + assertEquals(result.message().get(0).getData().split(lineSeparator).length, fileStatusLength); + t.close(); + + Properties properties = new Properties(); + final int maxLength = fileStatusLength - 2; + properties.setProperty("hdfs.maxlength", String.valueOf(maxLength)); + HDFSFileInterpreter t1 = new MockHDFSFileInterpreter(properties); + t1.open(); + InterpreterResult result1 = t1.interpret("ls -l /", null); + assertEquals(result1.message().get(0).getData().split(lineSeparator).length, maxLength); + t1.close(); + } - @Test - public void testMaxLength() { - - HDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties()); - t.open(); - InterpreterResult result = t.interpret("ls -l /", null); - String lineSeparator = "\n"; - int fileStatusLength = MockFileSystem.fileStatuses.split(lineSeparator).length; - assertEquals(result.message().get(0).getData().split(lineSeparator).length, fileStatusLength); - t.close(); - - Properties properties = new Properties(); - final int maxLength = fileStatusLength - 2; - properties.setProperty("hdfs.maxlength", String.valueOf(maxLength)); - HDFSFileInterpreter t1 = new MockHDFSFileInterpreter(properties); - t1.open(); - InterpreterResult result1 = t1.interpret("ls -l /", null); - assertEquals(result1.message().get(0).getData().split(lineSeparator).length, maxLength); - t1.close(); - } - - @Test - public void test() { - HDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties()); - t.open(); - - // We have info for /, /user, /tmp, /mr-history/done - - // Ensure - // 1. ls -l works - // 2. paths (. and ..) are correctly handled - // 3. flags and arguments to commands are correctly handled + @Test + public void test() { + HDFSFileInterpreter t = new MockHDFSFileInterpreter(new Properties()); + t.open(); - InterpreterResult result1 = t.interpret("ls -l /", null); - assertEquals(result1.message().get(0).getType(), InterpreterResult.Type.TEXT); + // We have info for /, /user, /tmp, /mr-history/done - InterpreterResult result2 = t.interpret("ls -l /./user/..", null); - assertEquals(result2.message().get(0).getType(), InterpreterResult.Type.TEXT); + // Ensure + // 1. ls -l works + // 2. paths (. and ..) are correctly handled + // 3. flags and arguments to commands are correctly handled + InterpreterResult result1 = t.interpret("ls -l /", null); + assertEquals(result1.message().get(0).getType(), InterpreterResult.Type.TEXT); - assertEquals(result1.message().get(0).getData(), result2.message().get(0).getData()); + InterpreterResult result2 = t.interpret("ls -l /./user/..", null); + assertEquals(result2.message().get(0).getType(), InterpreterResult.Type.TEXT); - // Ensure you can do cd and after that the ls uses current directory correctly + assertEquals(result1.message().get(0).getData(), result2.message().get(0).getData()); - InterpreterResult result3 = t.interpret("cd user", null); - assertEquals(result3.message().get(0).getType(), InterpreterResult.Type.TEXT); - assertEquals(result3.message().get(0).getData(), "OK"); + // Ensure you can do cd and after that the ls uses current directory correctly + InterpreterResult result3 = t.interpret("cd user", null); + assertEquals(result3.message().get(0).getType(), InterpreterResult.Type.TEXT); + assertEquals(result3.message().get(0).getData(), "OK"); - InterpreterResult result4 = t.interpret("ls", null); - assertEquals(result4.message().get(0).getType(), InterpreterResult.Type.TEXT); + InterpreterResult result4 = t.interpret("ls", null); + assertEquals(result4.message().get(0).getType(), InterpreterResult.Type.TEXT); - InterpreterResult result5 = t.interpret("ls /user", null); - assertEquals(result5.message().get(0).getType(), InterpreterResult.Type.TEXT); + InterpreterResult result5 = t.interpret("ls /user", null); + assertEquals(result5.message().get(0).getType(), InterpreterResult.Type.TEXT); - assertEquals(result4.message().get(0).getData(), result5.message().get(0).getData()); + assertEquals(result4.message().get(0).getData(), result5.message().get(0).getData()); - // Ensure pwd works correctly + // Ensure pwd works correctly + InterpreterResult result6 = t.interpret("pwd", null); + assertEquals(result6.message().get(0).getType(), InterpreterResult.Type.TEXT); + assertEquals(result6.message().get(0).getData(), "/user"); - InterpreterResult result6 = t.interpret("pwd", null); - assertEquals(result6.message().get(0).getType(), InterpreterResult.Type.TEXT); - assertEquals(result6.message().get(0).getData(), "/user"); + // Move a couple of levels and check we're in the right place + InterpreterResult result7 = t.interpret("cd ../mr-history/done", null); + assertEquals(result7.message().get(0).getType(), InterpreterResult.Type.TEXT); + assertEquals(result7.message().get(0).getData(), "OK"); - // Move a couple of levels and check we're in the right place + InterpreterResult result8 = t.interpret("ls -l ", null); + assertEquals(result8.message().get(0).getType(), InterpreterResult.Type.TEXT); - InterpreterResult result7 = t.interpret("cd ../mr-history/done", null); - assertEquals(result7.message().get(0).getType(), InterpreterResult.Type.TEXT); - assertEquals(result7.message().get(0).getData(), "OK"); + InterpreterResult result9 = t.interpret("ls -l /mr-history/done", null); + assertEquals(result9.message().get(0).getType(), InterpreterResult.Type.TEXT); - InterpreterResult result8 = t.interpret("ls -l ", null); - assertEquals(result8.message().get(0).getType(), InterpreterResult.Type.TEXT); + assertEquals(result8.message().get(0).getData(), result9.message().get(0).getData()); - InterpreterResult result9 = t.interpret("ls -l /mr-history/done", null); - assertEquals(result9.message().get(0).getType(), InterpreterResult.Type.TEXT); + InterpreterResult result10 = t.interpret("cd ../..", null); + assertEquals(result10.message().get(0).getType(), InterpreterResult.Type.TEXT); + assertEquals(result7.message().get(0).getData(), "OK"); - assertEquals(result8.message().get(0).getData(), result9.message().get(0).getData()); + InterpreterResult result11 = t.interpret("ls -l ", null); + assertEquals(result11.message().get(0).getType(), InterpreterResult.Type.TEXT); - InterpreterResult result10 = t.interpret("cd ../..", null); - assertEquals(result10.message().get(0).getType(), InterpreterResult.Type.TEXT); - assertEquals(result7.message().get(0).getData(), "OK"); + // we should be back to first result after all this navigation + assertEquals(result1.message().get(0).getData(), result11.message().get(0).getData()); - InterpreterResult result11 = t.interpret("ls -l ", null); - assertEquals(result11.message().get(0).getType(), InterpreterResult.Type.TEXT); + // auto completion test + List expectedResultOne = Arrays.asList( + new InterpreterCompletion("ls", "ls", CompletionType.command.name())); + List expectedResultTwo = Arrays.asList( + new InterpreterCompletion("pwd", "pwd", CompletionType.command.name())); + List resultOne = t.completion("l", 0, null); + List resultTwo = t.completion("p", 0, null); - // we should be back to first result after all this navigation - assertEquals(result1.message().get(0).getData(), result11.message().get(0).getData()); + assertEquals(expectedResultOne, resultOne); + assertEquals(expectedResultTwo, resultTwo); - // auto completion test - List expectedResultOne = Arrays.asList( - new InterpreterCompletion("ls", "ls", CompletionType.command.name())); - List expectedResultTwo = Arrays.asList( - new InterpreterCompletion("pwd", "pwd", CompletionType.command.name())); - List resultOne = t.completion("l", 0, null); - List resultTwo = t.completion("p", 0, null); + t.close(); + } +} - assertEquals(expectedResultOne, resultOne); - assertEquals(expectedResultTwo, resultTwo); +/** + * Store command results from curl against a real file system. + */ +class MockFileSystem { + HashMap mfs = new HashMap<>(); + static final String FILE_STATUSES = + "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16389," + + "\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1438548219672," + + "\"owner\":\"yarn\",\"pathSuffix\":\"app-logs\",\"permission\":\"777\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + + "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16395," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548030045," + + "\"owner\":\"hdfs\",\"pathSuffix\":\"hdp\",\"permission\":\"755\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + + "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16390," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547985336," + + "\"owner\":\"mapred\",\"pathSuffix\":\"mapred\",\"permission\":\"755\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + + "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":2,\"fileId\":16392," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547985346," + + "\"owner\":\"hdfs\",\"pathSuffix\":\"mr-history\",\"permission\":\"755\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + + "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16400," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548089725," + + "\"owner\":\"hdfs\",\"pathSuffix\":\"system\",\"permission\":\"755\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + + "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16386," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548150089," + + "\"owner\":\"hdfs\",\"pathSuffix\":\"tmp\",\"permission\":\"777\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + + "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16387," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547921792," + + "\"owner\":\"hdfs\",\"pathSuffix\":\"user\",\"permission\":\"755\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n"; + + void addListStatusData() { + mfs.put("/?op=LISTSTATUS", + "{\"FileStatuses\":{\"FileStatus\":[\n" + FILE_STATUSES + + "]}}" + ); + mfs.put("/user?op=LISTSTATUS", "{\"FileStatuses\":{\"FileStatus\":[\n" + + " {\"accessTime\":0,\"blockSize\":0,\"childrenNum\":4,\"fileId\":16388," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253161263," + + "\"owner\":\"ambari-qa\",\"pathSuffix\":\"ambari-qa\",\"permission\":\"770\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n" + + " ]}}" + ); + mfs.put("/tmp?op=LISTSTATUS", + "{\"FileStatuses\":{\"FileStatus\":[\n" + + " {\"accessTime\":1441253097489,\"blockSize\":134217728,\"childrenNum\":0," + + "\"fileId\":16400,\"group\":\"hdfs\",\"length\":1645," + + "\"modificationTime\":1441253097517,\"owner\":\"hdfs\"," + + "\"pathSuffix\":\"ida8c06540_date040315\",\"permission\":\"755\"," + + "\"replication\":3,\"storagePolicy\":0,\"type\":\"FILE\"}\n" + + " ]}}" + ); + mfs.put("/mr-history/done?op=LISTSTATUS", + "{\"FileStatuses\":{\"FileStatus\":[\n" + + "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16433," + + "\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197481," + + "\"owner\":\"mapred\",\"pathSuffix\":\"2015\",\"permission\":\"770\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n" + + "]}}" + ); + } - t.close(); - } + void addGetFileStatusData() { + mfs.put("/?op=GETFILESTATUS", + "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":7,\"fileId\":16385," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548089725," + + "\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"755\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); + mfs.put("/user?op=GETFILESTATUS", + "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16387," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253043188," + + "\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"755\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); + mfs.put("/tmp?op=GETFILESTATUS", + "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16386," + + "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253097489," + + "\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"777\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); + mfs.put("/mr-history/done?op=GETFILESTATUS", + "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16393," + + "\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197480," + + "\"owner\":\"mapred\",\"pathSuffix\":\"\",\"permission\":\"777\"," + + "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); } - /** - * Store command results from curl against a real file system - */ - class MockFileSystem { - HashMap mfs = new HashMap<>(); - static final String fileStatuses = - "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16389,\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1438548219672,\"owner\":\"yarn\",\"pathSuffix\":\"app-logs\",\"permission\":\"777\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + - "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16395,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548030045,\"owner\":\"hdfs\",\"pathSuffix\":\"hdp\",\"permission\":\"755\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + - "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16390,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547985336,\"owner\":\"mapred\",\"pathSuffix\":\"mapred\",\"permission\":\"755\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + - "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":2,\"fileId\":16392,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547985346,\"owner\":\"hdfs\",\"pathSuffix\":\"mr-history\",\"permission\":\"755\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + - "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16400,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548089725,\"owner\":\"hdfs\",\"pathSuffix\":\"system\",\"permission\":\"755\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + - "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16386,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548150089,\"owner\":\"hdfs\",\"pathSuffix\":\"tmp\",\"permission\":\"777\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" + - "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16387,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547921792,\"owner\":\"hdfs\",\"pathSuffix\":\"user\",\"permission\":\"755\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n"; - void addListStatusData() { - mfs.put("/?op=LISTSTATUS", - "{\"FileStatuses\":{\"FileStatus\":[\n" + fileStatuses + - "]}}" - ); - mfs.put("/user?op=LISTSTATUS", - "{\"FileStatuses\":{\"FileStatus\":[\n" + - " {\"accessTime\":0,\"blockSize\":0,\"childrenNum\":4,\"fileId\":16388,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253161263,\"owner\":\"ambari-qa\",\"pathSuffix\":\"ambari-qa\",\"permission\":\"770\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n" + - " ]}}" - ); - mfs.put("/tmp?op=LISTSTATUS", - "{\"FileStatuses\":{\"FileStatus\":[\n" + - " {\"accessTime\":1441253097489,\"blockSize\":134217728,\"childrenNum\":0,\"fileId\":16400,\"group\":\"hdfs\",\"length\":1645,\"modificationTime\":1441253097517,\"owner\":\"hdfs\",\"pathSuffix\":\"ida8c06540_date040315\",\"permission\":\"755\",\"replication\":3,\"storagePolicy\":0,\"type\":\"FILE\"}\n" + - " ]}}" - ); - mfs.put("/mr-history/done?op=LISTSTATUS", - "{\"FileStatuses\":{\"FileStatus\":[\n" + - "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16433,\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197481,\"owner\":\"mapred\",\"pathSuffix\":\"2015\",\"permission\":\"770\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n" + - "]}}" - ); - } - void addGetFileStatusData() { - mfs.put("/?op=GETFILESTATUS", - "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":7,\"fileId\":16385,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548089725,\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"755\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); - mfs.put("/user?op=GETFILESTATUS", - "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16387,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253043188,\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"755\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); - mfs.put("/tmp?op=GETFILESTATUS", - "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16386,\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253097489,\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"777\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); - mfs.put("/mr-history/done?op=GETFILESTATUS", - "{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16393,\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197480,\"owner\":\"mapred\",\"pathSuffix\":\"\",\"permission\":\"777\",\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}"); - } - public void addMockData(HDFSCommand.Op op) { - if (op.op.equals("LISTSTATUS")) { - addListStatusData(); - } else if (op.op.equals("GETFILESTATUS")) { - addGetFileStatusData(); - } - // do nothing - } - public String get(String key) { - return mfs.get(key); + public void addMockData(HDFSCommand.Op op) { + if (op.op.equals("LISTSTATUS")) { + addListStatusData(); + } else if (op.op.equals("GETFILESTATUS")) { + addGetFileStatusData(); } + // do nothing } - /** - * Run commands against mock file system that simulates webhdfs responses - */ - class MockHDFSCommand extends HDFSCommand { - MockFileSystem fs = null; - - public MockHDFSCommand(String url, String user, Logger logger, int maxLength) { - super(url, user, logger, maxLength); - fs = new MockFileSystem(); - fs.addMockData(getFileStatus); - fs.addMockData(listStatus); - } + public String get(String key) { + return mfs.get(key); + } +} - public MockHDFSCommand(String url, String user, Logger logger) { - this(url, user, logger, 1000); - } +/** + * Run commands against mock file system that simulates webhdfs responses. + */ +class MockHDFSCommand extends HDFSCommand { + MockFileSystem fs = null; + + MockHDFSCommand(String url, String user, Logger logger, int maxLength) { + super(url, user, logger, maxLength); + fs = new MockFileSystem(); + fs.addMockData(getFileStatus); + fs.addMockData(listStatus); + } - @Override - public String runCommand(Op op, String path, Arg[] args) throws Exception { + MockHDFSCommand(String url, String user, Logger logger) { + this(url, user, logger, 1000); + } - String error = checkArgs(op, path, args); - assertNull(error); + @Override + public String runCommand(Op op, String path, Arg[] args) throws Exception { + String error = checkArgs(op, path, args); + assertNull(error); - String c = path + "?op=" + op.op; + String c = path + "?op=" + op.op; - if (args != null) { - for (Arg a : args) { - c += "&" + a.key + "=" + a.value; - } + if (args != null) { + for (Arg a : args) { + c += "&" + a.key + "=" + a.value; } - return fs.get(c); } + return fs.get(c); } +} - /** - * Mock Interpreter - uses Mock HDFS command - */ - class MockHDFSFileInterpreter extends HDFSFileInterpreter { - - @Override - public void prepare() { - // Run commands against mock File System instead of WebHDFS - int i = Integer.parseInt(getProperty(HDFS_MAXLENGTH) == null ? "1000" - : getProperty(HDFS_MAXLENGTH)); - cmd = new MockHDFSCommand("", "", logger, i); - gson = new Gson(); - } - - public MockHDFSFileInterpreter(Properties property) { - super(property); - } +/** + * Mock Interpreter - uses Mock HDFS command. + */ +class MockHDFSFileInterpreter extends HDFSFileInterpreter { + @Override + public void prepare() { + // Run commands against mock File System instead of WebHDFS + int i = Integer.parseInt(getProperty(HDFS_MAXLENGTH) == null ? "1000" + : getProperty(HDFS_MAXLENGTH)); + cmd = new MockHDFSCommand("", "", logger, i); + gson = new Gson(); + } -} \ No newline at end of file + MockHDFSFileInterpreter(Properties property) { + super(property); + } +} From d4783040c4779b99c5559c908a827c61e4b10608 Mon Sep 17 00:00:00 2001 From: Savalek Date: Fri, 2 Feb 2018 12:24:33 +0300 Subject: [PATCH 030/386] [ZEPPELIN-3204] FIX: cursor in paragraph editor jumps ### What is this PR for? Sometimes when a user enters text in the paragraph field the lower part of the paragraph starts to jump. This PR fixes this. ### What type of PR is it? [Bug Fix] ### What is the Jira issue? [ZEPPELIN-3204](https://issues.apache.org/jira/browse/ZEPPELIN-3204) ### Screenshots ![gif](https://user-images.githubusercontent.com/30798933/35732168-812f4274-0829-11e8-9fd6-45c2665f9646.gif) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Savalek Closes #2762 from Savalek/ZEPPELIN-3131 and squashes the following commits: 0521e25 [Savalek] FIX: cursor in paragraph editor jumps --- .../src/app/notebook/paragraph/paragraph.controller.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index fb99e636ceb..75a0fecac3b 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -1028,8 +1028,7 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca const autoAdjustEditorHeight = function (editor) { let height = editor.getSession().getScreenLength() * - editor.renderer.lineHeight + - editor.renderer.scrollBar.getWidth() + editor.renderer.lineHeight angular.element('#' + editor.container.id).height(height.toString() + 'px') editor.resize() From 91b5d69be2aa8f72dc49d27800a90f8bed9781cc Mon Sep 17 00:00:00 2001 From: Karthik Palaniappan Date: Sun, 28 Jan 2018 19:21:34 -0800 Subject: [PATCH 031/386] [ZEPPELIN-3182] Support saving notebooks to Google Cloud Storage ### What is this PR for? Support saving notebooks to Google Cloud Storage, similar to implementations for S3 and Azure. It uses the same authentication mechanisms as the BigQuery interpreter. I am new to Maven, so please check my work on the pom.xml files. In particular, I upgraded Guava to 23.0, which was required for `google-cloud-java`. Going through hello-world with my changes seems to work. Also, I modified the BigQuery interpreter docs to point to the **latest** GCS storage docs. Is it more appropriate to pin to the version you are viewing? How can I do that? ### What type of PR is it? Improvement ### Todos * [Low priority] Support encryption keys I don't this is particularly important, at least for v1. ### How should this be tested? * I added unit tests for the core functionality * I manually tested the authentication instructions (but that could use a second pair of eyes) ### Questions: * Does the licenses files need update? * No idea. `google-cloud-java` is Apache 2: https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/LICENSE * Is there breaking changes for older versions? * Nope. * Does this needs documentation? * Yes, and I tried to update the docs (but there are likely other things that need to be updated) Author: Karthik Palaniappan Closes #2738 from karth295/master and squashes the following commits: c4a45b7 [Karthik Palaniappan] [ZEPPELIN-3182] Support saving notebooks to Google Cloud Storage 8dc819e [Karthik Palaniappan] Unify logic to clear notebook runtime state on load from storage --- LICENSE | 5 +- conf/zeppelin-env.sh.template | 6 + conf/zeppelin-site.xml.template | 17 ++ docs/index.md | 1 + docs/interpreter/bigquery.md | 20 +- docs/setup/storage/storage.md | 92 +++++++ .../zeppelin/conf/ZeppelinConfiguration.java | 11 +- zeppelin-zengine/pom.xml | 103 +++++++- .../org/apache/zeppelin/notebook/Note.java | 13 + .../notebook/repo/AzureNotebookRepo.java | 28 +-- .../notebook/repo/GCSNotebookRepo.java | 234 +++++++++++++++++ .../notebook/repo/MongoNotebookRepo.java | 42 +--- .../notebook/repo/S3NotebookRepo.java | 17 +- .../notebook/repo/VFSNotebookRepo.java | 27 +- .../notebook/repo/GCSNotebookRepoTest.java | 235 ++++++++++++++++++ 15 files changed, 744 insertions(+), 107 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java diff --git a/LICENSE b/LICENSE index 2252d6567c6..3b340531246 100644 --- a/LICENSE +++ b/LICENSE @@ -259,6 +259,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version (Apache 2.0) Gson extra (https://github.com/DanySK/gson-extras) (Apache 2.0) Nimbus JOSE+JWT (https://bitbucket.org/connect2id/nimbus-jose-jwt/wiki/Home) (Apache 2.0) jarchivelib (https://github.com/thrau/jarchivelib) + (Apache 2.0) Google Cloud Client Library for Java (https://github.com/GoogleCloudPlatform/google-cloud-java) ======================================================================== BSD 3-Clause licenses @@ -274,6 +275,8 @@ The following components are provided under the BSD 3-Clause license. See file (BSD 3 Clause) diff.js (https://github.com/kpdecker/jsdiff) + (BSD 3-Clause) Google Auth Library for Java (https://github.com/google/google-auth-library-java) + ======================================================================== BSD 2-Clause licenses ======================================================================== @@ -287,4 +290,4 @@ Jython Software License ======================================================================== The following components are provided under the Jython Software License. See file headers and project links for details. - (Jython Software License) jython-standalone - http://www.jython.org/ \ No newline at end of file + (Jython Software License) jython-standalone - http://www.jython.org/ diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 7bc38d633e1..c7204bd2187 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -30,16 +30,22 @@ # export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved # export ZEPPELIN_NOTEBOOK_HOMESCREEN # Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z # export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from list when this value set to "true". default "false" + # export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved # export ZEPPELIN_NOTEBOOK_S3_ENDPOINT # Endpoint of the bucket # export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json # export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID # AWS KMS key ID # export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION # AWS KMS key region # export ZEPPELIN_NOTEBOOK_S3_SSE # Server-side encryption enabled for notebooks + +# export ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR # GCS "directory" (prefix) under which notebooks are saved. E.g. gs://example-bucket/path/to/dir +# export GOOGLE_APPLICATION_CREDENTIALS # Provide a service account key file for GCS and BigQuery API calls (overrides application default credentials) + # export ZEPPELIN_NOTEBOOK_MONGO_URI # MongoDB connection URI used to connect to a MongoDB database server. Default "mongodb://localhost" # export ZEPPELIN_NOTEBOOK_MONGO_DATABASE # Database name to store notebook. Default "zeppelin" # export ZEPPELIN_NOTEBOOK_MONGO_COLLECTION # Collection name to store notebook. Default "notes" # export ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT # If "true" import local notes under ZEPPELIN_NOTEBOOK_DIR on startup. Default "false" + # export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default. # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 9e9898bb4db..9774f0d7c5c 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -67,6 +67,23 @@ hide homescreen notebook from list when this value set to true + + diff --git a/docs/index.md b/docs/index.md index 587ae93ed58..3d42735d092 100644 --- a/docs/index.md +++ b/docs/index.md @@ -104,6 +104,7 @@ limitations under the License. * [Git Storage](./setup/storage/storage.html#notebook-storage-in-local-git-repository) * [S3 Storage](./setup/storage/storage.html#notebook-storage-in-s3) * [Azure Storage](./setup/storage/storage.html#notebook-storage-in-azure) + * [Google Cloud Storage](./setup/storage/storage.html#notebook-storage-in-gcs) * [ZeppelinHub Storage](./setup/storage/storage.html#notebook-storage-in-zeppelinhub) * [MongoDB Storage](./setup/storage/storage.html#notebook-storage-in-mongodb) * Operation diff --git a/docs/interpreter/bigquery.md b/docs/interpreter/bigquery.md index 7ebe2e2fda8..1b90f99357a 100644 --- a/docs/interpreter/bigquery.md +++ b/docs/interpreter/bigquery.md @@ -58,20 +58,12 @@ Zeppelin is built against BigQuery API version v2-rev265-1.21.0 - [API Javadocs] In a notebook, to enable the **BigQuery** interpreter, click the **Gear** icon and select **bigquery**. -### Setup service account credentials - -In order to run BigQuery interpreter outside of Google Cloud Engine you need to provide authentication credentials, -by [following this instructions](https://developers.google.com/identity/protocols/application-default-credentials): - - - Go to the [API Console Credentials page](https://console.developers.google.com/project/_/apis/credentials) - - From the project drop-down, select your project. - - On the `Credentials` page, select the `Create credentials` drop-down, then select `Service account key`. - - From the Service account drop-down, select an existing service account or create a new one. - - For `Key type`, select the `JSON` key option, then select `Create`. The file automatically downloads to your computer. - - Put the `*.json` file you just downloaded in a directory of your choosing. This directory must be private (you can't let anyone get access to this), but accessible to your Zeppelin instance. - - Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the path of the JSON file downloaded. - * either though GUI: in interpreter configuration page property names in CAPITAL_CASE set up env vars - * or though `zeppelin-env.sh`: just add it to the end of the file. +### Provide Application Default Credentials + +Within Google Cloud Platform (e.g. Google App Engine, Google Compute Engine), +built-in credentials are used by default. + +Outside of GCP, follow the Google API authentication instructions for [Zeppelin Google Cloud Storage](https://zeppelin.apache.org/docs/latest/storage/storage.html#notebook-storage-in-gcs) ## Using the BigQuery Interpreter diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md index f34fc2cfc65..6f2ace43de5 100644 --- a/docs/setup/storage/storage.md +++ b/docs/setup/storage/storage.md @@ -33,6 +33,7 @@ There are few notebook storage systems available for a use out of the box: * all notes are saved in the notebook folder in hadoop compatible file system - `FileSystemNotebookRepo` * storage using Amazon S3 service - `S3NotebookRepo` * storage using Azure service - `AzureNotebookRepo` + * storage using Google Cloud Storage - `GCSNotebookRepo` * storage using MongoDB - `MongoNotebookRepo` * storage using GitHub - `GitHubNotebookRepo` @@ -263,6 +264,97 @@ Optionally, you can specify Azure folder structure name in the file **zeppelin-s ``` +
+## Notebook Storage in Google Cloud Storage + +Using `GCSNotebookRepo` you can connect Zeppelin with Google Cloud Storage using [Application Default Credentials](https://cloud.google.com/docs/authentication/production). + +First, choose a GCS path under which to store notebooks. + +``` + + zeppelin.notebook.gcs.dir + + + A GCS path in the form gs://bucketname/path/to/dir. + Notes are stored at {zeppelin.notebook.gcs.dir}/{notebook-id}/note.json + + +``` + +Then, initialize the `GCSNotebookRepo` class in the file **zeppelin-site.xml** by commenting the next property: + +``` + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.GitNotebookRepo + versioned notebook persistence layer implementation + +``` + +and commenting out: + +``` + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.GCSNotebookRepo + notebook persistence layer implementation + +``` + +Or, if you want to simultaneously use your local git storage with GCS, use the following property instead: + + ``` + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.GitNotebookRepo,org.apache.zeppelin.notebook.repo.GCSNotebookRepo + notebook persistence layer implementation + +``` + +### Google Cloud API Authentication + +Note: On Google App Engine, Google Cloud Shell, and Google Compute Engine, these +steps are not necessary, as build-in credentials are used by default. + +For more information, see [Application Default Credentials](https://cloud.google.com/docs/authentication/production) + +#### Using gcloud auth application-default login + +See the [gcloud docs](https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login) + +As the user running the zeppelin daemon, run: + +```bash +gcloud auth application-default login +``` + +You can also use `--scopes` to restrict access to specific Google APIs, such as +Cloud Storage and BigQuery. + +#### Using service account key files + +Alternatively, to use a [service account](https://cloud.google.com/compute/docs/access/service-accounts) +for authentication with GCS, you will need a JSON service account key file. + +1. Navigate to the [service accounts page](https://console.cloud.google.com/iam-admin/serviceaccounts/project) +2. Click `CREATE SERVICE ACCOUNT` +3. Select at least `Storage -> Storage Object Admin`. Note that this is + **different** than `Storage Admin`. +4. If you are also using the BigQuery Interpreter, add the appropriate + permissions (e.g. `Bigquery -> Bigquery Data Viewer and BigQuery User`) +5. Name your service account, and select "Furnish a new private key" to download + a `.json` file. Click "Create". +6. Move the downloaded file to a location of your choice (e.g. + `/path/to/my/key.json`), and give it appropriate permissions. Ensure at + least the user running the zeppelin daemon can read it. + +Then, point `GOOGLE_APPLICATION_CREDENTIALS` at your new key file in **zeppelin-env.sh**. For example: + +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/my/key.json +``` +
## Notebook Storage in ZeppelinHub diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index f7b3d7b09f5..5beb2c70e8e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -360,15 +360,19 @@ public boolean isRecoveryEnabled() { "org.apache.zeppelin.interpreter.recovery.NullRecoveryStorage"); } - public String getUser() { + public String getGCSStorageDir() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR); + } + + public String getS3User() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_USER); } - public String getBucketName() { + public String getS3BucketName() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_BUCKET); } - public String getEndpoint() { + public String getS3Endpoint() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_ENDPOINT); } @@ -697,6 +701,7 @@ public enum ConfVars { ZEPPELIN_NOTEBOOK_HOMESCREEN("zeppelin.notebook.homescreen", null), // whether homescreen notebook will be hidden from notebook list or not ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE("zeppelin.notebook.homescreen.hide", false), + ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR("zeppelin.notebook.gcs.dir", ""), ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"), ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"), ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"), diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index a864fdf122c..81ce7162b1a 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -39,6 +39,7 @@ 2.7.3 3.4 2.0 + 1.14.0 1.10.62 2.1.4 1.5.2 @@ -51,6 +52,7 @@ 0.27 + 0.32.0-alpha @@ -114,10 +116,84 @@ httpasyncclient + + com.google.cloud + google-cloud-storage + ${gcs.storage.version} + + + com.google.code.findbugs + jsr305 + + + com.google.protobuf + protobuf-java + + + com.google.guava + guava + + + com.google.api + api-common + + + com.google.http-client + google-http-client-jackson2 + + + com.google.http-client + google-http-client + + + org.codehaus.jackson + jackson-core-asl + + + + + + com.google.api + api-common + 1.2.0 + + + com.google.guava + guava + + + com.google.code.findbugs + jsr305 + + + + + + com.google.http-client + google-http-client-jackson2 + 1.23.0 + + + com.fasterxml.jackson.core + jackson-core + + + com.google.code.findbugs + jsr305 + + + + com.amazonaws aws-java-sdk-s3 ${aws.sdk.s3.version} + + + joda-time + joda-time + + @@ -146,7 +222,7 @@ com.google.guava guava - 15.0 + 20.0 @@ -227,6 +303,23 @@ test + + com.google.cloud + google-cloud-nio + ${google.testing.nio.version} + test + + + com.google.code.findbugs + jsr305 + + + com.google.guava + guava + + + + com.google.truth truth @@ -611,6 +704,10 @@ com.google.protobuf protobuf-java + + com.google.protobuf + protobuf-java-util + com.google.guava guava @@ -623,6 +720,10 @@ io.grpc grpc-context + + com.google.api.grpc + proto-google-common-protos + diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 281c4dec034..0a8fb12bba5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -963,6 +963,19 @@ public void postProcessParagraphs() { for (Paragraph p : paragraphs) { p.clearRuntimeInfos(); p.parseText(); + + if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { + p.setStatus(Status.ABORT); + } + + List appStates = p.getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + if (app.getStatus() != ApplicationState.Status.ERROR) { + app.setStatus(ApplicationState.Status.UNLOADED); + } + } + } } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java index de337faf169..731a3e8763e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java @@ -17,6 +17,13 @@ package org.apache.zeppelin.notebook.repo; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.file.CloudFile; +import com.microsoft.azure.storage.file.CloudFileClient; +import com.microsoft.azure.storage.file.CloudFileDirectory; +import com.microsoft.azure.storage.file.CloudFileShare; +import com.microsoft.azure.storage.file.ListFileItem; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -28,26 +35,15 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; -import org.apache.zeppelin.notebook.Paragraph; -import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.microsoft.azure.storage.CloudStorageAccount; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.file.CloudFile; -import com.microsoft.azure.storage.file.CloudFileClient; -import com.microsoft.azure.storage.file.CloudFileDirectory; -import com.microsoft.azure.storage.file.CloudFileShare; -import com.microsoft.azure.storage.file.ListFileItem; - /** * Azure storage backend for notebooks */ @@ -128,15 +124,7 @@ private Note getNote(String noteId) throws IOException { String json = IOUtils.toString(ins, conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_ENCODING)); ins.close(); - Note note = Note.fromJson(json); - - for (Paragraph p : note.getParagraphs()) { - if (p.getStatus() == Job.Status.PENDING || p.getStatus() == Job.Status.RUNNING) { - p.setStatus(Job.Status.ABORT); - } - } - - return note; + return Note.fromJson(json); } @Override diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java new file mode 100644 index 00000000000..591c532e031 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobListOption; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.gson.JsonParseException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A NotebookRepo implementation for storing notebooks in Google Cloud Storage. + * + * Notes are stored in the GCS "directory" specified by zeppelin.notebook.gcs.dir. This path + * must be in the form gs://bucketName/path/to/Dir. The bucket must already exist. N.B: GCS is an + * object store, so this "directory" should not itself be an object. Instead, it represents the base + * path for the note.json files. + * + * Authentication is provided by google-auth-library-java. + * @see + * google-auth-library-java. + */ +public class GCSNotebookRepo implements NotebookRepo { + + private static final Logger LOG = LoggerFactory.getLogger(GCSNotebookRepo.class); + private String encoding; + private String bucketName; + private Optional basePath; + private Pattern noteNamePattern; + private Storage storage; + + public GCSNotebookRepo(ZeppelinConfiguration conf) throws IOException { + this(conf, StorageOptions.getDefaultInstance().getService()); + } + + // For tests to use an in-memory storage implementation + GCSNotebookRepo(ZeppelinConfiguration conf, Storage storage) throws IOException { + this.encoding = conf.getString(ConfVars.ZEPPELIN_ENCODING); + + String gcsStorageDir = conf.getGCSStorageDir(); + if (gcsStorageDir.isEmpty()) { + throw new IOException("GCS storage directory must be set using 'zeppelin.notebook.gcs.dir'"); + } + if (!gcsStorageDir.startsWith("gs://")) { + throw new IOException(String.format( + "GCS storage directory '%s' must start with 'gs://'.", gcsStorageDir)); + } + String storageDirWithoutScheme = gcsStorageDir.substring("gs://".length()); + + // pathComponents excludes empty string if trailing slash is present + List pathComponents = Arrays.asList(storageDirWithoutScheme.split("/")); + if (pathComponents.size() < 1) { + throw new IOException(String.format( + "GCS storage directory '%s' must be in the form gs://bucketname/path/to/dir", + gcsStorageDir)); + } + this.bucketName = pathComponents.get(0); + if (pathComponents.size() > 1) { + this.basePath = Optional.of(StringUtils.join( + pathComponents.subList(1, pathComponents.size()), "/")); + } else { + this.basePath = Optional.absent(); + } + + // Notes are stored at gs://bucketName/basePath//note.json + if (basePath.isPresent()) { + this.noteNamePattern = Pattern.compile( + "^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$"); + } else { + this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$"); + } + + this.storage = storage; + } + + private BlobId makeBlobId(String noteId) { + if (basePath.isPresent()) { + return BlobId.of(bucketName, basePath.get() + "/" + noteId + "/note.json"); + } else { + return BlobId.of(bucketName, noteId + "/note.json"); + } + } + + @Override + public List list(AuthenticationInfo subject) throws IOException { + try { + List infos = new ArrayList<>(); + Iterable blobsUnderDir; + if (basePath.isPresent()) { + blobsUnderDir = storage + .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/")) + .iterateAll(); + } else { + blobsUnderDir = storage + .list(bucketName) + .iterateAll(); + } + for (Blob b : blobsUnderDir) { + Matcher matcher = noteNamePattern.matcher(b.getName()); + if (matcher.matches()) { + // Callers only use the id field, so do not fetch each note + // This matches the implementation in FileSystemNoteRepo#list + infos.add(new NoteInfo(matcher.group(1), "", null)); + } + } + return infos; + } catch (StorageException se) { + throw new IOException("Could not list GCS directory: " + se.getMessage(), se); + } + } + + @Override + public Note get(String noteId, AuthenticationInfo subject) throws IOException { + BlobId blobId = makeBlobId(noteId); + byte[] contents; + try { + contents = storage.readAllBytes(blobId); + } catch (StorageException se) { + throw new IOException("Could not read " + blobId.toString() + ": " + se.getMessage(), se); + } + + try { + return Note.fromJson(new String(contents, encoding)); + } catch (JsonParseException jpe) { + throw new IOException( + "Could note parse as json " + blobId.toString() + jpe.getMessage(), jpe); + } + } + + @Override + public void save(Note note, AuthenticationInfo subject) throws IOException { + BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) + .setContentType("application/json") + .build(); + try { + storage.create(info, note.toJson().getBytes("UTF-8")); + } catch (StorageException se) { + throw new IOException("Could not write " + info.toString() + ": " + se.getMessage(), se); + } + } + + @Override + public void remove(String noteId, AuthenticationInfo subject) throws IOException { + Preconditions.checkArgument(!Strings.isNullOrEmpty(noteId)); + BlobId blobId = makeBlobId(noteId); + try { + boolean deleted = storage.delete(blobId); + if (!deleted) { + throw new IOException("Tried to remove nonexistent blob " + blobId.toString()); + } + } catch (StorageException se) { + throw new IOException("Could not remove " + blobId.toString() + ": " + se.getMessage(), se); + } + } + + @Override + public void close() { + //no-op + } + + @Override + public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject) + throws IOException { + LOG.warn("checkpoint is not implemented for GCSNotebookRepo"); + return null; + } + + @Override + public Note get(String noteId, String revId, AuthenticationInfo subject) throws IOException { + LOG.warn("get revId is not implemented for GCSNotebookRepo"); + return null; + } + + @Override + public List revisionHistory(String noteId, AuthenticationInfo subject) { + LOG.warn("revisionHistory is not implemented for GCSNotebookRepo"); + return Collections.emptyList(); + } + + @Override + public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject) + throws IOException { + LOG.warn("setNoteRevision is not implemented for GCSNotebookRepo"); + return null; + } + + @Override + public List getSettings(AuthenticationInfo subject) { + LOG.warn("getSettings is not implemented for GCSNotebookRepo"); + return Collections.emptyList(); + } + + @Override + public void updateSettings(Map settings, AuthenticationInfo subject) { + LOG.warn("updateSettings is not implemented for GCSNotebookRepo"); + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java index 273d75d4a87..376a98695ee 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java @@ -1,5 +1,9 @@ package org.apache.zeppelin.notebook.repo; +import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.client.model.Filters.in; +import static com.mongodb.client.model.Filters.type; + import com.mongodb.MongoBulkWriteException; import com.mongodb.MongoClient; import com.mongodb.MongoClientURI; @@ -7,18 +11,18 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; -import static com.mongodb.client.model.Filters.eq; -import static com.mongodb.client.model.Filters.type; -import static com.mongodb.client.model.Filters.in; - import com.mongodb.client.model.InsertManyOptions; import com.mongodb.client.model.UpdateOptions; +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; -import org.apache.zeppelin.notebook.Paragraph; -import org.apache.zeppelin.notebook.ApplicationState; -import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.user.AuthenticationInfo; import org.bson.BsonType; import org.bson.Document; @@ -26,11 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - /** * Backend for storing Notebook on MongoDB */ @@ -161,24 +160,7 @@ private Note documentToNote(Document doc) { // document to JSON String json = doc.toJson(); // JSON to note - Note note = Note.fromJson(json); - - for (Paragraph p : note.getParagraphs()) { - if (p.getStatus() == Job.Status.PENDING || p.getStatus() == Job.Status.RUNNING) { - p.setStatus(Job.Status.ABORT); - } - - List appStates = p.getAllApplicationStates(); - if (appStates != null) { - for (ApplicationState app : appStates) { - if (app.getStatus() != ApplicationState.Status.ERROR) { - app.setStatus(ApplicationState.Status.UNLOADED); - } - } - } - } - - return note; + return Note.fromJson(json); } /** diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index 8828985e5e2..7d647024dd8 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -90,8 +90,8 @@ public class S3NotebookRepo implements NotebookRepo { public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException { this.conf = conf; - bucketName = conf.getBucketName(); - user = conf.getUser(); + bucketName = conf.getS3BucketName(); + user = conf.getS3User(); useServerSideEncryption = conf.isS3ServerSideEncryption(); // always use the default provider chain @@ -123,7 +123,7 @@ else if (conf.getS3EncryptionMaterialsProviderClass() != null) { } // set S3 endpoint to use - s3client.setEndpoint(conf.getEndpoint()); + s3client.setEndpoint(conf.getS3Endpoint()); } /** @@ -205,19 +205,10 @@ private Note getNote(String key) throws IOException { throw new IOException("Unable to retrieve object from S3: " + ace, ace); } - Note note; try (InputStream ins = s3object.getObjectContent()) { String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); - note = Note.fromJson(json); + return Note.fromJson(json); } - - for (Paragraph p : note.getParagraphs()) { - if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { - p.setStatus(Status.ABORT); - } - } - - return note; } private NoteInfo getNoteInfo(String key) throws IOException { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 63395f9a771..481ea3dcd0f 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.notebook.repo; +import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -24,11 +25,9 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.Collections; -import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.vfs2.FileContent; @@ -40,17 +39,12 @@ import org.apache.commons.vfs2.VFS; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.notebook.ApplicationState; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; -import org.apache.zeppelin.notebook.Paragraph; -import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Lists; - /** * */ @@ -167,24 +161,7 @@ private Note getNote(FileObject noteDir) throws IOException { String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); ins.close(); - Note note = Note.fromJson(json); - - for (Paragraph p : note.getParagraphs()) { - if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { - p.setStatus(Status.ABORT); - } - - List appStates = p.getAllApplicationStates(); - if (appStates != null) { - for (ApplicationState app : appStates) { - if (app.getStatus() != ApplicationState.Status.ERROR) { - app.setStatus(ApplicationState.Status.UNLOADED); - } - } - } - } - - return note; + return Note.fromJson(json); } private NoteInfo getNoteInfo(FileObject noteDir) throws IOException { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java new file mode 100644 index 00000000000..c1fae67da15 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.fail; + +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class GCSNotebookRepoTest { + private static final AuthenticationInfo AUTH_INFO = AuthenticationInfo.ANONYMOUS; + + private GCSNotebookRepo notebookRepo; + private Storage storage; + + @Parameters + public static Collection data() { + return Arrays.asList(new Object[][] { + { "bucketname", Optional.absent(), "gs://bucketname" }, + { "bucketname-with-slash", Optional.absent(), "gs://bucketname-with-slash/" }, + { "bucketname", Optional.of("path/to/dir"), "gs://bucketname/path/to/dir" }, + { "bucketname", Optional.of("trailing/slash"), "gs://bucketname/trailing/slash/" } + }); + } + + @Parameter(0) + public String bucketName; + + @Parameter(1) + public Optional basePath; + + @Parameter(2) + public String uriPath; + + private Note runningNote; + + @Before + public void setUp() throws Exception { + this.runningNote = makeRunningNote(); + + this.storage = LocalStorageHelper.getOptions().getService(); + + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), uriPath); + this.notebookRepo = new GCSNotebookRepo(new ZeppelinConfiguration(), storage); + } + + private static Note makeRunningNote() { + Note note = new Note(); + note.setConfig(ImmutableMap.of("key", "value")); + + Paragraph p = new Paragraph(note, null, null); + p.setText("text"); + p.setStatus(Status.RUNNING); + note.addParagraph(p); + return note; + } + + @Test + public void testList_nonexistent() throws Exception { + assertThat(notebookRepo.list(AUTH_INFO)).isEmpty(); + } + + @Test + public void testList() throws Exception { + createAt(runningNote, "note.json"); + createAt(runningNote, "/note.json"); + createAt(runningNote, "validid/note.json"); + createAt(runningNote, "validid-2/note.json"); + createAt(runningNote, "cannot-be-dir/note.json/foo"); + createAt(runningNote, "cannot/be/nested/note.json"); + + List infos = notebookRepo.list(AUTH_INFO); + List noteIds = new ArrayList<>(); + for (NoteInfo info : infos) { + noteIds.add(info.getId()); + } + // Only valid paths are gs://bucketname/path//note.json + assertThat(noteIds).containsExactlyElementsIn(ImmutableList.of("validid", "validid-2")); + } + + @Test + public void testGet_nonexistent() throws Exception { + try { + notebookRepo.get("id", AUTH_INFO); + fail(); + } catch (IOException e) {} + } + + @Test + public void testGet() throws Exception { + create(runningNote); + + // Status of saved running note is removed in get() + Note got = notebookRepo.get(runningNote.getId(), AUTH_INFO); + assertThat(got.getLastParagraph().getStatus()).isEqualTo(Status.ABORT); + + // But otherwise equal + got.getLastParagraph().setStatus(Status.RUNNING); + assertThat(got).isEqualTo(runningNote); + } + + @Test + public void testGet_malformed() throws Exception { + createMalformed("id"); + try { + notebookRepo.get("id", AUTH_INFO); + fail(); + } catch (IOException e) {} + } + + @Test + public void testSave_create() throws Exception { + notebookRepo.save(runningNote, AUTH_INFO); + // Output is saved + assertThat(storage.readAllBytes(makeBlobId(runningNote.getId()))) + .isEqualTo(runningNote.toJson().getBytes("UTF-8")); + } + + @Test + public void testSave_update() throws Exception { + notebookRepo.save(runningNote, AUTH_INFO); + // Change name of runningNote + runningNote.setName("new-name"); + notebookRepo.save(runningNote, AUTH_INFO); + assertThat(storage.readAllBytes(makeBlobId(runningNote.getId()))) + .isEqualTo(runningNote.toJson().getBytes("UTF-8")); + } + + @Test + public void testRemove_nonexistent() throws Exception { + try { + notebookRepo.remove("id", AUTH_INFO); + fail(); + } catch (IOException e) {} + } + + @Test + public void testRemove() throws Exception { + create(runningNote); + notebookRepo.remove(runningNote.getId(), AUTH_INFO); + assertThat(storage.get(makeBlobId(runningNote.getId()))).isNull(); + } + + private String makeName(String relativePath) { + if (basePath.isPresent()) { + return basePath.get() + "/" + relativePath; + } else { + return relativePath; + } + } + + private BlobId makeBlobId(String noteId) { + return BlobId.of(bucketName, makeName(noteId + "/note.json")); + } + + private void createAt(Note note, String relativePath) throws IOException { + BlobId id = BlobId.of(bucketName, makeName(relativePath)); + BlobInfo info = BlobInfo.newBuilder(id).setContentType("application/json").build(); + storage.create(info, note.toJson().getBytes("UTF-8")); + } + + private void create(Note note) throws IOException { + BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) + .setContentType("application/json") + .build(); + storage.create(info, note.toJson().getBytes("UTF-8")); + } + + private void createMalformed(String noteId) throws IOException { + BlobInfo info = BlobInfo.newBuilder(makeBlobId(noteId)) + .setContentType("application/json") + .build(); + storage.create(info, "{ invalid-json }".getBytes("UTF-8")); + } + + /* These tests test path parsing for illegal paths, and do not use the parameterized vars */ + + @Test + public void testInitialization_pathNotSet() throws Exception { + try { + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), ""); + new GCSNotebookRepo(new ZeppelinConfiguration(), storage); + fail(); + } catch (IOException e) {} + } + + @Test + public void testInitialization_malformedPath() throws Exception { + try { + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "foo"); + new GCSNotebookRepo(new ZeppelinConfiguration(), storage); + fail(); + } catch (IOException e) {} + } +} From fffdf258704c1937a570e3974ff64ade58df7b73 Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Sun, 11 Feb 2018 14:14:14 +0100 Subject: [PATCH 032/386] ZEPPELIN-3150. Fixed Checkstyle errors and warnings in the bigquery module ### What is this PR for? Fixed the Checkstyle errors and warnings in the bigquery module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3150 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2783 from HorizonNet/ZEPPELIN-3150 and squashes the following commits: 5ab3caa [Jan Hentschel] ZEPPELIN-3150. Removed SuppressWarnings annotation in BigQueryInterpreterTest c6be5be [Jan Hentschel] ZEPPELIN-3150. Fixed Checkstyle errors and warnings in the bigquery module --- bigquery/pom.xml | 7 ++ .../bigquery/BigQueryInterpreter.java | 68 ++++++------------- .../bigquery/BigQueryInterpreterTest.java | 47 ++++--------- 3 files changed, 43 insertions(+), 79 deletions(-) diff --git a/bigquery/pom.xml b/bigquery/pom.xml index c116c2fdd11..66fe3f2d6ed 100644 --- a/bigquery/pom.xml +++ b/bigquery/pom.xml @@ -133,6 +133,13 @@ + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java b/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java index ca06964129e..2cd6d479e51 100644 --- a/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java +++ b/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java @@ -16,62 +16,44 @@ package org.apache.zeppelin.bigquery; - -import static org.apache.commons.lang.StringUtils.containsIgnoreCase; - +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.GenericJson; import com.google.api.client.json.JsonFactory; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.json.jackson2.JacksonFactory; - import com.google.api.services.bigquery.Bigquery; -import com.google.api.services.bigquery.BigqueryScopes; -import com.google.api.client.json.GenericJson; -import com.google.api.services.bigquery.Bigquery.Datasets; +import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults; import com.google.api.services.bigquery.BigqueryRequest; -import com.google.api.services.bigquery.model.DatasetList; +import com.google.api.services.bigquery.BigqueryScopes; +import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.Job; +import com.google.api.services.bigquery.model.JobCancelResponse; +import com.google.api.services.bigquery.model.QueryRequest; +import com.google.api.services.bigquery.model.QueryResponse; import com.google.api.services.bigquery.model.TableCell; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableRow; -import com.google.api.services.bigquery.model.TableSchema; -import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults; -import com.google.api.services.bigquery.model.GetQueryResultsResponse; -import com.google.api.services.bigquery.model.QueryRequest; -import com.google.api.services.bigquery.model.QueryResponse; -import com.google.api.services.bigquery.model.JobCancelResponse; -import com.google.gson.Gson; +import com.google.common.base.Function; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; import java.util.Properties; -import java.util.Set; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Function; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.google.common.collect.Sets.SetView; -import java.io.PrintStream; -import java.io.Reader; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; /** * BigQuery interpreter for Zeppelin. @@ -95,10 +77,7 @@ *

* */ - - public class BigQueryInterpreter extends Interpreter { - private static Logger logger = LoggerFactory.getLogger(BigQueryInterpreter.class); private static final char NEWLINE = '\n'; private static final char TAB = '\t'; @@ -128,7 +107,6 @@ public BigQueryInterpreter(Properties property) { super(property); } - //Function to return valid BigQuery Service @Override public void open() { @@ -182,7 +160,7 @@ public static String printRows(final GetQueryResultsResponse response) { msg.append(NEWLINE); } return msg.toString(); - } catch ( NullPointerException ex ) { + } catch (NullPointerException ex) { throw new NullPointerException("SQL Execution returned an error!"); } } @@ -207,7 +185,7 @@ public static Iterator getPages( class PageIterator implements Iterator { private BigqueryRequest request; private boolean hasNext = true; - public PageIterator(final BigqueryRequest requestTemplate) { + PageIterator(final BigqueryRequest requestTemplate) { this.request = requestTemplate; } public boolean hasNext() { @@ -251,7 +229,7 @@ private InterpreterResult executeSql(String sql) { Iterator pages; try { pages = run(sql, projId, wTime, maxRows, useLegacySql); - } catch ( IOException ex ) { + } catch (IOException ex) { logger.error(ex.getMessage()); return new InterpreterResult(Code.ERROR, ex.getMessage()); } @@ -260,15 +238,15 @@ private InterpreterResult executeSql(String sql) { finalmessage.append(printRows(pages.next())); } return new InterpreterResult(Code.SUCCESS, finalmessage.toString()); - } catch ( NullPointerException ex ) { + } catch (NullPointerException ex) { return new InterpreterResult(Code.ERROR, ex.getMessage()); } } //Function to run the SQL on bigQuery service public static Iterator run(final String queryString, - final String projId, final long wTime, final long maxRows, boolean useLegacySql) - throws IOException { + final String projId, final long wTime, final long maxRows, boolean useLegacySql) + throws IOException { try { logger.info("Use legacy sql: {}", useLegacySql); QueryResponse query; @@ -292,7 +270,6 @@ public static Iterator run(final String queryString, @Override public void close() { - logger.info("Close bqsql connection!"); service = null; @@ -322,7 +299,6 @@ public int getProgress(InterpreterContext context) { @Override public void cancel(InterpreterContext context) { - logger.info("Trying to Cancel current query statement."); if (service != null && jobId != null && projectId != null) { diff --git a/bigquery/src/test/java/org/apache/zeppelin/bigquery/BigQueryInterpreterTest.java b/bigquery/src/test/java/org/apache/zeppelin/bigquery/BigQueryInterpreterTest.java index 53c4dc30943..2ffc67b73d2 100644 --- a/bigquery/src/test/java/org/apache/zeppelin/bigquery/BigQueryInterpreterTest.java +++ b/bigquery/src/test/java/org/apache/zeppelin/bigquery/BigQueryInterpreterTest.java @@ -14,41 +14,27 @@ * limitations under the License. */ - package org.apache.zeppelin.bigquery; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Properties; - -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.display.GUI; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterContextRunner; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.apache.zeppelin.interpreter.InterpreterOutputListener; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.apache.zeppelin.user.AuthenticationInfo; -import org.junit.Before; -import org.junit.Test; import com.google.gson.Gson; import com.google.gson.JsonIOException; import com.google.gson.JsonSyntaxException; +import org.junit.Before; +import org.junit.Test; + import java.io.FileNotFoundException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.Properties; -public class BigQueryInterpreterTest { +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterResult; +public class BigQueryInterpreterTest { protected static class Constants { private String projectId; private String oneQuery; @@ -65,17 +51,15 @@ public String getOne() { public String getWrong() { return wrongQuery; } - } - @SuppressWarnings("checkstyle:abbreviationaswordinname") - protected static Constants CONSTANTS = null; + protected static Constants constants = null; public BigQueryInterpreterTest() throws JsonSyntaxException, JsonIOException, FileNotFoundException { - if (CONSTANTS == null) { + if (constants == null) { InputStream is = this.getClass().getResourceAsStream("/constants.json"); - CONSTANTS = (new Gson()).fromJson(new InputStreamReader(is), Constants.class); + constants = (new Gson()).fromJson(new InputStreamReader(is), Constants.class); } } @@ -87,7 +71,7 @@ public BigQueryInterpreterTest() @Before public void setUp() throws Exception { Properties p = new Properties(); - p.setProperty("zeppelin.bigquery.project_id", CONSTANTS.getProjectId()); + p.setProperty("zeppelin.bigquery.project_id", constants.getProjectId()); p.setProperty("zeppelin.bigquery.wait_time", "5000"); p.setProperty("zeppelin.bigquery.max_no_of_rows", "100"); @@ -96,23 +80,20 @@ public void setUp() throws Exception { bqInterpreter = new BigQueryInterpreter(p); bqInterpreter.setInterpreterGroup(intpGroup); bqInterpreter.open(); - } @Test public void sqlSuccess() { - InterpreterResult ret = bqInterpreter.interpret(CONSTANTS.getOne(), context); + InterpreterResult ret = bqInterpreter.interpret(constants.getOne(), context); assertEquals(InterpreterResult.Code.SUCCESS, ret.code()); assertEquals(ret.message().get(0).getType(), InterpreterResult.Type.TABLE); - } @Test public void badSqlSyntaxFails() { - InterpreterResult ret = bqInterpreter.interpret(CONSTANTS.getWrong(), context); + InterpreterResult ret = bqInterpreter.interpret(constants.getWrong(), context); assertEquals(InterpreterResult.Code.ERROR, ret.code()); } - } From 0605aae240f00c1c04667756599c0e61e4d8e48a Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Mon, 12 Feb 2018 14:47:44 +0530 Subject: [PATCH 033/386] [ZEPPELIN-3198] UI should not show Version/GIT Control if the same if not supported Currently, UI shows an option for version/GIT Control even when it is not supported by the underlying implementing storage configuration. It is only after users try to save a commit and get an error "Couldn't checkpoint note revision: possibly storage doesn't support versioning. Please check the logs for more details.". So, if implementing storage configuration doesn't support git storage, UI should not show those options. [Improvement] * [ZEPPELIN-3198](https://issues.apache.org/jira/projects/ZEPPELIN/issues/ZEPPELIN-3198) On using "org.apache.zeppelin.notebook.repo.GitNotebookRepo" for `zeppelin.notebook.storage` user should see revision/version control option, for rest of the others e.g. "FileSystemNotebookRepo" user should not see that option * Does the licenses files need update? N/A * Is there breaking changes for older versions? N/A * Does this needs documentation? N/A Author: Prabhjyot Singh Closes #2757 from prabhjyotsingh/ZEPPELIN-3198 and squashes the following commits: 1f854e80c [Prabhjyot Singh] check for NotebookRepoSync and fall back d2e19094d [Prabhjyot Singh] rename isDefaultRepoGit to isRevisionSupportedInDefaultRepo 70abc4420 [Prabhjyot Singh] revert NotebookRepoCommon e152ec194 [Prabhjyot Singh] rename NotebookGitRepo to NotebookRepoWithVersionControl 987edf05d [Prabhjyot Singh] fix test 580674d9c [Prabhjyot Singh] use {{isRepoGit(0)}} 11baf6f2d [Prabhjyot Singh] refactor into NotebookRepo and NotebookGitRepo c1d34df26 [Prabhjyot Singh] revert imports cd7fde105 [Prabhjyot Singh] ZEPPELIN-3198: add isRevisionSupported for NotebookRepo Change-Id: Ib464af447af49ee9e70da86e0e0f293ef38dd3a1 --- .../zeppelin/socket/NotebookServer.java | 2 +- .../src/app/notebook/notebook-actionBar.html | 2 +- .../org/apache/zeppelin/notebook/Note.java | 13 +++ .../apache/zeppelin/notebook/Notebook.java | 34 +++++-- .../notebook/repo/AzureNotebookRepo.java | 26 ----- .../notebook/repo/FileSystemNotebookRepo.java | 27 +----- .../notebook/repo/GitNotebookRepo.java | 3 +- .../notebook/repo/MongoNotebookRepo.java | 28 +----- .../zeppelin/notebook/repo/NotebookRepo.java | 62 ------------ .../notebook/repo/NotebookRepoSync.java | 54 +++++++---- .../repo/NotebookRepoWithVersionControl.java | 97 +++++++++++++++++++ .../notebook/repo/S3NotebookRepo.java | 26 ----- .../notebook/repo/VFSNotebookRepo.java | 28 +----- .../repo/zeppelinhub/ZeppelinHubRepo.java | 4 +- .../notebook/repo/GitNotebookRepoTest.java | 2 +- .../NotebookRepoSyncInitializationTest.java | 4 +- 16 files changed, 184 insertions(+), 228 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoWithVersionControl.java diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 20d5ba9cc90..d1cf9e5c081 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -49,7 +49,7 @@ import org.apache.zeppelin.notebook.NotebookImportDeserializer; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.notebook.ParagraphJobListener; -import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision; +import org.apache.zeppelin.notebook.repo.NotebookRepoWithVersionControl.Revision; import org.apache.zeppelin.notebook.socket.Message; import org.apache.zeppelin.notebook.socket.Message.OP; import org.apache.zeppelin.notebook.socket.WatcherMessage; diff --git a/zeppelin-web/src/app/notebook/notebook-actionBar.html b/zeppelin-web/src/app/notebook/notebook-actionBar.html index 9b50e819f79..f8ff830846c 100644 --- a/zeppelin-web/src/app/notebook/notebook-actionBar.html +++ b/zeppelin-web/src/app/notebook/notebook-actionBar.html @@ -100,7 +100,7 @@

- +

From 3c502cd948e9b877adea9c6589ab42d126cd4fbc Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Thu, 15 Feb 2018 14:19:00 +0800 Subject: [PATCH 038/386] ZEPPELIN-3234. z.show() compatibility with previous release ### What is this PR for? Enhance the ZeppelinContext in IPySparkInterpreter ### What type of PR is it? [Bug Fix | Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3234 ### How should this be tested? * Unit test is added ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2807 from zjffdu/ZEPPELIN-3234 and squashes the following commits: 39637ee [Jeff Zhang] ZEPPELIN-3234. z.show() compatibility with previous release --- .../zeppelin/python/IPythonInterpreter.java | 16 +++++++++++----- .../zeppelin/spark/IPySparkInterpreter.java | 6 ++++++ .../main/resources/python/zeppelin_ipyspark.py | 14 ++++++++++++++ .../spark/IPySparkInterpreterTest.java | 18 ++++++++++++++++++ .../src/test/resources/log4j.properties | 3 +-- 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java index 81cfeb24d6c..8078670f8ac 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java @@ -30,6 +30,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.zeppelin.interpreter.BaseZeppelinContext; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -76,7 +77,7 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand private IPythonClient ipythonClient; private GatewayServer gatewayServer; - private PythonZeppelinContext zeppelinContext; + protected BaseZeppelinContext zeppelinContext; private String pythonExecutable; private long ipythonLaunchTimeout; private String additionalPythonPath; @@ -114,6 +115,12 @@ public void setAddBulitinPy4j(boolean add) { this.useBuiltinPy4j = add; } + public BaseZeppelinContext buildZeppelinContext() { + return new PythonZeppelinContext( + getInterpreterGroup().getInterpreterHookRegistry(), + Integer.parseInt(getProperty("zeppelin.python.maxResult", "1000"))); + } + @Override public void open() throws InterpreterException { try { @@ -130,9 +137,7 @@ public void open() throws InterpreterException { } ipythonLaunchTimeout = Long.parseLong( getProperty("zeppelin.ipython.launch.timeout", "30000")); - this.zeppelinContext = new PythonZeppelinContext( - getInterpreterGroup().getInterpreterHookRegistry(), - Integer.parseInt(getProperty("zeppelin.python.maxResult", "1000"))); + this.zeppelinContext = buildZeppelinContext(); int ipythonPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); LOGGER.info("Launching IPython Kernel at port: " + ipythonPort); @@ -312,6 +317,7 @@ public void close() throws InterpreterException { public InterpreterResult interpret(String st, InterpreterContext context) { zeppelinContext.setGui(context.getGui()); zeppelinContext.setNoteGui(context.getNoteGui()); + zeppelinContext.setInterpreterContext(context); interpreterOutput.setInterpreterOutput(context.out); ExecuteResponse response = ipythonClient.stream_execute(ExecuteRequest.newBuilder().setCode(st).build(), @@ -361,7 +367,7 @@ public List completion(String buf, int cursor, return completions; } - public PythonZeppelinContext getZeppelinContext() { + public BaseZeppelinContext getZeppelinContext() { return zeppelinContext; } diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java index 37896f982a5..a75fda8c1d9 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java @@ -19,6 +19,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; +import org.apache.zeppelin.interpreter.BaseZeppelinContext; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -92,6 +93,11 @@ private SparkInterpreter getSparkInterpreter() throws InterpreterException { return spark; } + @Override + public BaseZeppelinContext buildZeppelinContext() { + return sparkInterpreter.getZeppelinContext(); + } + @Override public void cancel(InterpreterContext context) throws InterpreterException { super.cancel(context); diff --git a/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py b/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py index 324f48155ec..5723f455336 100644 --- a/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py +++ b/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py @@ -51,3 +51,17 @@ sqlContext = sqlc = __zSqlc__ = __zSpark__._wrapped else: sqlContext = sqlc = __zSqlc__ = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext()) + +class IPySparkZeppelinContext(PyZeppelinContext): + + def __init__(self, z): + super(IPySparkZeppelinContext, self).__init__(z) + + def show(self, obj): + from pyspark.sql import DataFrame + if isinstance(obj, DataFrame): + print(self.z.showData(obj._jdf)) + else: + super(IPySparkZeppelinContext, self).show(obj) + +z = __zeppelin__ = IPySparkZeppelinContext(intp.getZeppelinContext()) diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java index 10d87a63e0a..5eaa42c4625 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java @@ -116,6 +116,15 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx "| 1| a|\n" + "| 2| b|\n" + "+---+---+\n\n", interpreterResultMessages.get(0).getData()); + + context = getInterpreterContext(); + result = iPySparkInterpreter.interpret("z.show(df)", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + interpreterResultMessages = context.out.getInterpreterResultMessages(); + assertEquals( + "_1 _2\n" + + "1 a\n" + + "2 b\n", interpreterResultMessages.get(0).getData()); } else { result = iPySparkInterpreter.interpret("df = spark.createDataFrame([(1,'a'),(2,'b')])\ndf.show()", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -127,6 +136,15 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx "| 1| a|\n" + "| 2| b|\n" + "+---+---+\n\n", interpreterResultMessages.get(0).getData()); + + context = getInterpreterContext(); + result = iPySparkInterpreter.interpret("z.show(df)", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + interpreterResultMessages = context.out.getInterpreterResultMessages(); + assertEquals( + "_1 _2\n" + + "1 a\n" + + "2 b\n", interpreterResultMessages.get(0).getData()); } // cancel diff --git a/spark/interpreter/src/test/resources/log4j.properties b/spark/interpreter/src/test/resources/log4j.properties index 6958d4c30fb..0dc7c89701f 100644 --- a/spark/interpreter/src/test/resources/log4j.properties +++ b/spark/interpreter/src/test/resources/log4j.properties @@ -46,7 +46,6 @@ log4j.logger.org.hibernate.type=ALL log4j.logger.org.apache.zeppelin.interpreter=DEBUG log4j.logger.org.apache.zeppelin.spark=DEBUG -log4j.logger.org.apache.zeppelin.python.IPythonInterpreter=DEBUG -log4j.logger.org.apache.zeppelin.python.IPythonClient=DEBUG +log4j.logger.org.apache.zeppelin.python=DEBUG log4j.logger.org.apache.spark.repl.Main=INFO From 8df623b634cc1fa23b268e2bb27931ea391a1381 Mon Sep 17 00:00:00 2001 From: Magyari Sandor Szilard Date: Wed, 31 Jan 2018 18:10:18 +0100 Subject: [PATCH 039/386] ZEPPELIN-3209. Preserve thread context classloader when running jobs in RemoteInterpreterServer ### What is this PR for? Spark jobs may change current thread context classloader sometimes. For example in case of issue ZEPPELIN-2475 using Spark 2.2 and Scala 2.11.8 when you run DepInterpreter that will start SparkILoop --> ILoop which changes the current thread context classloader, from LauncherAppClassloader to ScalaClassloaderURLClassloader. This result in classloading problems when SparkInterpreter is trying to build up Spark Context, in case SparkInterpreter is started by scheduler on same thread. In short when running subsequent paragraphs, users will get an ambiguous NullPointerException, which is hard to understand as it hides the root cause of the problem. As a safety measure to prevent such cases RemoteInterpreterServer should save & restore original thread context classloader. ### What type of PR is it? [Bug Fix] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3209 ### How should this be tested? * Use Spark 2.2 and Scala 2.11.8 * create a notebook with two paragraphs * in first paragraph use %spark.dep interpreter and add load some dependencies * in second paragraph use %spark interpreter and run some spark code ### Questions: * Does the licenses files need update? * Is there breaking changes for older versions? * Does this needs documentation? Author: Magyari Sandor Szilard Closes #2771 from sancyx/master-ZEPPELIN-3209 and squashes the following commits: dd3a305da [Magyari Sandor Szilard] ZEPPELIN-3209. Preserve thread context classloader when running jobs in RemoteInterpreterServer (magyari_sandor) --- .../zeppelin/interpreter/remote/RemoteInterpreterServer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 37db1fce8ec..d50d0ed3e2b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -604,6 +604,7 @@ public void onPostExecute(String script) { @Override protected Object jobRun() throws Throwable { + ClassLoader currentThreadContextClassloader = Thread.currentThread().getContextClassLoader(); try { InterpreterContext.set(context); @@ -652,6 +653,7 @@ protected Object jobRun() throws Throwable { } return new InterpreterResult(result.code(), resultMessages); } finally { + Thread.currentThread().setContextClassLoader(currentThreadContextClassloader); InterpreterContext.remove(); } } From b335caed34e2d874c2e7a2efd103042db3ea5863 Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Sat, 10 Feb 2018 13:06:58 +0100 Subject: [PATCH 040/386] ZEPPELIN-3152. Fixed Checkstyle errors and warnings in elasticsearch module ### What is this PR for? Fixed all Checkstyle errors and warnings in the elasticsearch module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3152 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2786 from HorizonNet/ZEPPELIN-3152 and squashes the following commits: 17e0325 [Jan Hentschel] ZEPPELIN-3152. Fixed Checkstyle errors and warnings in elasticsearch module --- elasticsearch/pom.xml | 7 ++ .../ElasticsearchInterpreter.java | 91 +++++++------------ .../elasticsearch/action/AggWrapper.java | 5 +- .../elasticsearch/client/HttpBasedClient.java | 85 +++++++---------- .../client/TransportBasedClient.java | 51 +++++------ .../ElasticsearchInterpreterTest.java | 85 ++++++++--------- 6 files changed, 142 insertions(+), 182 deletions(-) diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml index f80cbc5e72c..4e4021f7140 100644 --- a/elasticsearch/pom.xml +++ b/elasticsearch/pom.xml @@ -101,6 +101,13 @@ maven-resources-plugin + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java index 6251b92512c..45b37c4ebc6 100644 --- a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java +++ b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java @@ -17,6 +17,23 @@ package org.apache.zeppelin.elasticsearch; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; + +import org.apache.commons.lang3.StringUtils; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; +import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -32,7 +49,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.lang3.StringUtils; +import com.github.wnameless.json.flattener.JsonFlattener; + import org.apache.zeppelin.completer.CompletionType; import org.apache.zeppelin.elasticsearch.action.ActionResponse; import org.apache.zeppelin.elasticsearch.action.AggWrapper; @@ -44,29 +62,11 @@ import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; -import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation; -import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; -import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.github.wnameless.json.flattener.JsonFlattener; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonObject; - /** * Elasticsearch Interpreter for Zeppelin. */ public class ElasticsearchInterpreter extends Interpreter { - private static Logger logger = LoggerFactory.getLogger(ElasticsearchInterpreter.class); private static final String HELP = "Elasticsearch interpreter:\n" @@ -92,7 +92,6 @@ public class ElasticsearchInterpreter extends Interpreter { private static final Pattern FIELD_NAME_PATTERN = Pattern.compile("\\[\\\\\"(.+)\\\\\"\\](.*)"); - public static final String ELASTICSEARCH_HOST = "elasticsearch.host"; public static final String ELASTICSEARCH_PORT = "elasticsearch.port"; public static final String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type"; @@ -107,7 +106,6 @@ public class ElasticsearchInterpreter extends Interpreter { public ElasticsearchInterpreter(Properties property) { super(property); - } @Override @@ -119,8 +117,7 @@ public void open() { try { this.resultSize = Integer.parseInt(getProperty(ELASTICSEARCH_RESULT_SIZE)); - } - catch (final NumberFormatException e) { + } catch (final NumberFormatException e) { this.resultSize = 10; logger.error("Unable to parse " + ELASTICSEARCH_RESULT_SIZE + " : " + getProperty(ELASTICSEARCH_RESULT_SIZE), e); @@ -129,15 +126,12 @@ public void open() { try { if (StringUtils.isEmpty(clientType) || "transport".equals(clientType)) { elsClient = new TransportBasedClient(getProperties()); - } - else if ("http".equals(clientType)) { + } else if ("http".equals(clientType)) { elsClient = new HttpBasedClient(getProperties()); - } - else { + } else { logger.error("Unknown type of Elasticsearch client: " + clientType); } - } - catch (final IOException e) { + } catch (final IOException e) { logger.error("Open connection with Elasticsearch", e); } } @@ -203,23 +197,18 @@ public InterpreterResult interpret(String cmd, InterpreterContext interpreterCon try { if ("get".equalsIgnoreCase(method)) { return processGet(urlItems, interpreterContext); - } - else if ("count".equalsIgnoreCase(method)) { + } else if ("count".equalsIgnoreCase(method)) { return processCount(urlItems, data, interpreterContext); - } - else if ("search".equalsIgnoreCase(method)) { + } else if ("search".equalsIgnoreCase(method)) { return processSearch(urlItems, data, currentResultSize, interpreterContext); - } - else if ("index".equalsIgnoreCase(method)) { + } else if ("index".equalsIgnoreCase(method)) { return processIndex(urlItems, data); - } - else if ("delete".equalsIgnoreCase(method)) { + } else if ("delete".equalsIgnoreCase(method)) { return processDelete(urlItems); } return processHelp(InterpreterResult.Code.ERROR, "Unknown command"); - } - catch (final Exception e) { + } catch (final Exception e) { return new InterpreterResult(InterpreterResult.Code.ERROR, "Error : " + e.getMessage()); } } @@ -259,7 +248,6 @@ private void addAngularObject(InterpreterContext interpreterContext, String pref } private String[] getIndexTypeId(String[] urlItems) { - if (urlItems.length < 3) { return null; } @@ -279,7 +267,6 @@ private String[] getIndexTypeId(String[] urlItems) { private InterpreterResult processHelp(InterpreterResult.Code code, String additionalMessage) { final StringBuffer buffer = new StringBuffer(); - if (additionalMessage != null) { buffer.append(additionalMessage).append("\n"); } @@ -297,7 +284,6 @@ private InterpreterResult processHelp(InterpreterResult.Code code, String additi * @return Result of the get request, it contains a JSON-formatted string */ private InterpreterResult processGet(String[] urlItems, InterpreterContext interpreterContext) { - final String[] indexTypeId = getIndexTypeId(urlItems); if (indexTypeId == null) { @@ -332,7 +318,6 @@ private InterpreterResult processGet(String[] urlItems, InterpreterContext inter */ private InterpreterResult processCount(String[] urlItems, String data, InterpreterContext interpreterContext) { - if (urlItems.length > 2) { return new InterpreterResult(InterpreterResult.Code.ERROR, "Bad URL (it should be /index1,index2,.../type1,type2,...)"); @@ -359,7 +344,6 @@ private InterpreterResult processCount(String[] urlItems, String data, */ private InterpreterResult processSearch(String[] urlItems, String data, int size, InterpreterContext interpreterContext) { - if (urlItems.length > 2) { return new InterpreterResult(InterpreterResult.Code.ERROR, "Bad URL (it should be /index1,index2,.../type1,type2,...)"); @@ -382,7 +366,6 @@ private InterpreterResult processSearch(String[] urlItems, String data, int size * @return Result of the index request, it contains the id of the document */ private InterpreterResult processIndex(String[] urlItems, String data) { - if (urlItems.length < 2 || urlItems.length > 3) { return new InterpreterResult(InterpreterResult.Code.ERROR, "Bad URL (it should be /index/type or /index/type/id)"); @@ -404,7 +387,6 @@ private InterpreterResult processIndex(String[] urlItems, String data) { * @return Result of the delete request, it contains the id of the deleted document */ private InterpreterResult processDelete(String[] urlItems) { - final String[] indexTypeId = getIndexTypeId(urlItems); if (indexTypeId == null) { @@ -426,7 +408,6 @@ private InterpreterResult processDelete(String[] urlItems) { } private ActionResponse searchData(String[] urlItems, String query, int size) { - String[] indices = null; String[] types = null; @@ -441,7 +422,6 @@ private ActionResponse searchData(String[] urlItems, String query, int size) { } private InterpreterResult buildAggResponseMessage(Aggregations aggregations) { - // Only the result of the first aggregation is returned // final Aggregation agg = aggregations.asList().get(0); @@ -450,11 +430,9 @@ private InterpreterResult buildAggResponseMessage(Aggregations aggregations) { if (agg instanceof InternalMetricsAggregation) { resMsg = XContentHelper.toString((InternalMetricsAggregation) agg).toString(); - } - else if (agg instanceof InternalSingleBucketAggregation) { + } else if (agg instanceof InternalSingleBucketAggregation) { resMsg = XContentHelper.toString((InternalSingleBucketAggregation) agg).toString(); - } - else if (agg instanceof InternalMultiBucketAggregation) { + } else if (agg instanceof InternalMultiBucketAggregation) { final Set headerKeys = new HashSet<>(); final List> buckets = new LinkedList<>(); final InternalMultiBucketAggregation multiBucketAgg = (InternalMultiBucketAggregation) agg; @@ -466,8 +444,7 @@ else if (agg instanceof InternalMultiBucketAggregation) { final Map bucketMap = JsonFlattener.flattenAsMap(builder.string()); headerKeys.addAll(bucketMap.keySet()); buckets.add(bucketMap); - } - catch (final IOException e) { + } catch (final IOException e) { logger.error("Processing bucket: " + e.getMessage(), e); } } @@ -496,7 +473,6 @@ else if (agg instanceof InternalMultiBucketAggregation) { } private InterpreterResult buildAggResponseMessage(List aggregations) { - final InterpreterResult.Type resType = InterpreterResult.Type.TABLE; String resMsg = ""; @@ -531,7 +507,6 @@ private InterpreterResult buildAggResponseMessage(List aggregations) } private String buildSearchHitsResponseMessage(ActionResponse response) { - if (response.getHits() == null || response.getHits().size() == 0) { return ""; } @@ -553,8 +528,7 @@ private String buildSearchHitsResponseMessage(ActionResponse response) { if (fieldNameMatcher.matches()) { flattenMap.put(fieldNameMatcher.group(1) + fieldNameMatcher.group(2), flattenJsonMap.get(fieldName)); - } - else { + } else { flattenMap.put(fieldName, flattenJsonMap.get(fieldName)); } } @@ -590,7 +564,6 @@ private String buildSearchHitsResponseMessage(ActionResponse response) { } private InterpreterResult buildResponseMessage(ActionResponse response) { - final List aggregations = response.getAggregations(); if (aggregations != null && aggregations.size() > 0) { diff --git a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/action/AggWrapper.java b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/action/AggWrapper.java index 14446dbe3b8..a3ed951b224 100644 --- a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/action/AggWrapper.java +++ b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/action/AggWrapper.java @@ -21,8 +21,9 @@ * Contains the result of an aggregation. */ public class AggWrapper { - - /** Type of an aggregation (to know if there are buckets or not) */ + /** + * Type of an aggregation (to know if there are buckets or not). + */ public enum AggregationType { SIMPLE, MULTI_BUCKETS }; private final AggregationType type; diff --git a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/HttpBasedClient.java b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/HttpBasedClient.java index d691597c852..f2a9f0287f2 100644 --- a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/HttpBasedClient.java +++ b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/HttpBasedClient.java @@ -17,26 +17,21 @@ package org.apache.zeppelin.elasticsearch.client; +import com.google.common.base.Joiner; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; + +import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; + import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.Iterator; import java.util.Map; import java.util.Properties; -import org.apache.commons.lang3.StringUtils; -import org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter; -import org.apache.zeppelin.elasticsearch.action.ActionException; -import org.apache.zeppelin.elasticsearch.action.ActionResponse; -import org.apache.zeppelin.elasticsearch.action.AggWrapper; -import org.apache.zeppelin.elasticsearch.action.AggWrapper.AggregationType; -import org.apache.zeppelin.elasticsearch.action.HitWrapper; -import org.json.JSONArray; -import org.json.JSONObject; - -import com.google.common.base.Joiner; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonParseException; import com.mashape.unirest.http.HttpResponse; import com.mashape.unirest.http.JsonNode; import com.mashape.unirest.http.Unirest; @@ -44,11 +39,17 @@ import com.mashape.unirest.request.HttpRequest; import com.mashape.unirest.request.HttpRequestWithBody; +import org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter; +import org.apache.zeppelin.elasticsearch.action.ActionException; +import org.apache.zeppelin.elasticsearch.action.ActionResponse; +import org.apache.zeppelin.elasticsearch.action.AggWrapper; +import org.apache.zeppelin.elasticsearch.action.AggWrapper.AggregationType; +import org.apache.zeppelin.elasticsearch.action.HitWrapper; + /** * Elasticsearch client using the HTTP API. */ public class HttpBasedClient implements ElasticsearchClient { - private static final String QUERY_STRING_TEMPLATE = "{ \"query\": { \"query_string\": { \"query\": \"_Q_\", \"analyze_wildcard\": \"true\" } } }"; @@ -114,24 +115,20 @@ private String getUrl(String index, String type, String id, boolean useSearch) { if (id.equals(encodedId)) { // No difference, use directly the id buffer.append("/").append(id); - } - else { + } else { // There are differences: to avoid problems with some special characters // such as / and # in id, use a "terms" query - buffer.append("/_search?source=") - .append(URLEncoder + buffer.append("/_search?source=").append(URLEncoder .encode("{\"query\":{\"terms\":{\"_id\":[\"" + id + "\"]}}}", "UTF-8")); } - } - else { + } else { buffer.append("/").append(id); } } } } return buffer.toString(); - } - catch (final UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { throw new ActionException(e); } } @@ -164,8 +161,7 @@ public ActionResponse get(String index, String type, String id) { getFieldAsString(body, "_type"), getFieldAsString(body, "_id"), getFieldAsString(body, "_source"))); - } - else { + } else { final JSONArray hits = getFieldAsArray(body.getObject(), "hits/hits"); final JSONObject hit = (JSONObject) hits.iterator().next(); response = new ActionResponse() @@ -176,18 +172,15 @@ public ActionResponse get(String index, String type, String id) { hit.getString("_id"), hit.opt("_source").toString())); } - } - else { + } else { if (result.getStatus() == 404) { response = new ActionResponse() .succeeded(false); - } - else { + } else { throw new ActionException(result.getBody()); } } - } - catch (final UnirestException e) { + } catch (final UnirestException e) { throw new ActionException(e); } return response; @@ -214,12 +207,10 @@ public ActionResponse delete(String index, String type, String id) { getFieldAsString(body, "_type"), getFieldAsString(body, "_id"), null)); - } - else { + } else { throw new ActionException(result.getBody()); } - } - catch (final UnirestException e) { + } catch (final UnirestException e) { throw new ActionException(e); } return response; @@ -232,8 +223,7 @@ public ActionResponse index(String index, String type, String id, String data) { HttpRequestWithBody request = null; if (StringUtils.isEmpty(id)) { request = Unirest.post(getUrl(index, type, id, false)); - } - else { + } else { request = Unirest.put(getUrl(index, type, id, false)); } request @@ -255,12 +245,10 @@ public ActionResponse index(String index, String type, String id, String data) { getFieldAsString(result, "_type"), getFieldAsString(result, "_id"), null)); - } - else { + } else { throw new ActionException(result.getBody().toString()); } - } - catch (final UnirestException e) { + } catch (final UnirestException e) { throw new ActionException(e); } return response; @@ -275,8 +263,7 @@ public ActionResponse search(String[] indices, String[] types, String query, int // So, try to parse as a JSON => if there is an error, consider the query a Lucene one try { gson.fromJson(query, Map.class); - } - catch (final JsonParseException e) { + } catch (final JsonParseException e) { // This is not a JSON (or maybe not well formatted...) query = QUERY_STRING_TEMPLATE.replace("_Q_", query); } @@ -320,15 +307,13 @@ public ActionResponse search(String[] indices, String[] types, String query, int response.addAggregation( new AggWrapper(AggregationType.MULTI_BUCKETS, buckets.next().toString())); } - } - else { + } else { response.addAggregation( new AggWrapper(AggregationType.SIMPLE, aggregationsMap.toString())); } break; // Keep only one aggregation } - } - else if (size > 0 && total > 0) { + } else if (size > 0 && total > 0) { final JSONArray hits = getFieldAsArray(body, "hits/hits"); final Iterator iter = hits.iterator(); @@ -343,12 +328,10 @@ else if (size > 0 && total > 0) { data.toString())); } } - } - else { + } else { throw new ActionException(body.get("error").toString()); } - } - catch (final UnirestException e) { + } catch (final UnirestException e) { throw new ActionException(e); } diff --git a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/TransportBasedClient.java b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/TransportBasedClient.java index 14510198c9e..2af37bd3cbe 100644 --- a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/TransportBasedClient.java +++ b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/TransportBasedClient.java @@ -17,22 +17,11 @@ package org.apache.zeppelin.elasticsearch.client; -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonSyntaxException; import org.apache.commons.lang3.StringUtils; -import org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter; -import org.apache.zeppelin.elasticsearch.action.ActionResponse; -import org.apache.zeppelin.elasticsearch.action.AggWrapper; -import org.apache.zeppelin.elasticsearch.action.HitWrapper; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; @@ -56,15 +45,26 @@ import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonSyntaxException; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter; +import org.apache.zeppelin.elasticsearch.action.ActionResponse; +import org.apache.zeppelin.elasticsearch.action.AggWrapper; +import org.apache.zeppelin.elasticsearch.action.HitWrapper; /** * Elasticsearch client using the transport protocol. */ public class TransportBasedClient implements ElasticsearchClient { - private final Gson gson = new GsonBuilder().setPrettyPrinting().create(); private final Client client; @@ -151,8 +151,7 @@ public ActionResponse search(String[] indices, String[] types, String query, int @SuppressWarnings("rawtypes") final Map source = gson.fromJson(query, Map.class); reqBuilder.setExtraSource(source); - } - catch (final JsonSyntaxException e) { + } catch (final JsonSyntaxException e) { // This is not a JSON (or maybe not well formatted...) reqBuilder.setQuery(QueryBuilders.queryStringQuery(query).analyzeWildcard(true)); } @@ -168,8 +167,7 @@ public ActionResponse search(String[] indices, String[] types, String query, int if (searchResp.getAggregations() != null) { setAggregations(searchResp.getAggregations(), actionResp); - } - else { + } else { for (final SearchHit hit: searchResp.getHits()) { // Fields can be found either in _source, or in fields (it depends on the query) // => specific for elasticsearch's version < 5 @@ -197,12 +195,10 @@ private void setAggregations(Aggregations aggregations, ActionResponse actionRes if (agg instanceof InternalMetricsAggregation) { actionResp.addAggregation(new AggWrapper(AggWrapper.AggregationType.SIMPLE, XContentHelper.toString((InternalMetricsAggregation) agg).toString())); - } - else if (agg instanceof InternalSingleBucketAggregation) { + } else if (agg instanceof InternalSingleBucketAggregation) { actionResp.addAggregation(new AggWrapper(AggWrapper.AggregationType.SIMPLE, XContentHelper.toString((InternalSingleBucketAggregation) agg).toString())); - } - else if (agg instanceof InternalMultiBucketAggregation) { + } else if (agg instanceof InternalMultiBucketAggregation) { final Set headerKeys = new HashSet<>(); final List> buckets = new LinkedList<>(); final InternalMultiBucketAggregation multiBucketAgg = (InternalMultiBucketAggregation) agg; @@ -213,8 +209,7 @@ else if (agg instanceof InternalMultiBucketAggregation) { bucket.toXContent(builder, null); actionResp.addAggregation( new AggWrapper(AggWrapper.AggregationType.MULTI_BUCKETS, builder.string())); - } - catch (final IOException e) { + } catch (final IOException e) { // Ignored } } diff --git a/elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java b/elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java index 64562b1cb65..e186b44ff5f 100644 --- a/elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java +++ b/elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java @@ -21,6 +21,20 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import org.apache.commons.lang.math.RandomUtils; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.experimental.theories.DataPoint; +import org.junit.experimental.theories.Theories; +import org.junit.experimental.theories.Theory; +import org.junit.runner.RunWith; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -30,29 +44,15 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.lang.math.RandomUtils; import org.apache.zeppelin.completer.CompletionType; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.experimental.theories.DataPoint; -import org.junit.experimental.theories.Theories; -import org.junit.experimental.theories.Theory; -import org.junit.runner.RunWith; @RunWith(Theories.class) public class ElasticsearchInterpreterTest { - @DataPoint public static ElasticsearchInterpreter transportInterpreter; @DataPoint public static ElasticsearchInterpreter httpInterpreter; @@ -70,17 +70,15 @@ public class ElasticsearchInterpreterTest { private static final AtomicInteger deleteId = new AtomicInteger(2); - @BeforeClass public static void populate() throws IOException { - final Settings settings = Settings.settingsBuilder() - .put("cluster.name", ELS_CLUSTER_NAME) - .put("network.host", ELS_HOST) - .put("http.port", ELS_HTTP_PORT) - .put("transport.tcp.port", ELS_TRANSPORT_PORT) - .put("path.home", ELS_PATH) - .build(); + .put("cluster.name", ELS_CLUSTER_NAME) + .put("network.host", ELS_HOST) + .put("http.port", ELS_HTTP_PORT) + .put("transport.tcp.port", ELS_TRANSPORT_PORT) + .put("path.home", ELS_PATH) + .build(); elsNode = NodeBuilder.nodeBuilder().settings(settings).node(); elsClient = elsNode.client(); @@ -170,7 +168,6 @@ private InterpreterContext buildContext(String noteAndParagraphId) { @Theory public void testCount(ElasticsearchInterpreter interpreter) { - final InterpreterContext ctx = buildContext("testCount"); InterpreterResult res = interpreter.interpret("count /unknown", ctx); @@ -180,15 +177,15 @@ public void testCount(ElasticsearchInterpreter interpreter) { assertEquals(Code.SUCCESS, res.code()); assertEquals("50", res.message().get(0).getData()); assertNotNull(ctx.getAngularObjectRegistry().get("count_testCount", null, null)); - assertEquals(50l, ctx.getAngularObjectRegistry().get("count_testCount", null, null).get()); + assertEquals(50L, ctx.getAngularObjectRegistry().get("count_testCount", null, null).get()); - res = interpreter.interpret("count /logs { \"query\": { \"match\": { \"status\": 500 } } }", ctx); + res = interpreter.interpret("count /logs { \"query\": { \"match\": { \"status\": 500 } } }", + ctx); assertEquals(Code.SUCCESS, res.code()); } @Theory public void testGet(ElasticsearchInterpreter interpreter) { - final InterpreterContext ctx = buildContext("get"); InterpreterResult res = interpreter.interpret("get /logs/http/unknown", ctx); @@ -212,7 +209,6 @@ public void testGet(ElasticsearchInterpreter interpreter) { @Theory public void testSearch(ElasticsearchInterpreter interpreter) { - final InterpreterContext ctx = buildContext("search"); InterpreterResult res = interpreter.interpret("size 10\nsearch /logs *", ctx); @@ -221,23 +217,25 @@ public void testSearch(ElasticsearchInterpreter interpreter) { res = interpreter.interpret("search /logs {{{hello}}}", ctx); assertEquals(Code.ERROR, res.code()); - res = interpreter.interpret("search /logs { \"query\": { \"match\": { \"status\": 500 } } }", ctx); + res = interpreter.interpret("search /logs { \"query\": { \"match\": { \"status\": 500 } } }", + ctx); assertEquals(Code.SUCCESS, res.code()); res = interpreter.interpret("search /logs status:404", ctx); assertEquals(Code.SUCCESS, res.code()); - res = interpreter.interpret("search /logs { \"fields\": [ \"date\", \"request.headers\" ], \"query\": { \"match\": { \"status\": 500 } } }", ctx); + res = interpreter.interpret("search /logs { \"fields\": [ \"date\", \"request.headers\" ], " + + "\"query\": { \"match\": { \"status\": 500 } } }", ctx); assertEquals(Code.SUCCESS, res.code()); } @Theory public void testAgg(ElasticsearchInterpreter interpreter) { - final InterpreterContext ctx = buildContext("agg"); // Single-value metric - InterpreterResult res = interpreter.interpret("search /logs { \"aggs\" : { \"distinct_status_count\" : " + + InterpreterResult res = interpreter.interpret("search /logs { \"aggs\" : " + + "{ \"distinct_status_count\" : " + " { \"cardinality\" : { \"field\" : \"status\" } } } }", ctx); assertEquals(Code.SUCCESS, res.code()); @@ -249,7 +247,8 @@ public void testAgg(ElasticsearchInterpreter interpreter) { // Single bucket res = interpreter.interpret("search /logs { \"aggs\" : { " + " \"200_OK\" : { \"filter\" : { \"term\": { \"status\": \"200\" } }, " + - " \"aggs\" : { \"avg_length\" : { \"avg\" : { \"field\" : \"content_length\" } } } } } }", ctx); + " \"aggs\" : { \"avg_length\" : { \"avg\" : " + + "{ \"field\" : \"content_length\" } } } } } }", ctx); assertEquals(Code.SUCCESS, res.code()); // Multi-buckets @@ -259,29 +258,31 @@ public void testAgg(ElasticsearchInterpreter interpreter) { res = interpreter.interpret("search /logs { \"aggs\" : { " + " \"length\" : { \"terms\": { \"field\": \"status\" }, " + - " \"aggs\" : { \"sum_length\" : { \"sum\" : { \"field\" : \"content_length\" } }, \"sum_status\" : { \"sum\" : { \"field\" : \"status\" } } } } } }", ctx); + " \"aggs\" : { \"sum_length\" : { \"sum\" : { \"field\" : \"content_length\" } }, " + + "\"sum_status\" : { \"sum\" : { \"field\" : \"status\" } } } } } }", ctx); assertEquals(Code.SUCCESS, res.code()); } @Theory public void testIndex(ElasticsearchInterpreter interpreter) { - - InterpreterResult res = interpreter.interpret("index /logs { \"date\": \"" + new Date() + "\", \"method\": \"PUT\", \"status\": \"500\" }", null); + InterpreterResult res = interpreter.interpret("index /logs { \"date\": \"" + new Date() + + "\", \"method\": \"PUT\", \"status\": \"500\" }", null); assertEquals(Code.ERROR, res.code()); res = interpreter.interpret("index /logs/http { bad ", null); assertEquals(Code.ERROR, res.code()); - res = interpreter.interpret("index /logs/http { \"date\": \"2015-12-06T14:54:23.368Z\", \"method\": \"PUT\", \"status\": \"500\" }", null); + res = interpreter.interpret("index /logs/http { \"date\": \"2015-12-06T14:54:23.368Z\", " + + "\"method\": \"PUT\", \"status\": \"500\" }", null); assertEquals(Code.SUCCESS, res.code()); - res = interpreter.interpret("index /logs/http/1000 { \"date\": \"2015-12-06T14:54:23.368Z\", \"method\": \"PUT\", \"status\": \"500\" }", null); + res = interpreter.interpret("index /logs/http/1000 { \"date\": " + + "\"2015-12-06T14:54:23.368Z\", \"method\": \"PUT\", \"status\": \"500\" }", null); assertEquals(Code.SUCCESS, res.code()); } @Theory public void testDelete(ElasticsearchInterpreter interpreter) { - InterpreterResult res = interpreter.interpret("delete /logs/http/unknown", null); assertEquals(Code.ERROR, res.code()); @@ -296,7 +297,6 @@ public void testDelete(ElasticsearchInterpreter interpreter) { @Theory public void testMisc(ElasticsearchInterpreter interpreter) { - InterpreterResult res = interpreter.interpret(null, null); assertEquals(Code.SUCCESS, res.code()); @@ -306,8 +306,10 @@ public void testMisc(ElasticsearchInterpreter interpreter) { @Theory public void testCompletion(ElasticsearchInterpreter interpreter) { - final List expectedResultOne = Arrays.asList(new InterpreterCompletion("count", "count", CompletionType.command.name())); - final List expectedResultTwo = Arrays.asList(new InterpreterCompletion("help", "help", CompletionType.command.name())); + final List expectedResultOne = Arrays.asList( + new InterpreterCompletion("count", "count", CompletionType.command.name())); + final List expectedResultTwo = Arrays.asList( + new InterpreterCompletion("help", "help", CompletionType.command.name())); final List resultOne = interpreter.completion("co", 0, null); final List resultTwo = interpreter.completion("he", 0, null); @@ -322,5 +324,4 @@ public void testCompletion(ElasticsearchInterpreter interpreter) { } Assert.assertEquals(ElasticsearchInterpreter.COMMANDS, allCompletionList); } - } From 8bb888b494b9f611a4e5d81bd8451626eadd7e12 Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Sat, 10 Feb 2018 12:05:20 +0100 Subject: [PATCH 041/386] ZEPPELIN-3151. Fixed Checkstyle errors and warnings in cassandra module ### What is this PR for? Fixed the Checkstyle errors and warnings in the cassandra module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3151 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2785 from HorizonNet/ZEPPELIN-3151 and squashes the following commits: 646c410 [Jan Hentschel] ZEPPELIN-3151. Fixed Checkstyle errors and warnings in cassandra module --- cassandra/pom.xml | 7 + .../cassandra/CassandraInterpreter.java | 48 +- .../zeppelin/cassandra/ParsingException.java | 2 +- .../zeppelin/cassandra/JavaDriverConfig.scala | 80 +- .../cassandra/CassandraInterpreterTest.java | 1463 +++++++++-------- .../cassandra/InterpreterLogicTest.java | 636 +++---- 6 files changed, 1157 insertions(+), 1079 deletions(-) diff --git a/cassandra/pom.xml b/cassandra/pom.xml index a07ed824fc6..c5b08f776e0 100644 --- a/cassandra/pom.xml +++ b/cassandra/pom.xml @@ -249,6 +249,13 @@ maven-resources-plugin + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java b/cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java index 0f986be3189..105e271ef31 100644 --- a/cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java +++ b/cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java @@ -16,21 +16,11 @@ */ package org.apache.zeppelin.cassandra; -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.JdkSSLOptions; -import com.datastax.driver.core.ProtocolOptions.Compression; -import com.datastax.driver.core.Session; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; +import static java.lang.Integer.parseInt; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManagerFactory; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; @@ -39,10 +29,23 @@ import java.util.List; import java.util.Properties; -import static java.lang.Integer.parseInt; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.JdkSSLOptions; +import com.datastax.driver.core.ProtocolOptions.Compression; +import com.datastax.driver.core.Session; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; /** - * Interpreter for Apache Cassandra CQL query language + * Interpreter for Apache Cassandra CQL query language. */ public class CassandraInterpreter extends Interpreter { @@ -128,14 +131,14 @@ public class CassandraInterpreter extends Interpreter { public static final String DEFAULT_CREDENTIAL = "none"; public static final String DEFAULT_POLICY = "DEFAULT"; public static final String DEFAULT_PARALLELISM = "10"; - static String DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "100"; - static String DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "100"; - static String DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "2"; - static String DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"; - static String DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "8"; - static String DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "2"; - static String DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024"; - static String DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256"; + static String defaultNewConnectionThresholdLocal = "100"; + static String defaultNewConnectionThresholdRemote = "100"; + static String defaultCoreConnectionPerHostLocal = "2"; + static String defaultCoreConnectionPerHostRemote = "1"; + static String defaultMaxConnectionPerHostLocal = "8"; + static String defaultMaxConnectionPerHostRemote = "2"; + static String defaultMaxRequestPerConnectionLocal = "1024"; + static String defaultMaxRequestPerConnectionRemote = "256"; public static final String DEFAULT_IDLE_TIMEOUT = "120"; public static final String DEFAULT_POOL_TIMEOUT = "5000"; public static final String DEFAULT_HEARTBEAT_INTERVAL = "30"; @@ -244,7 +247,6 @@ public InterpreterResult interpret(String st, InterpreterContext context) { @Override public void cancel(InterpreterContext context) { - } @Override diff --git a/cassandra/src/main/java/org/apache/zeppelin/cassandra/ParsingException.java b/cassandra/src/main/java/org/apache/zeppelin/cassandra/ParsingException.java index da9bb0cc91f..b87e8b2b530 100644 --- a/cassandra/src/main/java/org/apache/zeppelin/cassandra/ParsingException.java +++ b/cassandra/src/main/java/org/apache/zeppelin/cassandra/ParsingException.java @@ -17,7 +17,7 @@ package org.apache.zeppelin.cassandra; /** - * Parsing Exception for Cassandra CQL statement + * Parsing Exception for Cassandra CQL statement. */ public class ParsingException extends RuntimeException{ public ParsingException(String message) { diff --git a/cassandra/src/main/scala/org/apache/zeppelin/cassandra/JavaDriverConfig.scala b/cassandra/src/main/scala/org/apache/zeppelin/cassandra/JavaDriverConfig.scala index 5b2dbefdc19..865d89f9d93 100644 --- a/cassandra/src/main/scala/org/apache/zeppelin/cassandra/JavaDriverConfig.scala +++ b/cassandra/src/main/scala/org/apache/zeppelin/cassandra/JavaDriverConfig.scala @@ -178,54 +178,54 @@ class JavaDriverConfig { protocolVersion match { case "1" => - DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "8" - DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "2" - DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "2" - DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1" - DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "100" - DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "1" - DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "128" - DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "128" + defaultMaxConnectionPerHostLocal = "8" + defaultMaxConnectionPerHostRemote = "2" + defaultCoreConnectionPerHostLocal = "2" + defaultCoreConnectionPerHostRemote = "1" + defaultNewConnectionThresholdLocal = "100" + defaultNewConnectionThresholdRemote = "1" + defaultMaxRequestPerConnectionLocal = "128" + defaultMaxRequestPerConnectionRemote = "128" return ProtocolVersion.V1 case "2" => - DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "8" - DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "2" - DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "2" - DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1" - DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "100" - DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "1" - DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "128" - DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "128" + defaultMaxConnectionPerHostLocal = "8" + defaultMaxConnectionPerHostRemote = "2" + defaultCoreConnectionPerHostLocal = "2" + defaultCoreConnectionPerHostRemote = "1" + defaultNewConnectionThresholdLocal = "100" + defaultNewConnectionThresholdRemote = "1" + defaultMaxRequestPerConnectionLocal = "128" + defaultMaxRequestPerConnectionRemote = "128" return ProtocolVersion.V2 case "3" => - DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "1" - DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "1" - DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "1" - DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1" - DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "800" - DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "200" - DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024" - DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256" + defaultMaxConnectionPerHostLocal = "1" + defaultMaxConnectionPerHostRemote = "1" + defaultCoreConnectionPerHostLocal = "1" + defaultCoreConnectionPerHostRemote = "1" + defaultNewConnectionThresholdLocal = "800" + defaultNewConnectionThresholdRemote = "200" + defaultMaxRequestPerConnectionLocal = "1024" + defaultMaxRequestPerConnectionRemote = "256" return ProtocolVersion.V3 case "4" => - DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "1" - DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "1" - DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "1" - DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1" - DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "800" - DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "200" - DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024" - DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256" + defaultMaxConnectionPerHostLocal = "1" + defaultMaxConnectionPerHostRemote = "1" + defaultCoreConnectionPerHostLocal = "1" + defaultCoreConnectionPerHostRemote = "1" + defaultNewConnectionThresholdLocal = "800" + defaultNewConnectionThresholdRemote = "200" + defaultMaxRequestPerConnectionLocal = "1024" + defaultMaxRequestPerConnectionRemote = "256" return ProtocolVersion.V4 case _ => - DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "1" - DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "1" - DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "1" - DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1" - DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "800" - DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "200" - DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024" - DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256" + defaultMaxConnectionPerHostLocal = "1" + defaultMaxConnectionPerHostRemote = "1" + defaultCoreConnectionPerHostLocal = "1" + defaultCoreConnectionPerHostRemote = "1" + defaultNewConnectionThresholdLocal = "800" + defaultNewConnectionThresholdRemote = "200" + defaultMaxRequestPerConnectionLocal = "1024" + defaultMaxRequestPerConnectionRemote = "256" return ProtocolVersion.NEWEST_SUPPORTED } } diff --git a/cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java b/cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java index cf392bb428c..df5e4acb018 100644 --- a/cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java +++ b/cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java @@ -16,18 +16,57 @@ */ package org.apache.zeppelin.cassandra; -import static com.datastax.driver.core.ProtocolOptions.DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS; import static com.google.common.collect.FluentIterable.from; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import static com.datastax.driver.core.ProtocolOptions.DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS; + import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_CLUSTER_NAME; import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_COMPRESSION_PROTOCOL; import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_CREDENTIALS_PASSWORD; import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_CREDENTIALS_USERNAME; import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_HOSTS; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_LOAD_BALANCING_POLICY; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_HEARTBEAT_INTERVAL_SECONDS; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_POOLING_POOL_TIMEOUT_MILLIS; import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_PORT; import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_PROTOCOL_VERSION; -import static org.apache.zeppelin.cassandra.CassandraInterpreter.*; -import static org.assertj.core.api.Assertions.*; -import static org.mockito.Mockito.when; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_QUERY_DEFAULT_CONSISTENCY; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_QUERY_DEFAULT_FETCH_SIZE; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_QUERY_DEFAULT_SERIAL_CONSISTENCY; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_RECONNECTION_POLICY; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_RETRY_POLICY; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_SOCKET_CONNECTION_TIMEOUT_MILLIS; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_SOCKET_TCP_NO_DELAY; +import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_SPECULATIVE_EXECUTION_POLICY; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Properties; import com.datastax.driver.core.Cluster; import com.datastax.driver.core.ProtocolVersion; @@ -40,714 +79,716 @@ import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.junit.*; -import org.junit.runner.RunWith; -import org.mockito.Answers; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Properties; @RunWith(MockitoJUnitRunner.class) public class CassandraInterpreterTest { - - private static final String ARTISTS_TABLE = "zeppelin.artists"; - - public static Session session = CassandraEmbeddedServerBuilder - .noEntityPackages() - .withKeyspaceName("zeppelin") - .withScript("prepare_schema.cql") - .withScript("prepare_data.cql") - .withProtocolVersion(ProtocolVersion.V3) - .buildNativeSessionOnly(); - - private static CassandraInterpreter interpreter; - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private InterpreterContext intrContext; - - @BeforeClass - public static void setUp() { - Properties properties = new Properties(); - final Cluster cluster = session.getCluster(); - - properties.setProperty(CASSANDRA_CLUSTER_NAME, cluster.getClusterName()); - properties.setProperty(CASSANDRA_COMPRESSION_PROTOCOL, "NONE"); - properties.setProperty(CASSANDRA_CREDENTIALS_USERNAME, "none"); - properties.setProperty(CASSANDRA_CREDENTIALS_PASSWORD, "none"); - - properties.setProperty(CASSANDRA_PROTOCOL_VERSION, "3"); - properties.setProperty(CASSANDRA_LOAD_BALANCING_POLICY, "DEFAULT"); - properties.setProperty(CASSANDRA_RETRY_POLICY, "DEFAULT"); - properties.setProperty(CASSANDRA_RECONNECTION_POLICY, "DEFAULT"); - properties.setProperty(CASSANDRA_SPECULATIVE_EXECUTION_POLICY, "DEFAULT"); - - properties.setProperty(CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS, - DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS + ""); - - properties.setProperty(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL, "100"); - properties.setProperty(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE, "100"); - properties.setProperty(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL, "2"); - properties.setProperty(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE, "1"); - properties.setProperty(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL, "8"); - properties.setProperty(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE, "2"); - properties.setProperty(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL, "1024"); - properties.setProperty(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE, "256"); - - properties.setProperty(CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS, "120"); - properties.setProperty(CASSANDRA_POOLING_POOL_TIMEOUT_MILLIS, "5000"); - properties.setProperty(CASSANDRA_POOLING_HEARTBEAT_INTERVAL_SECONDS, "30"); - - properties.setProperty(CASSANDRA_QUERY_DEFAULT_CONSISTENCY, "ONE"); - properties.setProperty(CASSANDRA_QUERY_DEFAULT_SERIAL_CONSISTENCY, "SERIAL"); - properties.setProperty(CASSANDRA_QUERY_DEFAULT_FETCH_SIZE, "5000"); - - properties.setProperty(CASSANDRA_SOCKET_CONNECTION_TIMEOUT_MILLIS, "5000"); - properties.setProperty(CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS, "12000"); - properties.setProperty(CASSANDRA_SOCKET_TCP_NO_DELAY, "true"); - - properties.setProperty(CASSANDRA_HOSTS, from(cluster.getMetadata().getAllHosts()).first().get().getAddress().getHostAddress()); - properties.setProperty(CASSANDRA_PORT, cluster.getConfiguration().getProtocolOptions().getPort()+""); - interpreter = new CassandraInterpreter(properties); - interpreter.open(); - } - - @AfterClass - public static void tearDown() { - interpreter.close(); - } - - @Before - public void prepareContext() { - when(intrContext.getParagraphTitle()).thenReturn("Paragraph1"); - } - - @Test - public void should_create_cluster_and_session_upon_call_to_open() throws Exception { - assertThat(interpreter.cluster).isNotNull(); - assertThat(interpreter.cluster.getClusterName()).isEqualTo(session.getCluster().getClusterName()); - assertThat(interpreter.session).isNotNull(); - assertThat(interpreter.helper).isNotNull(); - } - - @Test - public void should_interpret_simple_select() throws Exception { - //Given - - //When - final InterpreterResult actual = interpreter.interpret("SELECT * FROM " + ARTISTS_TABLE + " LIMIT 10;", intrContext); - - //Then - assertThat(actual).isNotNull(); - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).isEqualTo("name\tborn\tcountry\tdied\tgender\tstyles\ttype\n" + - "Bogdan Raczynski\t1977-01-01\tPoland\tnull\tMale\t[Dance, Electro]\tPerson\n" + - "Krishna Das\t1947-05-31\tUSA\tnull\tMale\t[Unknown]\tPerson\n" + - "Sheryl Crow\t1962-02-11\tUSA\tnull\tFemale\t[Classic, Rock, Country, Blues, Pop, Folk]\tPerson\n" + - "Doof\t1968-08-31\tUnited Kingdom\tnull\tnull\t[Unknown]\tPerson\n" + - "House of Large Sizes\t1986-01-01\tUSA\t2003\tnull\t[Unknown]\tGroup\n" + - "Fanfarlo\t2006-01-01\tUnited Kingdom\tnull\tnull\t[Rock, Indie, Pop, Classic]\tGroup\n" + - "Jeff Beck\t1944-06-24\tUnited Kingdom\tnull\tMale\t[Rock, Pop, Classic]\tPerson\n" + - "Los Paranoias\tnull\tUnknown\tnull\tnull\t[Unknown]\tnull\n" + - "…And You Will Know Us by the Trail of Dead\t1994-01-01\tUSA\tnull\tnull\t[Rock, Pop, Classic]\tGroup\n"); - - } - - @Test - public void should_interpret_select_statement() throws Exception { - //Given - - //When - final InterpreterResult actual = interpreter.interpret("SELECT * FROM " + ARTISTS_TABLE + " LIMIT 2;", intrContext); - - //Then - assertThat(actual).isNotNull(); - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).isEqualTo("name\tborn\tcountry\tdied\tgender\tstyles\ttype\n" + - "Bogdan Raczynski\t1977-01-01\tPoland\tnull\tMale\t[Dance, Electro]\tPerson\n" + - "Krishna Das\t1947-05-31\tUSA\tnull\tMale\t[Unknown]\tPerson\n"); - - } - - @Test - public void should_interpret_multiple_statements_with_single_line_logged_batch() throws Exception { - //Given - String statements = "CREATE TABLE IF NOT EXISTS zeppelin.albums(\n" + - " title text PRIMARY KEY,\n" + - " artist text,\n" + - " year int\n" + - ");\n" + - "BEGIN BATCH"+ - " INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Impossible Dream EP','Carter the Unstoppable Sex Machine',1992);"+ - " INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Way You Are','Tears for Fears',1983);"+ - " INSERT INTO zeppelin.albums(title,artist,year) VALUES('Primitive','Soulfly',2003);"+ - "APPLY BATCH;\n"+ - "SELECT * FROM zeppelin.albums;"; - //When - final InterpreterResult actual = interpreter.interpret(statements, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).isEqualTo("title\tartist\tyear\n" + - "The Impossible Dream EP\tCarter the Unstoppable Sex Machine\t1992\n" + - "The Way You Are\tTears for Fears\t1983\n" + - "Primitive\tSoulfly\t2003\n"); - } + private static final String ARTISTS_TABLE = "zeppelin.artists"; + + public static Session session = CassandraEmbeddedServerBuilder + .noEntityPackages() + .withKeyspaceName("zeppelin") + .withScript("prepare_schema.cql") + .withScript("prepare_data.cql") + .withProtocolVersion(ProtocolVersion.V3) + .buildNativeSessionOnly(); + + private static CassandraInterpreter interpreter; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private InterpreterContext intrContext; + + @BeforeClass + public static void setUp() { + Properties properties = new Properties(); + final Cluster cluster = session.getCluster(); + + properties.setProperty(CASSANDRA_CLUSTER_NAME, cluster.getClusterName()); + properties.setProperty(CASSANDRA_COMPRESSION_PROTOCOL, "NONE"); + properties.setProperty(CASSANDRA_CREDENTIALS_USERNAME, "none"); + properties.setProperty(CASSANDRA_CREDENTIALS_PASSWORD, "none"); + + properties.setProperty(CASSANDRA_PROTOCOL_VERSION, "3"); + properties.setProperty(CASSANDRA_LOAD_BALANCING_POLICY, "DEFAULT"); + properties.setProperty(CASSANDRA_RETRY_POLICY, "DEFAULT"); + properties.setProperty(CASSANDRA_RECONNECTION_POLICY, "DEFAULT"); + properties.setProperty(CASSANDRA_SPECULATIVE_EXECUTION_POLICY, "DEFAULT"); + + properties.setProperty(CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS, + DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS + ""); + + properties.setProperty(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL, "100"); + properties.setProperty(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE, "100"); + properties.setProperty(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL, "2"); + properties.setProperty(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE, "1"); + properties.setProperty(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL, "8"); + properties.setProperty(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE, "2"); + properties.setProperty(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL, "1024"); + properties.setProperty(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE, "256"); + + properties.setProperty(CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS, "120"); + properties.setProperty(CASSANDRA_POOLING_POOL_TIMEOUT_MILLIS, "5000"); + properties.setProperty(CASSANDRA_POOLING_HEARTBEAT_INTERVAL_SECONDS, "30"); + + properties.setProperty(CASSANDRA_QUERY_DEFAULT_CONSISTENCY, "ONE"); + properties.setProperty(CASSANDRA_QUERY_DEFAULT_SERIAL_CONSISTENCY, "SERIAL"); + properties.setProperty(CASSANDRA_QUERY_DEFAULT_FETCH_SIZE, "5000"); + + properties.setProperty(CASSANDRA_SOCKET_CONNECTION_TIMEOUT_MILLIS, "5000"); + properties.setProperty(CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS, "12000"); + properties.setProperty(CASSANDRA_SOCKET_TCP_NO_DELAY, "true"); + + properties.setProperty(CASSANDRA_HOSTS, from(cluster.getMetadata().getAllHosts()).first() + .get().getAddress().getHostAddress()); + properties.setProperty(CASSANDRA_PORT, cluster.getConfiguration().getProtocolOptions() + .getPort() + ""); + interpreter = new CassandraInterpreter(properties); + interpreter.open(); + } + + @AfterClass + public static void tearDown() { + interpreter.close(); + } + + @Before + public void prepareContext() { + when(intrContext.getParagraphTitle()).thenReturn("Paragraph1"); + } + + @Test + public void should_create_cluster_and_session_upon_call_to_open() throws Exception { + assertThat(interpreter.cluster).isNotNull(); + assertThat(interpreter.cluster.getClusterName()).isEqualTo(session.getCluster() + .getClusterName()); + assertThat(interpreter.session).isNotNull(); + assertThat(interpreter.helper).isNotNull(); + } + + @Test + public void should_interpret_simple_select() throws Exception { + //Given + + //When + final InterpreterResult actual = interpreter.interpret("SELECT * FROM " + ARTISTS_TABLE + + " LIMIT 10;", intrContext); + + //Then + assertThat(actual).isNotNull(); + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()).isEqualTo("name\tborn\tcountry\tdied\tgender\t" + + "styles\ttype\n" + + "Bogdan Raczynski\t1977-01-01\tPoland\tnull\tMale\t[Dance, Electro]\tPerson\n" + + "Krishna Das\t1947-05-31\tUSA\tnull\tMale\t[Unknown]\tPerson\n" + + "Sheryl Crow\t1962-02-11\tUSA\tnull\tFemale\t" + + "[Classic, Rock, Country, Blues, Pop, Folk]\tPerson\n" + + "Doof\t1968-08-31\tUnited Kingdom\tnull\tnull\t[Unknown]\tPerson\n" + + "House of Large Sizes\t1986-01-01\tUSA\t2003\tnull\t[Unknown]\tGroup\n" + + "Fanfarlo\t2006-01-01\tUnited Kingdom\tnull\tnull\t" + + "[Rock, Indie, Pop, Classic]\tGroup\n" + + "Jeff Beck\t1944-06-24\tUnited Kingdom\tnull\tMale\t[Rock, Pop, Classic]\tPerson\n" + + "Los Paranoias\tnull\tUnknown\tnull\tnull\t[Unknown]\tnull\n" + + "…And You Will Know Us by the Trail of Dead\t1994-01-01\tUSA\tnull\tnull\t" + + "[Rock, Pop, Classic]\tGroup\n"); + } + + @Test + public void should_interpret_select_statement() throws Exception { + //Given + + //When + final InterpreterResult actual = interpreter.interpret("SELECT * FROM " + ARTISTS_TABLE + + " LIMIT 2;", intrContext); + + //Then + assertThat(actual).isNotNull(); + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()) + .isEqualTo("name\tborn\tcountry\tdied\tgender\tstyles\ttype\n" + + "Bogdan Raczynski\t1977-01-01\tPoland\tnull\tMale\t[Dance, Electro]\tPerson\n" + + "Krishna Das\t1947-05-31\tUSA\tnull\tMale\t[Unknown]\tPerson\n"); + } + + @Test + public void should_interpret_multiple_statements_with_single_line_logged_batch() { + //Given + String statements = "CREATE TABLE IF NOT EXISTS zeppelin.albums(\n" + + " title text PRIMARY KEY,\n" + + " artist text,\n" + + " year int\n" + + ");\n" + + "BEGIN BATCH" + + " INSERT INTO zeppelin.albums(title,artist,year) " + + "VALUES('The Impossible Dream EP','Carter the Unstoppable Sex Machine',1992);" + + " INSERT INTO zeppelin.albums(title,artist,year) " + + "VALUES('The Way You Are','Tears for Fears',1983);" + + " INSERT INTO zeppelin.albums(title,artist,year) " + + "VALUES('Primitive','Soulfly',2003);" + + "APPLY BATCH;\n" + + "SELECT * FROM zeppelin.albums;"; + //When + final InterpreterResult actual = interpreter.interpret(statements, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()).isEqualTo("title\tartist\tyear\n" + + "The Impossible Dream EP\tCarter the Unstoppable Sex Machine\t1992\n" + + "The Way You Are\tTears for Fears\t1983\n" + + "Primitive\tSoulfly\t2003\n"); + } - @Test - public void should_throw_statement_not_having_semi_colon() throws Exception { - //Given - String statement = "SELECT * zeppelin.albums"; - - //When - final InterpreterResult actual = interpreter.interpret(statement, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.ERROR); - assertThat(actual.message().get(0).getData()) - .contains("Error parsing input:\n" + - "\t'SELECT * zeppelin.albums'\n" + - "Did you forget to add ; (semi-colon) at the end of each CQL statement ?"); - } - - @Test - public void should_validate_statement() throws Exception { - //Given - String statement = "SELECT * zeppelin.albums;"; - - //When - final InterpreterResult actual = interpreter.interpret(statement, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.ERROR); - assertThat(actual.message().get(0).getData()).contains("line 1:9 missing K_FROM at 'zeppelin' (SELECT * [zeppelin]....)"); - } - - @Test - public void should_execute_statement_with_consistency_option() throws Exception { - //Given - String statement = "@consistency=THREE\n" + - "SELECT * FROM zeppelin.artists LIMIT 1;"; - - //When - final InterpreterResult actual = interpreter.interpret(statement, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.ERROR); - assertThat(actual.message().get(0).getData()) - .contains("Not enough replicas available for query at consistency THREE (3 required but only 1 alive)"); - } - - @Test - public void should_execute_statement_with_serial_consistency_option() throws Exception { - //Given - String statement = "@serialConsistency=SERIAL\n" + - "SELECT * FROM zeppelin.artists LIMIT 1;"; - - //When - final InterpreterResult actual = interpreter.interpret(statement, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - } - - @Test - public void should_execute_statement_with_timestamp_option() throws Exception { - //Given - String statement1 = "INSERT INTO zeppelin.ts(key,val) VALUES('k','v1');"; - String statement2 = "@timestamp=15\n" + - "INSERT INTO zeppelin.ts(key,val) VALUES('k','v2');"; - - // Insert v1 with current timestamp - interpreter.interpret(statement1, intrContext); - - Thread.sleep(1); - - //When - // Insert v2 with past timestamp - interpreter.interpret(statement2, intrContext); - final String actual = session.execute("SELECT * FROM zeppelin.ts LIMIT 1").one().getString("val"); - - //Then - assertThat(actual).isEqualTo("v1"); - } - - @Test - public void should_execute_statement_with_retry_policy() throws Exception { - //Given - String statement = "@retryPolicy=" + interpreter.LOGGING_DOWNGRADING_RETRY + "\n" + - "@consistency=THREE\n" + - "SELECT * FROM zeppelin.artists LIMIT 1;"; - - //When - final InterpreterResult actual = interpreter.interpret(statement, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - } - - @Test - public void should_execute_statement_with_request_timeout() throws Exception { - //Given - String statement = "@requestTimeOut=10000000\n" + - "SELECT * FROM zeppelin.artists;"; - - //When - final InterpreterResult actual = interpreter.interpret(statement, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - } - - @Test - public void should_execute_prepared_and_bound_statements() throws Exception { - //Given - String queries = "@prepare[ps]=INSERT INTO zeppelin.prepared(key,val) VALUES(?,?)\n" + - "@prepare[select]=SELECT * FROM zeppelin.prepared WHERE key=:key\n" + - "@bind[ps]='myKey','myValue'\n" + - "@bind[select]='myKey'"; - - //When - final InterpreterResult actual = interpreter.interpret(queries, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).isEqualTo("key\tval\n" + - "myKey\tmyValue\n"); - } - - @Test - public void should_execute_bound_statement() throws Exception { - //Given - String queries = "@prepare[users_insert]=INSERT INTO zeppelin.users" + - "(login,firstname,lastname,addresses,location)" + - "VALUES(:login,:fn,:ln,:addresses,:loc)\n" + - "@bind[users_insert]='jdoe','John','DOE'," + - "{street_number: 3, street_name: 'Beverly Hills Bld', zip_code: 90209," + - " country: 'USA', extra_info: ['Right on the hills','Next to the post box']," + - " phone_numbers: {'home': 2016778524, 'office': 2015790847}}," + - "('USA', 90209, 'Beverly Hills')\n" + - "SELECT * FROM zeppelin.users WHERE login='jdoe';"; - //When - final InterpreterResult actual = interpreter.interpret(queries, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).isEqualTo( - "login\taddresses\tage\tdeceased\tfirstname\tlast_update\tlastname\tlocation\n" + - "jdoe\t" + - "{street_number:3,street_name:'Beverly Hills Bld',zip_code:90209," + - "country:'USA',extra_info:['Right on the hills','Next to the post box']," + - "phone_numbers:{'office':2015790847,'home':2016778524}}\tnull\t" + - "null\t" + - "John\t" + - "null\t" + - "DOE\t" + - "('USA',90209,'Beverly Hills')\n"); - } - - @Test - public void should_exception_when_executing_unknown_bound_statement() throws Exception { - //Given - String queries = "@bind[select_users]='jdoe'"; - - //When - final InterpreterResult actual = interpreter.interpret(queries, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.ERROR); - assertThat(actual.message().get(0).getData()) - .isEqualTo("The statement 'select_users' can not be bound to values. " + - "Are you sure you did prepare it with @prepare[select_users] ?"); - } - - @Test - public void should_extract_variable_from_statement() throws Exception { - //Given - AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null); - when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry); - when(intrContext.getGui().input("login", "hsue")).thenReturn("hsue"); - when(intrContext.getGui().input("age", "27")).thenReturn("27"); - - String queries = "@prepare[test_insert_with_variable]=" + - "INSERT INTO zeppelin.users(login,firstname,lastname,age) VALUES(?,?,?,?)\n" + - "@bind[test_insert_with_variable]='{{login=hsue}}','Helen','SUE',{{age=27}}\n" + - "SELECT firstname,lastname,age FROM zeppelin.users WHERE login='hsue';"; - //When - final InterpreterResult actual = interpreter.interpret(queries, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).isEqualTo("firstname\tlastname\tage\n" + - "Helen\tSUE\t27\n"); - - } - - @Test - public void should_just_prepare_statement() throws Exception { - //Given - String queries = "@prepare[just_prepare]=SELECT name,country,styles FROM zeppelin.artists LIMIT 3"; - final String expected = reformatHtml( - readTestResource("/scalate/NoResult.html")); - - //When - final InterpreterResult actual = interpreter.interpret(queries, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); - } - - @Test - public void should_execute_bound_statement_with_no_bound_value() throws Exception { - //Given - String queries = "@prepare[select_no_bound_value]=SELECT name,country,styles FROM zeppelin.artists LIMIT 3\n" + - "@bind[select_no_bound_value]"; - - //When - final InterpreterResult actual = interpreter.interpret(queries, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).isEqualTo("name\tcountry\tstyles\n" + - "Bogdan Raczynski\tPoland\t[Dance, Electro]\n" + - "Krishna Das\tUSA\t[Unknown]\n" + - "Sheryl Crow\tUSA\t[Classic, Rock, Country, Blues, Pop, Folk]\n"); - - } - - @Test - public void should_parse_date_value() throws Exception { - //Given - String queries = "@prepare[parse_date]=INSERT INTO zeppelin.users(login,last_update) VALUES(?,?)\n" + - "@bind[parse_date]='last_update','2015-07-30 12:00:01'\n" + - "SELECT last_update FROM zeppelin.users WHERE login='last_update';"; - //When - final InterpreterResult actual = interpreter.interpret(queries, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).contains("last_update\n" + - "Thu Jul 30 12:00:01"); - } - - @Test - public void should_bind_null_value() throws Exception { - //Given - String queries = "@prepare[bind_null]=INSERT INTO zeppelin.users(login,firstname,lastname) VALUES(?,?,?)\n" + - "@bind[bind_null]='bind_null',null,'NULL'\n" + - "SELECT firstname,lastname FROM zeppelin.users WHERE login='bind_null';"; - //When - final InterpreterResult actual = interpreter.interpret(queries, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).isEqualTo("firstname\tlastname\n" + - "null\tNULL\n"); - } - - @Test - public void should_bind_boolean_value() throws Exception { - //Given - String queries = "@prepare[bind_boolean]=INSERT INTO zeppelin.users(login,deceased) VALUES(?,?)\n" + - "@bind[bind_boolean]='bind_bool',false\n" + - "SELECT login,deceased FROM zeppelin.users WHERE login='bind_bool';"; - //When - final InterpreterResult actual = interpreter.interpret(queries, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message().get(0).getData()).isEqualTo("login\tdeceased\n" + - "bind_bool\tfalse\n"); - } - - @Test - public void should_fail_when_executing_a_removed_prepared_statement() throws Exception { - //Given - String prepare_first = "@prepare[to_be_removed]=INSERT INTO zeppelin.users(login,deceased) VALUES(?,?)"; - interpreter.interpret(prepare_first, intrContext); - String remove_prepared = "@remove_prepare[to_be_removed]\n" + - "@bind[to_be_removed]='bind_bool'"; - - //When - final InterpreterResult actual = interpreter.interpret(remove_prepared, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.ERROR); - assertThat(actual.message().get(0).getData()).isEqualTo("The statement 'to_be_removed' can not be bound to values. " + - "Are you sure you did prepare it with @prepare[to_be_removed] ?"); - } - - @Test - public void should_display_statistics_for_non_select_statement() throws Exception { - //Given - String query = "USE zeppelin;\nCREATE TABLE IF NOT EXISTS no_select(id int PRIMARY KEY);"; - final String rawResult = reformatHtml(readTestResource("/scalate/NoResultWithExecutionInfo.html")); - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - final Cluster cluster = session.getCluster(); - final int port = cluster.getConfiguration().getProtocolOptions().getPort(); - final String address = cluster.getMetadata().getAllHosts().iterator().next() - .getAddress().getHostAddress() - .replaceAll("/", "").replaceAll("\\[", "").replaceAll("\\]",""); - //Then - final String expected = rawResult.replaceAll("TRIED_HOSTS", address+":"+port) - .replaceAll("QUERIED_HOSTS", address +":"+port); - - - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); - } - - @Test - public void should_error_and_display_stack_trace() throws Exception { - //Given - String query = "@consistency=THREE\n" + - "SELECT * FROM zeppelin.users LIMIT 3;"; - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.ERROR); - assertThat(actual.message().get(0).getData()).contains("All host(s) tried for query failed"); - } - - @Test - public void should_describe_cluster() throws Exception { - //Given - - String query = "DESCRIBE CLUSTER;"; - final String expected = reformatHtml( - readTestResource("/scalate/DescribeCluster.html")); - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - - assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); - } - - @Test - public void should_describe_keyspaces() throws Exception { - //Given - String query = "DESCRIBE KEYSPACES;"; - final String expected = reformatHtml( - readTestResource("/scalate/DescribeKeyspaces.html")); - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - - assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); - } - - @Test - public void should_describe_keyspace() throws Exception { - //Given - String query = "DESCRIBE KEYSPACE live_data;"; - final String expected = reformatHtml( - readTestResource("/scalate/DescribeKeyspace_live_data.html")); - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - - assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); - } - - @Test - @Ignore - //TODO activate test when using Java 8 and C* 3.x - public void should_describe_function() throws Exception { - //Given - Properties properties = new Properties(); - properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1"); - properties.setProperty(CASSANDRA_PORT, "9042"); - Interpreter interpreter = new CassandraInterpreter(properties); - interpreter.open(); - - String createFunction = "CREATE FUNCTION zeppelin.maxof(val1 int,val2 int) " + - "RETURNS NULL ON NULL INPUT " + - "RETURNS int " + - "LANGUAGE java " + - "AS $$" + - " return Math.max(val1, val2);\n" + - "$$;"; - interpreter.interpret(createFunction, intrContext); - String query = "DESCRIBE FUNCTION zeppelin.maxOf;"; - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(actual.message()).isEqualTo("xxxxx"); - } - - @Test - @Ignore - //TODO activate test when using Java 8 and C* 3.x - public void should_describe_aggregate() throws Exception { - //Given - Properties properties = new Properties(); - properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1"); - properties.setProperty(CASSANDRA_PORT, "9042"); - Interpreter interpreter = new CassandraInterpreter(properties); - interpreter.open(); - - final String query = "DESCRIBE AGGREGATES;"; - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - - } - - @Test - @Ignore - //TODO activate test when using Java 8 and C* 3.x - public void should_describe_materialized_view() throws Exception { - //Given - Properties properties = new Properties(); - properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1"); - properties.setProperty(CASSANDRA_PORT, "9042"); - Interpreter interpreter = new CassandraInterpreter(properties); - interpreter.open(); - - final String query = "DESCRIBE MATERIALIZED VIEWS;"; - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - } - - @Test - public void should_describe_table() throws Exception { - //Given - String query = "DESCRIBE TABLE live_data.complex_table;"; - final String expected = reformatHtml( - readTestResource("/scalate/DescribeTable_live_data_complex_table.html")); - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - - assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); - } - - @Test - public void should_describe_udt() throws Exception { - //Given - String query = "DESCRIBE TYPE live_data.address;"; - final String expected = reformatHtml( - readTestResource("/scalate/DescribeType_live_data_address.html")); - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - - assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); - } - - @Test - public void should_describe_udt_withing_logged_in_keyspace() throws Exception { - //Given - String query = "USE live_data;\n" + - "DESCRIBE TYPE address;"; - final String expected = reformatHtml( - readTestResource("/scalate/DescribeType_live_data_address_within_current_keyspace.html")); - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - - assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); - } - - @Test - public void should_error_describing_non_existing_table() throws Exception { - //Given - String query = "USE system;\n" + - "DESCRIBE TABLE complex_table;"; - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.ERROR); - assertThat(actual.message().get(0).getData()).contains("Cannot find table system.complex_table"); - } - - @Test - public void should_error_describing_non_existing_udt() throws Exception { - //Given - String query = "USE system;\n" + - "DESCRIBE TYPE address;"; - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.ERROR); - assertThat(actual.message().get(0).getData()).contains("Cannot find type system.address"); - } - - @Test - public void should_show_help() throws Exception { - //Given - String query = "HELP;"; - final String expected = reformatHtml(readTestResource("/scalate/Help.html")); - - //When - final InterpreterResult actual = interpreter.interpret(query, intrContext); - - //Then - assertThat(actual.code()).isEqualTo(Code.SUCCESS); - assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); - } - - private static String reformatHtml(String rawHtml) { - return rawHtml - .replaceAll("\\s*\n\\s*","") - .replaceAll(">\\s+<", "><") - .replaceAll("(?s)data-target=\"#[a-f0-9-]+(?:_asCQL|_indices_asCQL)?\"", "") - .replaceAll("(?s)id=\"[a-f0-9-]+(?:_asCQL|_indices_asCQL)?\"", "") - .trim(); - } - - private static String readTestResource(String testResource) { - StringBuilder builder = new StringBuilder(); - InputStream stream = testResource.getClass().getResourceAsStream(testResource); - - try (BufferedReader br = new BufferedReader(new InputStreamReader(stream))) { - String line; - while ((line = br.readLine()) != null) { - builder.append(line).append("\n"); - } - } catch (Exception ex) { - throw new RuntimeException(ex); - } - - return builder.toString(); - } -} \ No newline at end of file + @Test + public void should_throw_statement_not_having_semi_colon() throws Exception { + //Given + String statement = "SELECT * zeppelin.albums"; + + //When + final InterpreterResult actual = interpreter.interpret(statement, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.ERROR); + assertThat(actual.message().get(0).getData()) + .contains("Error parsing input:\n" + + "\t'SELECT * zeppelin.albums'\n" + + "Did you forget to add ; (semi-colon) at the end of each CQL statement ?"); + } + + @Test + public void should_validate_statement() throws Exception { + //Given + String statement = "SELECT * zeppelin.albums;"; + + //When + final InterpreterResult actual = interpreter.interpret(statement, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.ERROR); + assertThat(actual.message().get(0).getData()) + .contains("line 1:9 missing K_FROM at 'zeppelin' (SELECT * [zeppelin]....)"); + } + + @Test + public void should_execute_statement_with_consistency_option() throws Exception { + //Given + String statement = "@consistency=THREE\n" + + "SELECT * FROM zeppelin.artists LIMIT 1;"; + + //When + final InterpreterResult actual = interpreter.interpret(statement, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.ERROR); + assertThat(actual.message().get(0).getData()) + .contains("Not enough replicas available for query at consistency THREE (3 required " + + "but only 1 alive)"); + } + + @Test + public void should_execute_statement_with_serial_consistency_option() throws Exception { + //Given + String statement = "@serialConsistency=SERIAL\n" + + "SELECT * FROM zeppelin.artists LIMIT 1;"; + + //When + final InterpreterResult actual = interpreter.interpret(statement, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + } + + @Test + public void should_execute_statement_with_timestamp_option() throws Exception { + //Given + String statement1 = "INSERT INTO zeppelin.ts(key,val) VALUES('k','v1');"; + String statement2 = "@timestamp=15\n" + + "INSERT INTO zeppelin.ts(key,val) VALUES('k','v2');"; + + // Insert v1 with current timestamp + interpreter.interpret(statement1, intrContext); + + Thread.sleep(1); + + //When + // Insert v2 with past timestamp + interpreter.interpret(statement2, intrContext); + final String actual = session.execute("SELECT * FROM zeppelin.ts LIMIT 1").one() + .getString("val"); + + //Then + assertThat(actual).isEqualTo("v1"); + } + + @Test + public void should_execute_statement_with_retry_policy() throws Exception { + //Given + String statement = "@retryPolicy=" + interpreter.LOGGING_DOWNGRADING_RETRY + "\n" + + "@consistency=THREE\n" + + "SELECT * FROM zeppelin.artists LIMIT 1;"; + + //When + final InterpreterResult actual = interpreter.interpret(statement, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + } + + @Test + public void should_execute_statement_with_request_timeout() throws Exception { + //Given + String statement = "@requestTimeOut=10000000\n" + + "SELECT * FROM zeppelin.artists;"; + + //When + final InterpreterResult actual = interpreter.interpret(statement, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + } + + @Test + public void should_execute_prepared_and_bound_statements() throws Exception { + //Given + String queries = "@prepare[ps]=INSERT INTO zeppelin.prepared(key,val) VALUES(?,?)\n" + + "@prepare[select]=SELECT * FROM zeppelin.prepared WHERE key=:key\n" + + "@bind[ps]='myKey','myValue'\n" + + "@bind[select]='myKey'"; + + //When + final InterpreterResult actual = interpreter.interpret(queries, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()).isEqualTo("key\tval\n" + + "myKey\tmyValue\n"); + } + + @Test + public void should_execute_bound_statement() throws Exception { + //Given + String queries = "@prepare[users_insert]=INSERT INTO zeppelin.users" + + "(login,firstname,lastname,addresses,location)" + + "VALUES(:login,:fn,:ln,:addresses,:loc)\n" + + "@bind[users_insert]='jdoe','John','DOE'," + + "{street_number: 3, street_name: 'Beverly Hills Bld', zip_code: 90209," + + " country: 'USA', extra_info: ['Right on the hills','Next to the post box']," + + " phone_numbers: {'home': 2016778524, 'office': 2015790847}}," + + "('USA', 90209, 'Beverly Hills')\n" + + "SELECT * FROM zeppelin.users WHERE login='jdoe';"; + //When + final InterpreterResult actual = interpreter.interpret(queries, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()).isEqualTo( + "login\taddresses\tage\tdeceased\tfirstname\tlast_update\tlastname\tlocation\n" + + "jdoe\t" + + "{street_number:3,street_name:'Beverly Hills Bld',zip_code:90209," + + "country:'USA',extra_info:['Right on the hills','Next to the post box']," + + "phone_numbers:{'office':2015790847,'home':2016778524}}\tnull\t" + + "null\t" + + "John\t" + + "null\t" + + "DOE\t" + + "('USA',90209,'Beverly Hills')\n"); + } + + @Test + public void should_exception_when_executing_unknown_bound_statement() throws Exception { + //Given + String queries = "@bind[select_users]='jdoe'"; + + //When + final InterpreterResult actual = interpreter.interpret(queries, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.ERROR); + assertThat(actual.message().get(0).getData()) + .isEqualTo("The statement 'select_users' can not be bound to values. " + + "Are you sure you did prepare it with @prepare[select_users] ?"); + } + + @Test + public void should_extract_variable_from_statement() throws Exception { + //Given + AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null); + when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry); + when(intrContext.getGui().input("login", "hsue")).thenReturn("hsue"); + when(intrContext.getGui().input("age", "27")).thenReturn("27"); + + String queries = "@prepare[test_insert_with_variable]=" + + "INSERT INTO zeppelin.users(login,firstname,lastname,age) VALUES(?,?,?,?)\n" + + "@bind[test_insert_with_variable]='{{login=hsue}}','Helen','SUE',{{age=27}}\n" + + "SELECT firstname,lastname,age FROM zeppelin.users WHERE login='hsue';"; + //When + final InterpreterResult actual = interpreter.interpret(queries, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()).isEqualTo("firstname\tlastname\tage\n" + + "Helen\tSUE\t27\n"); + } + + @Test + public void should_just_prepare_statement() throws Exception { + //Given + String queries = "@prepare[just_prepare]=SELECT name,country,styles " + + "FROM zeppelin.artists LIMIT 3"; + final String expected = reformatHtml( + readTestResource("/scalate/NoResult.html")); + + //When + final InterpreterResult actual = interpreter.interpret(queries, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); + } + + @Test + public void should_execute_bound_statement_with_no_bound_value() throws Exception { + //Given + String queries = "@prepare[select_no_bound_value]=SELECT name,country,styles " + + "FROM zeppelin.artists LIMIT 3\n" + + "@bind[select_no_bound_value]"; + + //When + final InterpreterResult actual = interpreter.interpret(queries, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()).isEqualTo("name\tcountry\tstyles\n" + + "Bogdan Raczynski\tPoland\t[Dance, Electro]\n" + + "Krishna Das\tUSA\t[Unknown]\n" + + "Sheryl Crow\tUSA\t[Classic, Rock, Country, Blues, Pop, Folk]\n"); + } + + @Test + public void should_parse_date_value() throws Exception { + //Given + String queries = "@prepare[parse_date]=INSERT INTO zeppelin.users(login,last_update) " + + "VALUES(?,?)\n" + + "@bind[parse_date]='last_update','2015-07-30 12:00:01'\n" + + "SELECT last_update FROM zeppelin.users WHERE login='last_update';"; + //When + final InterpreterResult actual = interpreter.interpret(queries, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()).contains("last_update\n" + + "Thu Jul 30 12:00:01"); + } + + @Test + public void should_bind_null_value() throws Exception { + //Given + String queries = "@prepare[bind_null]=INSERT INTO zeppelin.users(login,firstname,lastname) " + + "VALUES(?,?,?)\n" + + "@bind[bind_null]='bind_null',null,'NULL'\n" + + "SELECT firstname,lastname FROM zeppelin.users WHERE login='bind_null';"; + //When + final InterpreterResult actual = interpreter.interpret(queries, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()).isEqualTo("firstname\tlastname\n" + + "null\tNULL\n"); + } + + @Test + public void should_bind_boolean_value() throws Exception { + //Given + String queries = "@prepare[bind_boolean]=INSERT INTO zeppelin.users(login,deceased) " + + "VALUES(?,?)\n" + + "@bind[bind_boolean]='bind_bool',false\n" + + "SELECT login,deceased FROM zeppelin.users WHERE login='bind_bool';"; + //When + final InterpreterResult actual = interpreter.interpret(queries, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message().get(0).getData()).isEqualTo("login\tdeceased\n" + + "bind_bool\tfalse\n"); + } + + @Test + public void should_fail_when_executing_a_removed_prepared_statement() throws Exception { + //Given + String prepareFirst = "@prepare[to_be_removed]=INSERT INTO zeppelin.users(login,deceased) " + + "VALUES(?,?)"; + interpreter.interpret(prepareFirst, intrContext); + String removePrepared = "@remove_prepare[to_be_removed]\n" + + "@bind[to_be_removed]='bind_bool'"; + + //When + final InterpreterResult actual = interpreter.interpret(removePrepared, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.ERROR); + assertThat(actual.message().get(0).getData()).isEqualTo("The statement 'to_be_removed' can " + + "not be bound to values. Are you sure you did prepare it with " + + "@prepare[to_be_removed] ?"); + } + + @Test + public void should_display_statistics_for_non_select_statement() throws Exception { + //Given + String query = "USE zeppelin;\nCREATE TABLE IF NOT EXISTS no_select(id int PRIMARY KEY);"; + final String rawResult = reformatHtml(readTestResource( + "/scalate/NoResultWithExecutionInfo.html")); + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + final Cluster cluster = session.getCluster(); + final int port = cluster.getConfiguration().getProtocolOptions().getPort(); + final String address = cluster.getMetadata().getAllHosts().iterator().next() + .getAddress().getHostAddress() + .replaceAll("/", "").replaceAll("\\[", "").replaceAll("\\]", ""); + //Then + final String expected = rawResult.replaceAll("TRIED_HOSTS", address + ":" + port) + .replaceAll("QUERIED_HOSTS", address + ":" + port); + + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); + } + + @Test + public void should_error_and_display_stack_trace() throws Exception { + //Given + String query = "@consistency=THREE\n" + + "SELECT * FROM zeppelin.users LIMIT 3;"; + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.ERROR); + assertThat(actual.message().get(0).getData()).contains("All host(s) tried for query failed"); + } + + @Test + public void should_describe_cluster() throws Exception { + //Given + + String query = "DESCRIBE CLUSTER;"; + final String expected = reformatHtml( + readTestResource("/scalate/DescribeCluster.html")); + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); + } + + @Test + public void should_describe_keyspaces() throws Exception { + //Given + String query = "DESCRIBE KEYSPACES;"; + final String expected = reformatHtml( + readTestResource("/scalate/DescribeKeyspaces.html")); + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); + } + + @Test + public void should_describe_keyspace() throws Exception { + //Given + String query = "DESCRIBE KEYSPACE live_data;"; + final String expected = reformatHtml( + readTestResource("/scalate/DescribeKeyspace_live_data.html")); + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); + } + + @Test + @Ignore + //TODO(n.a.) activate test when using Java 8 and C* 3.x + public void should_describe_function() throws Exception { + //Given + Properties properties = new Properties(); + properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1"); + properties.setProperty(CASSANDRA_PORT, "9042"); + Interpreter interpreter = new CassandraInterpreter(properties); + interpreter.open(); + + String createFunction = "CREATE FUNCTION zeppelin.maxof(val1 int,val2 int) " + + "RETURNS NULL ON NULL INPUT " + + "RETURNS int " + + "LANGUAGE java " + + "AS $$" + + " return Math.max(val1, val2);\n" + + "$$;"; + interpreter.interpret(createFunction, intrContext); + String query = "DESCRIBE FUNCTION zeppelin.maxOf;"; + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(actual.message()).isEqualTo("xxxxx"); + } + + @Test + @Ignore + //TODO(n.a.) activate test when using Java 8 and C* 3.x + public void should_describe_aggregate() throws Exception { + //Given + Properties properties = new Properties(); + properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1"); + properties.setProperty(CASSANDRA_PORT, "9042"); + Interpreter interpreter = new CassandraInterpreter(properties); + interpreter.open(); + + final String query = "DESCRIBE AGGREGATES;"; + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + } + + @Test + @Ignore + //TODO(n.a.) activate test when using Java 8 and C* 3.x + public void should_describe_materialized_view() throws Exception { + //Given + Properties properties = new Properties(); + properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1"); + properties.setProperty(CASSANDRA_PORT, "9042"); + Interpreter interpreter = new CassandraInterpreter(properties); + interpreter.open(); + + final String query = "DESCRIBE MATERIALIZED VIEWS;"; + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + } + + @Test + public void should_describe_table() throws Exception { + //Given + String query = "DESCRIBE TABLE live_data.complex_table;"; + final String expected = reformatHtml( + readTestResource("/scalate/DescribeTable_live_data_complex_table.html")); + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); + } + + @Test + public void should_describe_udt() throws Exception { + //Given + String query = "DESCRIBE TYPE live_data.address;"; + final String expected = reformatHtml( + readTestResource("/scalate/DescribeType_live_data_address.html")); + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); + } + + @Test + public void should_describe_udt_withing_logged_in_keyspace() throws Exception { + //Given + String query = "USE live_data;\n" + + "DESCRIBE TYPE address;"; + final String expected = reformatHtml(readTestResource( + "/scalate/DescribeType_live_data_address_within_current_keyspace.html")); + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); + } + + @Test + public void should_error_describing_non_existing_table() throws Exception { + //Given + String query = "USE system;\n" + + "DESCRIBE TABLE complex_table;"; + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.ERROR); + assertThat(actual.message().get(0).getData()) + .contains("Cannot find table system.complex_table"); + } + + @Test + public void should_error_describing_non_existing_udt() throws Exception { + //Given + String query = "USE system;\n" + + "DESCRIBE TYPE address;"; + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.ERROR); + assertThat(actual.message().get(0).getData()).contains("Cannot find type system.address"); + } + + @Test + public void should_show_help() throws Exception { + //Given + String query = "HELP;"; + final String expected = reformatHtml(readTestResource("/scalate/Help.html")); + + //When + final InterpreterResult actual = interpreter.interpret(query, intrContext); + + //Then + assertThat(actual.code()).isEqualTo(Code.SUCCESS); + assertThat(reformatHtml(actual.message().get(0).getData())).isEqualTo(expected); + } + + private static String reformatHtml(String rawHtml) { + return rawHtml + .replaceAll("\\s*\n\\s*", "") + .replaceAll(">\\s+<", "><") + .replaceAll("(?s)data-target=\"#[a-f0-9-]+(?:_asCQL|_indices_asCQL)?\"", "") + .replaceAll("(?s)id=\"[a-f0-9-]+(?:_asCQL|_indices_asCQL)?\"", "") + .trim(); + } + + private static String readTestResource(String testResource) { + StringBuilder builder = new StringBuilder(); + InputStream stream = testResource.getClass().getResourceAsStream(testResource); + + try (BufferedReader br = new BufferedReader(new InputStreamReader(stream))) { + String line; + while ((line = br.readLine()) != null) { + builder.append(line).append("\n"); + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + return builder.toString(); + } +} diff --git a/cassandra/src/test/java/org/apache/zeppelin/cassandra/InterpreterLogicTest.java b/cassandra/src/test/java/org/apache/zeppelin/cassandra/InterpreterLogicTest.java index f3848fd1a71..e096a0c8f5a 100644 --- a/cassandra/src/test/java/org/apache/zeppelin/cassandra/InterpreterLogicTest.java +++ b/cassandra/src/test/java/org/apache/zeppelin/cassandra/InterpreterLogicTest.java @@ -16,27 +16,23 @@ */ package org.apache.zeppelin.cassandra; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import static java.util.Arrays.asList; + import static com.datastax.driver.core.BatchStatement.Type.UNLOGGED; import static com.datastax.driver.core.ConsistencyLevel.ALL; import static com.datastax.driver.core.ConsistencyLevel.LOCAL_SERIAL; import static com.datastax.driver.core.ConsistencyLevel.ONE; import static com.datastax.driver.core.ConsistencyLevel.QUORUM; import static com.datastax.driver.core.ConsistencyLevel.SERIAL; -import static java.util.Arrays.asList; -import static org.assertj.core.api.Assertions.*; -import static org.mockito.Mockito.*; -import com.datastax.driver.core.BatchStatement; -import com.datastax.driver.core.ConsistencyLevel; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.SimpleStatement; -import com.datastax.driver.core.Statement; - -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.display.GUI; -import org.apache.zeppelin.display.ui.OptionInput.ParamOption; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -47,302 +43,334 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; -import scala.Option; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.List; -import org.apache.zeppelin.cassandra.TextBlockHierarchy.*; +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SimpleStatement; +import com.datastax.driver.core.Statement; + +import scala.Option; + +import org.apache.zeppelin.cassandra.TextBlockHierarchy.AnyBlock; +import org.apache.zeppelin.cassandra.TextBlockHierarchy.Consistency; +import org.apache.zeppelin.cassandra.TextBlockHierarchy.DowngradingRetryPolicy$; +import org.apache.zeppelin.cassandra.TextBlockHierarchy.LoggingDefaultRetryPolicy$; +import org.apache.zeppelin.cassandra.TextBlockHierarchy.QueryParameters; +import org.apache.zeppelin.cassandra.TextBlockHierarchy.RequestTimeOut; +import org.apache.zeppelin.cassandra.TextBlockHierarchy.RetryPolicy; +import org.apache.zeppelin.cassandra.TextBlockHierarchy.SerialConsistency; +import org.apache.zeppelin.cassandra.TextBlockHierarchy.SimpleStm; +import org.apache.zeppelin.cassandra.TextBlockHierarchy.Timestamp; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.display.ui.OptionInput.ParamOption; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; @RunWith(MockitoJUnitRunner.class) public class InterpreterLogicTest { - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private InterpreterContext intrContext; - - @Mock - private Session session; - - final InterpreterLogic helper = new InterpreterLogic(session); - - @Captor - ArgumentCaptor optionsCaptor; - - @Test - public void should_parse_input_string_block() throws Exception { - //Given - String input = "SELECT * FROM users LIMIT 10;"; - - //When - final List anyBlocks = this.toJavaList(helper.parseInput(input)); - - //Then - assertThat(anyBlocks).hasSize(1); - assertThat(anyBlocks.get(0)).isInstanceOf(SimpleStm.class); - } - - @Test - public void should_exception_while_parsing_input() throws Exception { - //Given - String input = "SELECT * FROM users LIMIT 10"; - - //When - expectedException.expect(InterpreterException.class); - expectedException.expectMessage("Error parsing input:\n" + - "\t'SELECT * FROM users LIMIT 10'\n" + - "Did you forget to add ; (semi-colon) at the end of each CQL statement ?"); - - helper.parseInput(input); - } - - @Test - public void should_extract_variable_and_default_value() throws Exception { - //Given - AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null); - when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry); - when(intrContext.getGui().input("table", "zeppelin.demo")).thenReturn("zeppelin.demo"); - when(intrContext.getGui().input("id", "'John'")).thenReturn("'John'"); - - //When - final String actual = helper.maybeExtractVariables("SELECT * FROM {{table=zeppelin.demo}} WHERE id={{id='John'}}", intrContext); - - //Then - assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='John'"); - } - - @Test - public void should_extract_variable_and_choices() throws Exception { - //Given - AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null); - when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry); - when(intrContext.getGui().select(eq("name"), eq("'Paul'"), optionsCaptor.capture())).thenReturn("'Jack'"); - - //When - final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.artists WHERE name={{name='Paul'|'Jack'|'Smith'}}", intrContext); - - //Then - assertThat(actual).isEqualTo("SELECT * FROM zeppelin.artists WHERE name='Jack'"); - final List paramOptions = asList(optionsCaptor.getValue()); - assertThat(paramOptions.get(0).getValue()).isEqualTo("'Paul'"); - assertThat(paramOptions.get(1).getValue()).isEqualTo("'Jack'"); - assertThat(paramOptions.get(2).getValue()).isEqualTo("'Smith'"); - } - - @Test - public void should_extract_no_variable() throws Exception { - //Given - GUI gui = mock(GUI.class); - when(intrContext.getGui()).thenReturn(gui); - - //When - final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.demo", intrContext); - - //Then - verifyZeroInteractions(gui); - assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo"); - } - - @Test - public void should_extract_variable_from_angular_object_registry() throws Exception { - //Given - AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null); - angularObjectRegistry.add("id", "from_angular_registry", "noteId", "paragraphId"); - when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry); - when(intrContext.getNoteId()).thenReturn("noteId"); - when(intrContext.getParagraphId()).thenReturn("paragraphId"); - - //When - final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.demo WHERE id='{{id=John}}'", intrContext); - - //Then - assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='from_angular_registry'"); - verify(intrContext, never()).getGui(); - } - - @Test - public void should_error_if_incorrect_variable_definition() throws Exception { - //Given - - //When - expectedException.expect(ParsingException.class); - expectedException.expectMessage("Invalid bound variable definition for '{{table?zeppelin.demo}}' in 'SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}'. It should be of form 'variable=defaultValue'"); - - //Then - helper.maybeExtractVariables("SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}", intrContext); - } - - - @Test - public void should_extract_consistency_option() throws Exception { - //Given - List options = Arrays.asList(new Consistency(ALL), new Consistency(ONE)); - - //When - final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); - - //Then - assertThat(actual.consistency().get()).isEqualTo(ALL); - } - - - @Test - public void should_extract_serial_consistency_option() throws Exception { - //Given - List options = Arrays.asList(new SerialConsistency(SERIAL), new SerialConsistency(LOCAL_SERIAL)); - - //When - final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); - - //Then - assertThat(actual.serialConsistency().get()).isEqualTo(SERIAL); - } - - @Test - public void should_extract_timestamp_option() throws Exception { - //Given - List options = Arrays.asList(new Timestamp(123L), new Timestamp(456L)); - - //When - final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); - - //Then - assertThat(actual.timestamp().get()).isEqualTo(123L); - } - - @Test - public void should_extract_retry_policy_option() throws Exception { - //Given - List options = Arrays.asList(DowngradingRetryPolicy$.MODULE$, LoggingDefaultRetryPolicy$.MODULE$); - - //When - final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); - - //Then - assertThat(actual.retryPolicy().get()).isSameAs(DowngradingRetryPolicy$.MODULE$); - } - - @Test - public void should_extract_request_timeout_option() throws Exception { - //Given - List options = Arrays.asList(new RequestTimeOut(100)); - - //When - final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); - - //Then - assertThat(actual.requestTimeOut().get()).isEqualTo(100); - } - - @Test - public void should_generate_simple_statement() throws Exception { - //Given - String input = "SELECT * FROM users LIMIT 10;"; - CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM), - Option.empty(), - Option.empty(), - Option.empty(), - Option.empty(), - Option.empty()); - - //When - final SimpleStatement actual = helper.generateSimpleStatement(new SimpleStm(input), options, intrContext); - - //Then - assertThat(actual).isNotNull(); - assertThat(actual.getQueryString()).isEqualTo("SELECT * FROM users LIMIT 10;"); - assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM); - } - - @Test - public void should_generate_batch_statement() throws Exception { - //Given - Statement st1 = new SimpleStatement("SELECT * FROM users LIMIT 10;"); - Statement st2 = new SimpleStatement("INSERT INTO users(id) VALUES(10);"); - Statement st3 = new SimpleStatement("UPDATE users SET name = 'John DOE' WHERE id=10;"); - CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM), - Option.empty(), - Option.empty(), - Option.empty(), - Option.empty(), - Option.empty()); - - //When - BatchStatement actual = helper.generateBatchStatement(UNLOGGED, options, toScalaList(asList(st1, st2, st3))); - - //Then - assertThat(actual).isNotNull(); - final List statements = new ArrayList<>(actual.getStatements()); - assertThat(statements).hasSize(3); - assertThat(statements.get(0)).isSameAs(st1); - assertThat(statements.get(1)).isSameAs(st2); - assertThat(statements.get(2)).isSameAs(st3); - assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM); - } - - @Test - public void should_parse_bound_values() throws Exception { - //Given - String bs="'jdoe',32,'John DOE',null, true, '2014-06-12 34:00:34'"; - - //When - final List actual = this.toJavaList(helper.parseBoundValues("ps", bs)); - - //Then - assertThat(actual).containsExactly("'jdoe'", "32", "'John DOE'", - "null", "true", "2014-06-12 34:00:34"); - } - - @Test - public void should_parse_simple_date() throws Exception { - //Given - String dateString = "2015-07-30 12:00:01"; - - //When - final Date actual = helper.parseDate(dateString); - - //Then - Calendar calendar = Calendar.getInstance(); - calendar.setTime(actual); - - assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015); - assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY); - assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30); - assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12); - assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0); - assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1); - } - - @Test - public void should_parse_accurate_date() throws Exception { - //Given - String dateString = "2015-07-30 12:00:01.123"; - - //When - final Date actual = helper.parseDate(dateString); - - //Then - Calendar calendar = Calendar.getInstance(); - calendar.setTime(actual); - - assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015); - assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY); - assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30); - assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12); - assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0); - assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1); - assertThat(calendar.get(Calendar.MILLISECOND)).isEqualTo(123); - } - - private scala.collection.immutable.List toScalaList(java.util.List list) { - return scala.collection.JavaConversions.collectionAsScalaIterable(list).toList(); - } - - private java.util.List toJavaList(scala.collection.immutable.List list){ - return scala.collection.JavaConversions.seqAsJavaList(list); - } -} \ No newline at end of file + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private InterpreterContext intrContext; + + @Mock + private Session session; + + final InterpreterLogic helper = new InterpreterLogic(session); + + @Captor + ArgumentCaptor optionsCaptor; + + @Test + public void should_parse_input_string_block() throws Exception { + //Given + String input = "SELECT * FROM users LIMIT 10;"; + + //When + final List anyBlocks = this.toJavaList(helper.parseInput(input)); + + //Then + assertThat(anyBlocks).hasSize(1); + assertThat(anyBlocks.get(0)).isInstanceOf(SimpleStm.class); + } + + @Test + public void should_exception_while_parsing_input() throws Exception { + //Given + String input = "SELECT * FROM users LIMIT 10"; + + //When + expectedException.expect(InterpreterException.class); + expectedException.expectMessage("Error parsing input:\n" + + "\t'SELECT * FROM users LIMIT 10'\n" + + "Did you forget to add ; (semi-colon) at the end of each CQL statement ?"); + + helper.parseInput(input); + } + + @Test + public void should_extract_variable_and_default_value() throws Exception { + //Given + AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null); + when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry); + when(intrContext.getGui().input("table", "zeppelin.demo")).thenReturn("zeppelin.demo"); + when(intrContext.getGui().input("id", "'John'")).thenReturn("'John'"); + + //When + final String actual = helper.maybeExtractVariables( + "SELECT * FROM {{table=zeppelin.demo}} WHERE id={{id='John'}}", intrContext); + + //Then + assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='John'"); + } + + @Test + public void should_extract_variable_and_choices() throws Exception { + //Given + AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null); + when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry); + when(intrContext.getGui().select(eq("name"), eq("'Paul'"), optionsCaptor.capture())) + .thenReturn("'Jack'"); + + //When + final String actual = helper.maybeExtractVariables( + "SELECT * FROM zeppelin.artists WHERE name={{name='Paul'|'Jack'|'Smith'}}", + intrContext); + + //Then + assertThat(actual).isEqualTo("SELECT * FROM zeppelin.artists WHERE name='Jack'"); + final List paramOptions = asList(optionsCaptor.getValue()); + assertThat(paramOptions.get(0).getValue()).isEqualTo("'Paul'"); + assertThat(paramOptions.get(1).getValue()).isEqualTo("'Jack'"); + assertThat(paramOptions.get(2).getValue()).isEqualTo("'Smith'"); + } + + @Test + public void should_extract_no_variable() throws Exception { + //Given + GUI gui = mock(GUI.class); + when(intrContext.getGui()).thenReturn(gui); + + //When + final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.demo", intrContext); + + //Then + verifyZeroInteractions(gui); + assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo"); + } + + @Test + public void should_extract_variable_from_angular_object_registry() throws Exception { + //Given + AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null); + angularObjectRegistry.add("id", "from_angular_registry", "noteId", "paragraphId"); + when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry); + when(intrContext.getNoteId()).thenReturn("noteId"); + when(intrContext.getParagraphId()).thenReturn("paragraphId"); + + //When + final String actual = helper.maybeExtractVariables( + "SELECT * FROM zeppelin.demo WHERE id='{{id=John}}'", intrContext); + + //Then + assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='from_angular_registry'"); + verify(intrContext, never()).getGui(); + } + + @Test + public void should_error_if_incorrect_variable_definition() throws Exception { + //Given + + //When + expectedException.expect(ParsingException.class); + expectedException.expectMessage("Invalid bound variable definition for " + + "'{{table?zeppelin.demo}}' in 'SELECT * FROM {{table?zeppelin.demo}} " + + "WHERE id={{id='John'}}'. It should be of form 'variable=defaultValue'"); + + //Then + helper.maybeExtractVariables("SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}", + intrContext); + } + + @Test + public void should_extract_consistency_option() throws Exception { + //Given + List options = Arrays.asList(new Consistency(ALL), + new Consistency(ONE)); + + //When + final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); + + //Then + assertThat(actual.consistency().get()).isEqualTo(ALL); + } + + @Test + public void should_extract_serial_consistency_option() throws Exception { + //Given + List options = Arrays.asList(new SerialConsistency(SERIAL), + new SerialConsistency(LOCAL_SERIAL)); + + //When + final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); + + //Then + assertThat(actual.serialConsistency().get()).isEqualTo(SERIAL); + } + + @Test + public void should_extract_timestamp_option() throws Exception { + //Given + List options = Arrays.asList(new Timestamp(123L), + new Timestamp(456L)); + + //When + final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); + + //Then + assertThat(actual.timestamp().get()).isEqualTo(123L); + } + + @Test + public void should_extract_retry_policy_option() throws Exception { + //Given + List options = Arrays.asList(DowngradingRetryPolicy$.MODULE$, + LoggingDefaultRetryPolicy$.MODULE$); + + //When + final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); + + //Then + assertThat(actual.retryPolicy().get()).isSameAs(DowngradingRetryPolicy$.MODULE$); + } + + @Test + public void should_extract_request_timeout_option() throws Exception { + //Given + List options = Arrays.asList(new RequestTimeOut(100)); + + //When + final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options)); + + //Then + assertThat(actual.requestTimeOut().get()).isEqualTo(100); + } + + @Test + public void should_generate_simple_statement() throws Exception { + //Given + String input = "SELECT * FROM users LIMIT 10;"; + CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM), + Option.empty(), + Option.empty(), + Option.empty(), + Option.empty(), + Option.empty()); + + //When + final SimpleStatement actual = helper.generateSimpleStatement(new SimpleStm(input), options, + intrContext); + + //Then + assertThat(actual).isNotNull(); + assertThat(actual.getQueryString()).isEqualTo("SELECT * FROM users LIMIT 10;"); + assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM); + } + + @Test + public void should_generate_batch_statement() throws Exception { + //Given + Statement st1 = new SimpleStatement("SELECT * FROM users LIMIT 10;"); + Statement st2 = new SimpleStatement("INSERT INTO users(id) VALUES(10);"); + Statement st3 = new SimpleStatement("UPDATE users SET name = 'John DOE' WHERE id=10;"); + CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM), + Option.empty(), + Option.empty(), + Option.empty(), + Option.empty(), + Option.empty()); + + //When + BatchStatement actual = helper.generateBatchStatement(UNLOGGED, options, + toScalaList(asList(st1, st2, st3))); + + //Then + assertThat(actual).isNotNull(); + final List statements = new ArrayList<>(actual.getStatements()); + assertThat(statements).hasSize(3); + assertThat(statements.get(0)).isSameAs(st1); + assertThat(statements.get(1)).isSameAs(st2); + assertThat(statements.get(2)).isSameAs(st3); + assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM); + } + + @Test + public void should_parse_bound_values() throws Exception { + //Given + String bs = "'jdoe',32,'John DOE',null, true, '2014-06-12 34:00:34'"; + + //When + final List actual = this.toJavaList(helper.parseBoundValues("ps", bs)); + + //Then + assertThat(actual).containsExactly("'jdoe'", "32", "'John DOE'", + "null", "true", "2014-06-12 34:00:34"); + } + + @Test + public void should_parse_simple_date() throws Exception { + //Given + String dateString = "2015-07-30 12:00:01"; + + //When + final Date actual = helper.parseDate(dateString); + + //Then + Calendar calendar = Calendar.getInstance(); + calendar.setTime(actual); + + assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015); + assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY); + assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30); + assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12); + assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0); + assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1); + } + + @Test + public void should_parse_accurate_date() throws Exception { + //Given + String dateString = "2015-07-30 12:00:01.123"; + + //When + final Date actual = helper.parseDate(dateString); + + //Then + Calendar calendar = Calendar.getInstance(); + calendar.setTime(actual); + + assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015); + assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY); + assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30); + assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12); + assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0); + assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1); + assertThat(calendar.get(Calendar.MILLISECOND)).isEqualTo(123); + } + + private scala.collection.immutable.List toScalaList(java.util.List list) { + return scala.collection.JavaConversions.collectionAsScalaIterable(list).toList(); + } + + private java.util.List toJavaList(scala.collection.immutable.List list){ + return scala.collection.JavaConversions.seqAsJavaList(list); + } +} From d220c011892a36b7645599fc028bc8754bd99ce6 Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Sun, 4 Feb 2018 17:38:33 +0100 Subject: [PATCH 042/386] ZEPPELIN-3164. Fixed Checkstyle errors and warnings in the scalding module ### What is this PR for? Fixed the errors and warnings from Checkstyle in the **scalding** module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3164 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2765 from HorizonNet/ZEPPELIN-3164 and squashes the following commits: 5202508 [Jan Hentschel] ZEPPELIN-3164. Fixed Checkstyle errors and warnings in the scalding module --- scalding/pom.xml | 8 ++++ .../scalding/ScaldingInterpreter.java | 37 ++++++++++--------- .../scalding/ScaldingInterpreterTest.java | 36 +++++++++++------- 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/scalding/pom.xml b/scalding/pom.xml index ec7fe40ba5b..2a9e456faee 100644 --- a/scalding/pom.xml +++ b/scalding/pom.xml @@ -184,6 +184,14 @@ + + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java b/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java index d3ebadaf057..f46a1d7d0e9 100644 --- a/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java +++ b/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java @@ -17,19 +17,9 @@ package org.apache.zeppelin.scalding; -import com.twitter.scalding.ScaldingILoop; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Console; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -42,6 +32,18 @@ import java.util.List; import java.util.Properties; +import com.twitter.scalding.ScaldingILoop; + +import scala.Console; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; + /** * Scalding interpreter for Zeppelin. Based off the Spark interpreter code. * @@ -54,8 +56,7 @@ public class ScaldingInterpreter extends Interpreter { static final String MAX_OPEN_INSTANCES = "max.open.instances"; static final String MAX_OPEN_INSTANCES_DEFAULT = "50"; - public static final List NO_COMPLETION = - Collections.unmodifiableList(new ArrayList<>()); + public static final List NO_COMPLETION = Collections.unmodifiableList(new ArrayList<>()); static int numOpenInstances = 0; private ScaldingILoop interpreter; @@ -111,7 +112,7 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr if (interpreter == null) { logger.error( - "interpreter == null, open may not have been called because max.open.instances reached"); + "interpreter == null, open may not have been called because max.open.instances reached"); return new InterpreterResult(Code.ERROR, "interpreter == null\n" + "open may not have been called because max.open.instances reached" @@ -135,11 +136,11 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr final String cmd1 = cmd; final InterpreterContext contextInterpreter1 = contextInterpreter; PrivilegedExceptionAction action = - new PrivilegedExceptionAction() { - public InterpreterResult run() throws Exception { - return interpret(cmd1.split("\n"), contextInterpreter1); - } - }; + new PrivilegedExceptionAction() { + public InterpreterResult run() throws Exception { + return interpret(cmd1.split("\n"), contextInterpreter1); + } + }; interpreterResult = ugi.doAs(action); } catch (Exception e) { logger.error("Error running command with ugi.doAs", e); diff --git a/scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java b/scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java index e5b1e90539b..2c94751b55c 100644 --- a/scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java +++ b/scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java @@ -17,7 +17,14 @@ package org.apache.zeppelin.scalding; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; import java.io.File; import java.util.HashMap; @@ -26,17 +33,12 @@ import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; -import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.junit.After; -import org.junit.Before; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; +import org.apache.zeppelin.user.AuthenticationInfo; /** * Tests for the Scalding interpreter for Zeppelin. @@ -50,7 +52,8 @@ public class ScaldingInterpreterTest { @Before public void setUp() throws Exception { - tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + + System.currentTimeMillis()); System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo"); tmpDir.mkdirs(); @@ -77,8 +80,9 @@ public void tearDown() throws Exception { } private void delete(File file) { - if (file.isFile()) file.delete(); - else if (file.isDirectory()) { + if (file.isFile()) { + file.delete(); + } else if (file.isDirectory()) { File[] files = file.listFiles(); if (files != null && files.length > 0) { for (File f : files) { @@ -91,12 +95,14 @@ else if (file.isDirectory()) { @Test public void testNextLineComments() { - assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("\"123\"\n/*comment here\n*/.toInt", context).code()); + assertEquals(InterpreterResult.Code.SUCCESS, + repl.interpret("\"123\"\n/*comment here\n*/.toInt", context).code()); } @Test public void testNextLineCompanionObject() { - String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter {\n def apply(x: Long) = new Counter()\n}"; + String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter " + + "{\n def apply(x: Long) = new Counter()\n}"; assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret(code, context).code()); } @@ -116,7 +122,8 @@ public void testBasicIntp() { public void testBasicScalding() { assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("case class Sale(state: String, name: String, sale: Int)\n" + - "val salesList = List(Sale(\"CA\", \"A\", 60), Sale(\"CA\", \"A\", 20), Sale(\"VA\", \"B\", 15))\n" + + "val salesList = List(Sale(\"CA\", \"A\", 60), Sale(\"CA\", \"A\", 20), " + + "Sale(\"VA\", \"B\", 15))\n" + "val salesPipe = TypedPipe.from(salesList)\n" + "val results = salesPipe.map{x => (1, Set(x.state), x.sale)}.\n" + " groupAll.sum.values.map{ case(count, set, sum) => (count, set.size, sum) }\n" + @@ -131,7 +138,8 @@ public void testNextLineInvocation() { @Test public void testEndWithComment() { - assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("val c=1\n//comment", context).code()); + assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("val c=1\n//comment", + context).code()); } @Test From a791fad5970905edd07bdb8afcc00497fb3540f5 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 20 Feb 2018 16:58:47 +0800 Subject: [PATCH 043/386] ZEPPELIN-3239. unicode characters in an iPython paragraph makes Spark interpreter irresponsive ### What is this PR for? Fix the unicode issues in python2. ### What type of PR is it? [Bug Fix | Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3239 ### How should this be tested? * Unit test is added ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2810 from zjffdu/ZEPPELIN-3239 and squashes the following commits: 70c8b4c [Jeff Zhang] ZEPPELIN-3239. unicode characters in an iPython paragraph makes Spark interpreter irrsponsive --- .../resources/grpc/python/ipython_server.py | 3 +-- .../python/IPythonInterpreterTest.java | 21 ++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/python/src/main/resources/grpc/python/ipython_server.py b/python/src/main/resources/grpc/python/ipython_server.py index 98fa616c2d0..4b68efdf274 100644 --- a/python/src/main/resources/grpc/python/ipython_server.py +++ b/python/src/main/resources/grpc/python/ipython_server.py @@ -27,7 +27,6 @@ _ONE_DAY_IN_SECONDS = 60 * 60 * 24 - is_py2 = sys.version[0] == '2' if is_py2: import Queue as queue @@ -51,7 +50,7 @@ def start(self): def execute(self, request, context): print("execute code:\n") - print(request.code) + print(request.code.encode('utf-8')) sys.stdout.flush() stdout_queue = queue.Queue(maxsize = 10) stderr_queue = queue.Queue(maxsize = 10) diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index cb854d65704..ec594828f50 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -81,18 +81,33 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep InterpreterResult result = interpreter.interpret("from __future__ import print_function", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - result = interpreter.interpret("import sys\nprint(sys.version_info)", getInterpreterContext()); + + InterpreterContext context = getInterpreterContext(); + result = interpreter.interpret("import sys\nprint(sys.version[0])", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + Thread.sleep(100); + List interpreterResultMessages = context.out.getInterpreterResultMessages(); + assertEquals(1, interpreterResultMessages.size()); + boolean isPython2 = interpreterResultMessages.get(0).getData().equals("2\n"); // single output without print - InterpreterContext context = getInterpreterContext(); + context = getInterpreterContext(); result = interpreter.interpret("'hello world'", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - List interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.getInterpreterResultMessages(); assertEquals(1, interpreterResultMessages.size()); assertEquals("'hello world'", interpreterResultMessages.get(0).getData()); + // unicode + context = getInterpreterContext(); + result = interpreter.interpret("print(u'你好')", context); + Thread.sleep(100); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + interpreterResultMessages = context.out.getInterpreterResultMessages(); + assertEquals(1, interpreterResultMessages.size()); + assertEquals("你好\n", interpreterResultMessages.get(0).getData()); + // only the last statement is printed context = getInterpreterContext(); result = interpreter.interpret("'hello world'\n'hello world2'", context); From de03a21ba62084a37fd4ef9bba8c00e33b6644cb Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Thu, 15 Feb 2018 10:30:10 +0800 Subject: [PATCH 044/386] ZEPPELIN-3236. Make grpc framesize configurable ### What is this PR for? Add one new property `zeppelin.ipython.grpc.framesize` which is an advanced configuration. By default it is 32M which should be sufficient for most of scenarios. ### What type of PR is it? [ Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3236 ### How should this be tested? * Unit test is added. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2802 from zjffdu/ZEPPELIN-3236 and squashes the following commits: ffce774 [Jeff Zhang] ZEPPELIN-3236. Make grpc framesize configurable --- .../apache/zeppelin/python/IPythonClient.java | 8 ++++ .../zeppelin/python/IPythonInterpreter.java | 6 ++- .../main/resources/interpreter-setting.json | 6 +++ .../python/IPythonInterpreterTest.java | 40 +++++++++++++++++-- 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java index ac1020498bc..b3bc7fd7d2c 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java @@ -20,6 +20,7 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.stub.StreamObserver; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.util.InterpreterOutputStream; import org.apache.zeppelin.python.proto.CancelRequest; import org.apache.zeppelin.python.proto.CancelResponse; @@ -131,11 +132,18 @@ public void onNext(ExecuteResponse executeResponse) { @Override public void onError(Throwable throwable) { try { + interpreterOutput.getInterpreterOutput().write(ExceptionUtils.getStackTrace(throwable)); interpreterOutput.getInterpreterOutput().flush(); } catch (IOException e) { LOGGER.error("Unexpected IOException", e); } LOGGER.error("Fail to call IPython grpc", throwable); + finalResponseBuilder.setStatus(ExecuteStatus.ERROR); + + completedFlag.set(true); + synchronized (completedFlag) { + completedFlag.notify(); + } } @Override diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java index 8078670f8ac..10bf530b4ae 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.python; +import io.grpc.ManagedChannelBuilder; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.ExecuteException; @@ -142,7 +143,10 @@ public void open() throws InterpreterException { int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); LOGGER.info("Launching IPython Kernel at port: " + ipythonPort); LOGGER.info("Launching JVM Gateway at port: " + jvmGatewayPort); - ipythonClient = new IPythonClient("127.0.0.1", ipythonPort); + int framesize = Integer.parseInt(getProperty("zeppelin.ipython.grpc.framesize", + 32 * 1024 * 1024 + "")); + ipythonClient = new IPythonClient(ManagedChannelBuilder.forAddress("127.0.0.1", ipythonPort) + .usePlaintext(true).maxInboundMessageSize(framesize)); launchIPythonKernel(ipythonPort); setupJVMGateway(jvmGatewayPort); } catch (Exception e) { diff --git a/python/src/main/resources/interpreter-setting.json b/python/src/main/resources/interpreter-setting.json index d6b35380385..3257e58abfb 100644 --- a/python/src/main/resources/interpreter-setting.json +++ b/python/src/main/resources/interpreter-setting.json @@ -40,6 +40,12 @@ "defaultValue": "30000", "description": "time out for ipython launch", "type": "number" + }, + "zeppelin.ipython.grpc.framesize": { + "propertyName": "zeppelin.ipython.grpc.framesize", + "defaultValue": "33554432", + "description": "grpc framesize, default is 32M", + "type": "number" } }, "editor": { diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index ec594828f50..dfc8c36b74a 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -56,9 +56,7 @@ public class IPythonInterpreterTest { private static final Logger LOGGER = LoggerFactory.getLogger(IPythonInterpreterTest.class); private IPythonInterpreter interpreter; - @Before - public void setUp() throws InterpreterException { - Properties properties = new Properties(); + public void startInterpreter(Properties properties) throws InterpreterException { interpreter = new IPythonInterpreter(properties); InterpreterGroup mockInterpreterGroup = mock(InterpreterGroup.class); interpreter.setInterpreterGroup(mockInterpreterGroup); @@ -73,9 +71,45 @@ public void close() throws InterpreterException { @Test public void testIPython() throws IOException, InterruptedException, InterpreterException { + startInterpreter(new Properties()); testInterpreter(interpreter); } + @Test + public void testGrpcFrameSize() throws InterpreterException, IOException { + Properties properties = new Properties(); + properties.setProperty("zeppelin.ipython.grpc.framesize", "4"); + startInterpreter(properties); + + // to make this test can run under both python2 and python3 + InterpreterResult result = interpreter.interpret("from __future__ import print_function", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + InterpreterContext context = getInterpreterContext(); + result = interpreter.interpret("print(11111111111111111111111111111)", context); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + List interpreterResultMessages = context.out.getInterpreterResultMessages(); + assertEquals(1, interpreterResultMessages.size()); + assertTrue(interpreterResultMessages.get(0).getData().contains("Frame size 32 exceeds maximum: 4")); + + // next call continue work + result = interpreter.interpret("print(1)", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + close(); + + // increase framesize to make it work + properties.setProperty("zeppelin.ipython.grpc.framesize", "40"); + startInterpreter(properties); + // to make this test can run under both python2 and python3 + result = interpreter.interpret("from __future__ import print_function", getInterpreterContext()); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + context = getInterpreterContext(); + result = interpreter.interpret("print(11111111111111111111111111111)", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + } + public static void testInterpreter(final Interpreter interpreter) throws IOException, InterruptedException, InterpreterException { // to make this test can run under both python2 and python3 InterpreterResult result = interpreter.interpret("from __future__ import print_function", getInterpreterContext()); From f6ef64f8470e30f2e166ee31d468174f0df0178a Mon Sep 17 00:00:00 2001 From: sravan Date: Mon, 22 Jan 2018 12:27:41 +0900 Subject: [PATCH 045/386] [ZEPPELIN-3177]Resize charts on paragaph resize ### What is this PR for? Resize charts on paragraph resize * Broadcast chart resize on para. resize with a timeout * Add warning on refresh missing ### What type of PR is it? [Bug Fix | Improvement] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN/ZEPPELIN-3177 ### How should this be tested? Open a paragraph with charts and resize paragraph width(see the gif) ps- helium charts should be updated accordingly ### Screenshots (if appropriate) Before: ![zeppelin3](https://user-images.githubusercontent.com/11382805/35181438-ec771338-fe04-11e7-8803-a6b3aa15b149.gif) After: ![zeppelin3](https://user-images.githubusercontent.com/11382805/35181425-9623d962-fe04-11e7-8660-8dc82c54cd0e.gif) ### Questions: * Does the licenses files need update? N * Is there breaking changes for older versions? N * Does this needs documentation? N Author: sravan Closes #2735 from sravan-s/fix/resize-chart and squashes the following commits: 2f2deecff [sravan] Activate app after refresh 9bf989496 [sravan] Resize charts on paragraph resize --- .../src/app/notebook/paragraph/paragraph.controller.js | 1 + .../src/app/notebook/paragraph/result/result.controller.js | 3 ++- zeppelin-web/src/app/visualization/visualization.js | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 75a0fecac3b..07ebf896dd0 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -673,6 +673,7 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca $scope.changeColWidth = function (paragraph, width) { angular.element('.navbar-right.open').removeClass('open') paragraph.config.colWidth = width + $scope.$broadcast('paragraphResized', $scope.paragraph.id) commitParagraph(paragraph) } diff --git a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js index ec4eeda0230..5dfe3143971 100644 --- a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js @@ -634,6 +634,7 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio builtInViz.instance.setConfig(config) builtInViz.instance.render(transformed) builtInViz.instance.renderSetting(visualizationSettingTargetEl) + builtInViz.instance.activate() } } else { afterLoaded = function (loadedElem) { @@ -755,7 +756,7 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio if (paragraphId === paragraph.id) { let builtInViz = builtInVisualizations[$scope.graphMode] if (builtInViz && builtInViz.instance) { - builtInViz.instance.resize() + $timeout(_ => builtInViz.instance.resize(), 200) } } }) diff --git a/zeppelin-web/src/app/visualization/visualization.js b/zeppelin-web/src/app/visualization/visualization.js index 82704e3a00b..6b6e36aa387 100644 --- a/zeppelin-web/src/app/visualization/visualization.js +++ b/zeppelin-web/src/app/visualization/visualization.js @@ -48,6 +48,7 @@ export default class Visualization { */ refresh () { // override this + console.warn('A chart is missing refresh function, it might not work preperly') } /** From ea2c944742cea6e8e37e225d1acc67cdb195056e Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Fri, 23 Feb 2018 09:35:30 +0530 Subject: [PATCH 046/386] [ZEPPELIN-3245] checkstyle/eslintrc for zeppelin-web (JavaScript) Have added this PR to add a rule in eslinerc to have semicolons in javascript source [Improvement | Refactoring] * [ZEPPELIN-3245](https://issues.apache.org/jira/browse/ZEPPELIN-3245) ``` cd zeppelin-web npm install (or yarn install if you have yarn) npm run lint:once ``` Author: Prabhjyot Singh Closes #2804 from prabhjyotsingh/discuss/eslint_semi_rule and squashes the following commits: 4506f243e [Prabhjyot Singh] eslint rule for space bc43d68a6 [Prabhjyot Singh] merge `[ZEPPELIN-3177]Resize charts on paragaph resize` changes 2d57ba30b [Prabhjyot Singh] fix failing WEB_E2E="true" f23cb61d4 [Prabhjyot Singh] remove `"linebreak-style": 0,` and `"no-use-before-define": 0,` 39f37fb88 [Prabhjyot Singh] remove "standard" from eslint 6edac44f6 [Prabhjyot Singh] add `"semi": [2, "always"]` rule in eslinerc Change-Id: I91546ea973c2c9e7540da1586d6329fc93088eb0 --- .../apache/zeppelin/AbstractZeppelinIT.java | 2 +- zeppelin-web/.eslintrc | 24 +- zeppelin-web/src/app/app.controller.js | 52 +- zeppelin-web/src/app/app.controller.test.js | 44 +- zeppelin-web/src/app/app.js | 162 +- .../configuration/configuration.controller.js | 42 +- .../app/configuration/configuration.test.js | 92 +- .../app/credential/credential.controller.js | 236 +- .../src/app/credential/credential.test.js | 132 +- zeppelin-web/src/app/helium/helium-conf.js | 96 +- zeppelin-web/src/app/helium/helium-package.js | 32 +- zeppelin-web/src/app/helium/helium-type.js | 2 +- .../src/app/helium/helium.controller.js | 424 +-- zeppelin-web/src/app/helium/helium.service.js | 334 +-- zeppelin-web/src/app/helium/index.js | 4 +- zeppelin-web/src/app/home/home.controller.js | 188 +- .../interpreter/interpreter-item.directive.js | 20 +- .../app/interpreter/interpreter.controller.js | 838 +++--- .../src/app/interpreter/interpreter.filter.js | 12 +- .../widget/number-widget.directive.js | 20 +- zeppelin-web/src/app/jobmanager/job-status.js | 26 +- .../src/app/jobmanager/job/job.component.js | 118 +- .../app/jobmanager/job/job.component.test.js | 70 +- .../app/jobmanager/jobmanager.component.js | 177 +- .../jobmanager/jobmanager.component.test.js | 34 +- .../src/app/jobmanager/jobmanager.filter.js | 38 +- .../src/app/jobmanager/jobmanager.service.js | 46 +- .../app/jobmanager/jobmanager.service.test.js | 66 +- .../notebook-repository.controller.js | 78 +- .../dropdown-input.directive.js | 14 +- .../dynamic-forms/dynamic-forms.directive.js | 42 +- .../elastic-input/elastic-input.controller.js | 10 +- .../app/notebook/note-var-share.service.js | 36 +- .../src/app/notebook/notebook.controller.js | 1590 ++++++------ .../app/notebook/notebook.controller.test.js | 240 +- .../paragraph/clipboard.controller.js | 36 +- .../code-editor/code-editor.directive.js | 24 +- .../paragraph/paragraph.controller.js | 1824 ++++++------- .../paragraph/paragraph.controller.test.js | 68 +- .../notebook/paragraph/paragraph.status.js | 16 +- .../notebook/paragraph/resizable.directive.js | 66 +- .../paragraph/result/result.controller.js | 1118 ++++---- .../revisions-comparator.component.js | 162 +- .../save-as/browser-detect.service.js | 26 +- .../app/notebook/save-as/save-as.service.js | 60 +- .../src/app/search/result-list.controller.js | 134 +- zeppelin-web/src/app/search/search.service.js | 24 +- zeppelin-web/src/app/spell/index.js | 4 +- zeppelin-web/src/app/spell/spell-base.js | 14 +- zeppelin-web/src/app/spell/spell-result.js | 182 +- .../tabledata/advanced-transformation-util.js | 1010 ++++---- .../advanced-transformation-util.test.js | 2288 +++++++++-------- .../app/tabledata/advanced-transformation.js | 202 +- .../src/app/tabledata/columnselector.js | 54 +- zeppelin-web/src/app/tabledata/dataset.js | 6 +- .../src/app/tabledata/datasetfactory.js | 12 +- .../src/app/tabledata/datasetfactory.test.js | 46 +- zeppelin-web/src/app/tabledata/network.js | 22 +- zeppelin-web/src/app/tabledata/networkdata.js | 76 +- .../src/app/tabledata/networkdata.test.js | 72 +- zeppelin-web/src/app/tabledata/passthrough.js | 10 +- zeppelin-web/src/app/tabledata/pivot.js | 240 +- zeppelin-web/src/app/tabledata/tabledata.js | 64 +- .../src/app/tabledata/tabledata.test.js | 132 +- .../src/app/tabledata/transformation.js | 68 +- .../builtins/visualization-areachart.js | 144 +- .../builtins/visualization-barchart.js | 122 +- .../builtins/visualization-d3network.js | 256 +- .../builtins/visualization-linechart.js | 162 +- .../builtins/visualization-nvd3chart.js | 242 +- .../builtins/visualization-piechart.js | 66 +- .../builtins/visualization-scatterchart.js | 326 +-- .../builtins/visualization-table.js | 318 +-- .../builtins/visualization-util.js | 146 +- .../src/app/visualization/visualization.js | 108 +- .../array-ordering/array-ordering.service.js | 44 +- .../components/base-url/base-url.service.js | 40 +- .../src/components/login/login.controller.js | 88 +- .../expand-collapse.directive.js | 30 +- .../components/navbar/navbar.controller.js | 258 +- .../navbar/navbar.controller.test.js | 28 +- .../components/ng-enter/ng-enter.directive.js | 18 +- .../ng-enter/ng-enter.directive.test.js | 26 +- .../ng-escape/ng-escape.directive.js | 18 +- .../note-action/note-action.service.js | 144 +- .../note-create/note-create.controller.js | 120 +- .../note-create.controller.test.js | 54 +- .../note-create/visible.directive.js | 40 +- .../note-import/note-import.controller.js | 176 +- .../components/note-list/note-list.factory.js | 62 +- .../note-list/note-list.factory.test.js | 122 +- .../note-rename/note-rename.controller.js | 44 +- .../note-rename/note-rename.service.js | 12 +- .../websocket/websocket-event.factory.js | 204 +- .../websocket/websocket-message.service.js | 292 +-- zeppelin-web/src/index.js | 118 +- 96 files changed, 8777 insertions(+), 8452 deletions(-) diff --git a/zeppelin-integration/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java b/zeppelin-integration/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java index b4ebfe9668e..e1992fb4d92 100644 --- a/zeppelin-integration/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java +++ b/zeppelin-integration/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java @@ -110,7 +110,7 @@ protected void createNewNote() { WebDriverWait block = new WebDriverWait(driver, MAX_BROWSER_TIMEOUT_SEC); block.until(ExpectedConditions.visibilityOfElementLocated(By.id("noteCreateModal"))); clickAndWait(By.id("createNoteButton")); - block.until(ExpectedConditions.invisibilityOfElementLocated(By.className("pull-right"))); + block.until(ExpectedConditions.invisibilityOfElementLocated(By.id("createNoteButton"))); } protected void deleteTestNotebook(final WebDriver driver) { diff --git a/zeppelin-web/.eslintrc b/zeppelin-web/.eslintrc index 6dca5c8982b..6207bb9de8c 100644 --- a/zeppelin-web/.eslintrc +++ b/zeppelin-web/.eslintrc @@ -1,5 +1,5 @@ { - "extends": ["eslint:recommended", "google", "standard"], + "extends": ["eslint:recommended", "google"], "env": { "browser": true, "jasmine": true, @@ -31,26 +31,11 @@ "process": false }, "rules": { - "array-bracket-spacing": 0, - "space-before-function-paren": 0, - "no-unneeded-ternary": 0, - "comma-dangle": 0, - "object-curly-spacing": 0, - "standard/object-curly-even-spacing": 0, - "arrow-parens": 0, - "require-jsdoc": 0, - "valid-jsdoc": 0, - "no-invalid-this": 0, - "no-console": 0, - "guard-for-in": 0, - "no-mixed-operators": 1, - "no-useless-escape": 1, "no-bitwise": 2, "camelcase": 2, "curly": 2, "eqeqeq": 2, "wrap-iife": [2, "any"], - "no-use-before-define": 0, "new-cap": 2, "no-caller": 2, "quotes": [2, "single"], @@ -59,6 +44,11 @@ "no-unused-vars": [2, { "vars": "local", "args": "none" }], "strict": [2, "global"], "max-len": [2, {"code": 120, "ignoreComments": true, "ignoreRegExpLiterals": true}], - "linebreak-style": 0 + "require-jsdoc": "off", + "no-console": ["off"], + "valid-jsdoc": "off", + "semi": [2, "always"], + "no-invalid-this": 1, + "indent": ["error", 2, { "SwitchCase": 1 }] } } diff --git a/zeppelin-web/src/app/app.controller.js b/zeppelin-web/src/app/app.controller.js index 6c64a33d180..904fbd797e4 100644 --- a/zeppelin-web/src/app/app.controller.js +++ b/zeppelin-web/src/app/app.controller.js @@ -12,48 +12,48 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').controller('MainCtrl', MainCtrl) +angular.module('zeppelinWebApp').controller('MainCtrl', MainCtrl); -function MainCtrl ($scope, $rootScope, $window, arrayOrderingSrv) { - 'ngInject' +function MainCtrl($scope, $rootScope, $window, arrayOrderingSrv) { + 'ngInject'; - $scope.looknfeel = 'default' + $scope.looknfeel = 'default'; - let init = function () { - $scope.asIframe = (($window.location.href.indexOf('asIframe') > -1) ? true : false) - } + let init = function() { + $scope.asIframe = (($window.location.href.indexOf('asIframe') > -1) ? true : false); + }; - init() + init(); - $rootScope.$on('setIframe', function (event, data) { + $rootScope.$on('setIframe', function(event, data) { if (!event.defaultPrevented) { - $scope.asIframe = data - event.preventDefault() + $scope.asIframe = data; + event.preventDefault(); } - }) + }); - $rootScope.$on('setLookAndFeel', function (event, data) { + $rootScope.$on('setLookAndFeel', function(event, data) { if (!event.defaultPrevented && data && data !== '' && data !== $scope.looknfeel) { - $scope.looknfeel = data - event.preventDefault() + $scope.looknfeel = data; + event.preventDefault(); } - }) + }); // Set The lookAndFeel to default on every page - $rootScope.$on('$routeChangeStart', function (event, next, current) { - $rootScope.$broadcast('setLookAndFeel', 'default') - }) + $rootScope.$on('$routeChangeStart', function(event, next, current) { + $rootScope.$broadcast('setLookAndFeel', 'default'); + }); - $rootScope.noteName = function (note) { + $rootScope.noteName = function(note) { if (!_.isEmpty(note)) { - return arrayOrderingSrv.getNoteName(note) + return arrayOrderingSrv.getNoteName(note); } - } + }; - BootstrapDialog.defaultOptions.onshown = function () { - angular.element('#' + this.id).find('.btn:last').focus() - } + BootstrapDialog.defaultOptions.onshown = function() { + angular.element('#' + this.id).find('.btn:last').focus(); + }; // Remove BootstrapDialog animation - BootstrapDialog.configDefaultOptions({animate: false}) + BootstrapDialog.configDefaultOptions({animate: false}); } diff --git a/zeppelin-web/src/app/app.controller.test.js b/zeppelin-web/src/app/app.controller.test.js index 67d50342946..b6c6261fe49 100644 --- a/zeppelin-web/src/app/app.controller.test.js +++ b/zeppelin-web/src/app/app.controller.test.js @@ -1,28 +1,28 @@ -describe('Controller: MainCtrl', function () { - beforeEach(angular.mock.module('zeppelinWebApp')) +describe('Controller: MainCtrl', function() { + beforeEach(angular.mock.module('zeppelinWebApp')); - let scope - let rootScope + let scope; + let rootScope; - beforeEach(inject(function ($controller, $rootScope) { - rootScope = $rootScope - scope = $rootScope.$new() + beforeEach(inject(function($controller, $rootScope) { + rootScope = $rootScope; + scope = $rootScope.$new(); $controller('MainCtrl', { - $scope: scope - }) - })) + $scope: scope, + }); + })); - it('should attach "asIframe" to the scope and the default value should be false', function () { - expect(scope.asIframe).toBeDefined() - expect(scope.asIframe).toEqual(false) - }) + it('should attach "asIframe" to the scope and the default value should be false', function() { + expect(scope.asIframe).toBeDefined(); + expect(scope.asIframe).toEqual(false); + }); - it('should set the default value of "looknfeel to "default"', function () { - expect(scope.looknfeel).toEqual('default') - }) + it('should set the default value of "looknfeel to "default"', function() { + expect(scope.looknfeel).toEqual('default'); + }); - it('should set "asIframe" flag to true when a controller broadcasts setIframe event', function () { - rootScope.$broadcast('setIframe', true) - expect(scope.asIframe).toEqual(true) - }) -}) + it('should set "asIframe" flag to true when a controller broadcasts setIframe event', function() { + rootScope.$broadcast('setIframe', true); + expect(scope.asIframe).toEqual(true); + }); +}); diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index ed89dd8abe9..64ceff00d79 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -15,14 +15,14 @@ * limitations under the License. */ -import 'headroom.js' -import 'headroom.js/dist/angular.headroom' +import 'headroom.js'; +import 'headroom.js/dist/angular.headroom'; -import 'scrollmonitor/scrollMonitor.js' -import 'angular-viewport-watch/angular-viewport-watch.js' +import 'scrollmonitor/scrollMonitor.js'; +import 'angular-viewport-watch/angular-viewport-watch.js'; -import 'angular-ui-grid/ui-grid.css' -import 'angular-ui-grid' +import 'angular-ui-grid/ui-grid.css'; +import 'angular-ui-grid'; const requiredModules = [ 'ngCookies', @@ -56,167 +56,169 @@ const requiredModules = [ 'ui.grid.moveColumns', 'ui.grid.pagination', 'ui.grid.saveState', -] +]; // headroom should not be used for CI, since we have to execute some integration tests. // otherwise, they will fail. -if (!process.env.BUILD_CI) { requiredModules.push('headroom') } +if (!process.env.BUILD_CI) { + requiredModules.push('headroom'); +} let zeppelinWebApp = angular.module('zeppelinWebApp', requiredModules) - .filter('breakFilter', function () { - return function (text) { + .filter('breakFilter', function() { + return function(text) { // eslint-disable-next-line no-extra-boolean-cast if (!!text) { - return text.replace(/\n/g, '
') + return text.replace(/\n/g, '
'); } - } + }; }) - .config(function ($httpProvider, $routeProvider, ngToastProvider) { + .config(function($httpProvider, $routeProvider, ngToastProvider) { // withCredentials when running locally via grunt - $httpProvider.defaults.withCredentials = true + $httpProvider.defaults.withCredentials = true; let visBundleLoad = { - load: ['heliumService', function (heliumService) { - return heliumService.load - }] - } + load: ['heliumService', function(heliumService) { + return heliumService.load; + }], + }; $routeProvider .when('/', { - templateUrl: 'app/home/home.html' + templateUrl: 'app/home/home.html', }) .when('/notebook/:noteId', { templateUrl: 'app/notebook/notebook.html', controller: 'NotebookCtrl', - resolve: visBundleLoad + resolve: visBundleLoad, }) .when('/notebook/:noteId/paragraph?=:paragraphId', { templateUrl: 'app/notebook/notebook.html', controller: 'NotebookCtrl', - resolve: visBundleLoad + resolve: visBundleLoad, }) .when('/notebook/:noteId/paragraph/:paragraphId?', { templateUrl: 'app/notebook/notebook.html', controller: 'NotebookCtrl', - resolve: visBundleLoad + resolve: visBundleLoad, }) .when('/notebook/:noteId/revision/:revisionId', { templateUrl: 'app/notebook/notebook.html', controller: 'NotebookCtrl', - resolve: visBundleLoad + resolve: visBundleLoad, }) .when('/jobmanager', { templateUrl: 'app/jobmanager/jobmanager.html', - controller: 'JobManagerCtrl' + controller: 'JobManagerCtrl', }) .when('/interpreter', { templateUrl: 'app/interpreter/interpreter.html', - controller: 'InterpreterCtrl' + controller: 'InterpreterCtrl', }) .when('/notebookRepos', { templateUrl: 'app/notebook-repository/notebook-repository.html', controller: 'NotebookRepositoryCtrl', - controllerAs: 'noterepo' + controllerAs: 'noterepo', }) .when('/credential', { templateUrl: 'app/credential/credential.html', - controller: 'CredentialCtrl' + controller: 'CredentialCtrl', }) .when('/helium', { templateUrl: 'app/helium/helium.html', - controller: 'HeliumCtrl' + controller: 'HeliumCtrl', }) .when('/configuration', { templateUrl: 'app/configuration/configuration.html', - controller: 'ConfigurationCtrl' + controller: 'ConfigurationCtrl', }) .when('/search/:searchTerm', { templateUrl: 'app/search/result-list.html', - controller: 'SearchResultCtrl' + controller: 'SearchResultCtrl', }) .otherwise({ - redirectTo: '/' - }) + redirectTo: '/', + }); ngToastProvider.configure({ dismissButton: true, dismissOnClick: false, combineDuplications: true, - timeout: 6000 - }) + timeout: 6000, + }); }) // handel logout on API failure - .config(function ($httpProvider, $provide) { + .config(function($httpProvider, $provide) { if (process.env.PROD) { - $httpProvider.defaults.headers.common['X-Requested-With'] = 'XMLHttpRequest' + $httpProvider.defaults.headers.common['X-Requested-With'] = 'XMLHttpRequest'; } - $provide.factory('httpInterceptor', function ($q, $rootScope) { + $provide.factory('httpInterceptor', function($q, $rootScope) { return { - 'responseError': function (rejection) { + 'responseError': function(rejection) { if (rejection.status === 405) { - let data = {} - data.info = '' - $rootScope.$broadcast('session_logout', data) + let data = {}; + data.info = ''; + $rootScope.$broadcast('session_logout', data); } - $rootScope.$broadcast('httpResponseError', rejection) - return $q.reject(rejection) - } - } - }) - $httpProvider.interceptors.push('httpInterceptor') + $rootScope.$broadcast('httpResponseError', rejection); + return $q.reject(rejection); + }, + }; + }); + $httpProvider.interceptors.push('httpInterceptor'); }) - .constant('TRASH_FOLDER_ID', '~Trash') + .constant('TRASH_FOLDER_ID', '~Trash'); -function auth () { - let $http = angular.injector(['ng']).get('$http') - let baseUrlSrv = angular.injector(['zeppelinWebApp']).get('baseUrlSrv') +function auth() { + let $http = angular.injector(['ng']).get('$http'); + let baseUrlSrv = angular.injector(['zeppelinWebApp']).get('baseUrlSrv'); // withCredentials when running locally via grunt - $http.defaults.withCredentials = true + $http.defaults.withCredentials = true; jQuery.ajaxSetup({ dataType: 'json', xhrFields: { - withCredentials: true + withCredentials: true, }, - crossDomain: true - }) - let config = (process.env.PROD) ? {headers: { 'X-Requested-With': 'XMLHttpRequest' }} : {} - return $http.get(baseUrlSrv.getRestApiBase() + '/security/ticket', config).then(function (response) { - zeppelinWebApp.run(function ($rootScope) { - let res = angular.fromJson(response.data).body + crossDomain: true, + }); + let config = (process.env.PROD) ? {headers: {'X-Requested-With': 'XMLHttpRequest'}} : {}; + return $http.get(baseUrlSrv.getRestApiBase() + '/security/ticket', config).then(function(response) { + zeppelinWebApp.run(function($rootScope) { + let res = angular.fromJson(response.data).body; if (res['redirectURL']) { - window.location.href = res['redirectURL'] + window.location.href + window.location.href = res['redirectURL'] + window.location.href; } else { - $rootScope.ticket = res - $rootScope.ticket.screenUsername = $rootScope.ticket.principal + $rootScope.ticket = res; + $rootScope.ticket.screenUsername = $rootScope.ticket.principal; if ($rootScope.ticket.principal.indexOf('#Pac4j') === 0) { - let re = ', name=(.*?),' - $rootScope.ticket.screenUsername = $rootScope.ticket.principal.match(re)[1] + let re = ', name=(.*?),'; + $rootScope.ticket.screenUsername = $rootScope.ticket.principal.match(re)[1]; } } - }) - }, function (errorResponse) { + }); + }, function(errorResponse) { // Handle error case - let redirect = errorResponse.headers('Location') + let redirect = errorResponse.headers('Location'); if (errorResponse.status === 401 && redirect !== undefined) { // Handle page redirect - window.location.href = redirect + window.location.href = redirect; } - }) + }); } -function bootstrapApplication () { - zeppelinWebApp.run(function ($rootScope, $location) { - $rootScope.$on('$routeChangeStart', function (event, next, current) { - $rootScope.pageTitle = 'Zeppelin' +function bootstrapApplication() { + zeppelinWebApp.run(function($rootScope, $location) { + $rootScope.$on('$routeChangeStart', function(event, next, current) { + $rootScope.pageTitle = 'Zeppelin'; if (!$rootScope.ticket && next.$$route && !next.$$route.publicAccess) { - $location.path('/') + $location.path('/'); } - }) - }) - angular.bootstrap(document, ['zeppelinWebApp']) + }); + }); + angular.bootstrap(document, ['zeppelinWebApp']); } -angular.element(document).ready(function () { - auth().then(bootstrapApplication) -}) +angular.element(document).ready(function() { + auth().then(bootstrapApplication); +}); diff --git a/zeppelin-web/src/app/configuration/configuration.controller.js b/zeppelin-web/src/app/configuration/configuration.controller.js index 0d845ded83f..0f5eba339a6 100644 --- a/zeppelin-web/src/app/configuration/configuration.controller.js +++ b/zeppelin-web/src/app/configuration/configuration.controller.js @@ -12,37 +12,37 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').controller('ConfigurationCtrl', ConfigurationCtrl) +angular.module('zeppelinWebApp').controller('ConfigurationCtrl', ConfigurationCtrl); -function ConfigurationCtrl ($scope, $http, baseUrlSrv, ngToast) { - 'ngInject' +function ConfigurationCtrl($scope, $http, baseUrlSrv, ngToast) { + 'ngInject'; - $scope.configrations = [] - ngToast.dismiss() + $scope.configrations = []; + ngToast.dismiss(); - let getConfigurations = function () { + let getConfigurations = function() { $http.get(baseUrlSrv.getRestApiBase() + '/configurations/all') - .success(function (data, status, headers, config) { - $scope.configurations = data.body + .success(function(data, status, headers, config) { + $scope.configurations = data.body; }) - .error(function (data, status, headers, config) { + .error(function(data, status, headers, config) { if (status === 401) { ngToast.danger({ content: 'You don\'t have permission on this page', verticalPosition: 'bottom', - timeout: '3000' - }) - setTimeout(function () { - window.location = baseUrlSrv.getBase() - }, 3000) + timeout: '3000', + }); + setTimeout(function() { + window.location = baseUrlSrv.getBase(); + }, 3000); } - console.log('Error %o %o', status, data.message) - }) - } + console.log('Error %o %o', status, data.message); + }); + }; - let init = function () { - getConfigurations() - } + let init = function() { + getConfigurations(); + }; - init() + init(); } diff --git a/zeppelin-web/src/app/configuration/configuration.test.js b/zeppelin-web/src/app/configuration/configuration.test.js index 8add1029f7b..4d98a08a533 100644 --- a/zeppelin-web/src/app/configuration/configuration.test.js +++ b/zeppelin-web/src/app/configuration/configuration.test.js @@ -1,69 +1,69 @@ -import template from './configuration.html' +import template from './configuration.html'; -describe('Controller: Configuration', function () { - beforeEach(angular.mock.module('zeppelinWebApp')) +describe('Controller: Configuration', function() { + beforeEach(angular.mock.module('zeppelinWebApp')); - let baseUrlSrvMock = { getRestApiBase: () => '' } + let baseUrlSrvMock = {getRestApiBase: () => ''}; - let ctrl // controller instance - let $scope - let $compile - let $controller // controller generator - let $httpBackend - let ngToast + let ctrl; // controller instance + let $scope; + let $compile; + let $controller; // controller generator + let $httpBackend; + let ngToast; beforeEach(inject((_$controller_, _$rootScope_, _$compile_, _$httpBackend_, _ngToast_) => { - $scope = _$rootScope_.$new() - $compile = _$compile_ - $controller = _$controller_ - $httpBackend = _$httpBackend_ - ngToast = _ngToast_ - })) + $scope = _$rootScope_.$new(); + $compile = _$compile_; + $controller = _$controller_; + $httpBackend = _$httpBackend_; + ngToast = _ngToast_; + })); - afterEach(function () { - $httpBackend.verifyNoOutstandingExpectation() - $httpBackend.verifyNoOutstandingRequest() - }) + afterEach(function() { + $httpBackend.verifyNoOutstandingExpectation(); + $httpBackend.verifyNoOutstandingRequest(); + }); it('should get configuration initially', () => { - const conf = { 'conf1': 'value1' } - ctrl = $controller('ConfigurationCtrl', { $scope: $scope, baseUrlSrv: baseUrlSrvMock, }) - expect(ctrl).toBeDefined() + const conf = {'conf1': 'value1'}; + ctrl = $controller('ConfigurationCtrl', {$scope: $scope, baseUrlSrv: baseUrlSrvMock}); + expect(ctrl).toBeDefined(); $httpBackend .when('GET', '/configurations/all') - .respond(200, { body: conf, }) - $httpBackend.expectGET('/configurations/all') - $httpBackend.flush() + .respond(200, {body: conf}); + $httpBackend.expectGET('/configurations/all'); + $httpBackend.flush(); - expect($scope.configurations).toEqual(conf) // scope is updated after $httpBackend.flush() - }) + expect($scope.configurations).toEqual(conf); // scope is updated after $httpBackend.flush() + }); it('should display ngToast when failed to get configuration properly', () => { - ctrl = $controller('ConfigurationCtrl', { $scope: $scope, baseUrlSrv: baseUrlSrvMock, }) - spyOn(ngToast, 'danger') + ctrl = $controller('ConfigurationCtrl', {$scope: $scope, baseUrlSrv: baseUrlSrvMock}); + spyOn(ngToast, 'danger'); - $httpBackend.when('GET', '/configurations/all').respond(401, {}) - $httpBackend.expectGET('/configurations/all') - $httpBackend.flush() + $httpBackend.when('GET', '/configurations/all').respond(401, {}); + $httpBackend.expectGET('/configurations/all'); + $httpBackend.flush(); - expect(ngToast.danger).toHaveBeenCalled() - }) + expect(ngToast.danger).toHaveBeenCalled(); + }); it('should render list of configurations as the sorted order', () => { $scope.configurations = { 'zeppelin.server.port': '8080', 'zeppelin.server.addr': '0.0.0.0', - } - const elem = $compile(template)($scope) - $scope.$digest() - const tbody = elem.find('tbody') - const tds = tbody.find('td') + }; + const elem = $compile(template)($scope); + $scope.$digest(); + const tbody = elem.find('tbody'); + const tds = tbody.find('td'); // should be sorted - expect(tds[0].innerText.trim()).toBe('zeppelin.server.addr') - expect(tds[1].innerText.trim()).toBe('0.0.0.0') - expect(tds[2].innerText.trim()).toBe('zeppelin.server.port') - expect(tds[3].innerText.trim()).toBe('8080') - }) -}) + expect(tds[0].innerText.trim()).toBe('zeppelin.server.addr'); + expect(tds[1].innerText.trim()).toBe('0.0.0.0'); + expect(tds[2].innerText.trim()).toBe('zeppelin.server.port'); + expect(tds[3].innerText.trim()).toBe('8080'); + }); +}); diff --git a/zeppelin-web/src/app/credential/credential.controller.js b/zeppelin-web/src/app/credential/credential.controller.js index 102876e32c4..cf6c3405bd9 100644 --- a/zeppelin-web/src/app/credential/credential.controller.js +++ b/zeppelin-web/src/app/credential/credential.controller.js @@ -12,194 +12,196 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').controller('CredentialCtrl', CredentialController) +angular.module('zeppelinWebApp').controller('CredentialCtrl', CredentialController); function CredentialController($scope, $http, baseUrlSrv, ngToast) { - 'ngInject' + 'ngInject'; - ngToast.dismiss() + ngToast.dismiss(); - $scope.credentialInfo = [] - $scope.showAddNewCredentialInfo = false - $scope.availableInterpreters = [] + $scope.credentialInfo = []; + $scope.showAddNewCredentialInfo = false; + $scope.availableInterpreters = []; - $scope.entity = '' - $scope.password = '' - $scope.username = '' + $scope.entity = ''; + $scope.password = ''; + $scope.username = ''; $scope.hasCredential = () => { - return Array.isArray($scope.credentialInfo) && $scope.credentialInfo.length - } + return Array.isArray($scope.credentialInfo) && $scope.credentialInfo.length; + }; - let getCredentialInfo = function () { + let getCredentialInfo = function() { $http.get(baseUrlSrv.getRestApiBase() + '/credential') - .success(function (data, status, headers, config) { - $scope.credentialInfo.length = 0 // keep the ref while cleaning - const returnedCredentials = data.body.userCredentials + .success(function(data, status, headers, config) { + $scope.credentialInfo.length = 0; // keep the ref while cleaning + const returnedCredentials = data.body.userCredentials; for (let key in returnedCredentials) { - const value = returnedCredentials[key] - $scope.credentialInfo.push({ - entity: key, - password: value.password, - username: value.username, - }) + if (returnedCredentials.hasOwnProperty(key)) { + const value = returnedCredentials[key]; + $scope.credentialInfo.push({ + entity: key, + password: value.password, + username: value.username, + }); + } } - console.log('Success %o %o', status, $scope.credentialInfo) + console.log('Success %o %o', status, $scope.credentialInfo); }) - .error(function (data, status, headers, config) { + .error(function(data, status, headers, config) { if (status === 401) { - showToast('You do not have permission on this page', 'danger') - setTimeout(function () { - window.location = baseUrlSrv.getBase() - }, 3000) + showToast('You do not have permission on this page', 'danger'); + setTimeout(function() { + window.location = baseUrlSrv.getBase(); + }, 3000); } - console.log('Error %o %o', status, data.message) - }) - } + console.log('Error %o %o', status, data.message); + }); + }; $scope.isValidCredential = function() { - return $scope.entity.trim() !== '' && $scope.username.trim() !== '' - } + return $scope.entity.trim() !== '' && $scope.username.trim() !== ''; + }; - $scope.addNewCredentialInfo = function () { + $scope.addNewCredentialInfo = function() { if (!$scope.isValidCredential()) { - showToast('Username \\ Entity can not be empty.', 'danger') - return + showToast('Username \\ Entity can not be empty.', 'danger'); + return; } let newCredential = { 'entity': $scope.entity, 'username': $scope.username, - 'password': $scope.password - } + 'password': $scope.password, + }; $http.put(baseUrlSrv.getRestApiBase() + '/credential', newCredential) - .success(function (data, status, headers, config) { - showToast('Successfully saved credentials.', 'success') - $scope.credentialInfo.push(newCredential) - resetCredentialInfo() - $scope.showAddNewCredentialInfo = false - console.log('Success %o %o', status, data.message) + .success(function(data, status, headers, config) { + showToast('Successfully saved credentials.', 'success'); + $scope.credentialInfo.push(newCredential); + resetCredentialInfo(); + $scope.showAddNewCredentialInfo = false; + console.log('Success %o %o', status, data.message); }) - .error(function (data, status, headers, config) { - showToast('Error saving credentials', 'danger') - console.log('Error %o %o', status, data.message) - }) - } + .error(function(data, status, headers, config) { + showToast('Error saving credentials', 'danger'); + console.log('Error %o %o', status, data.message); + }); + }; - let getAvailableInterpreters = function () { + let getAvailableInterpreters = function() { $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/setting') - .success(function (data, status, headers, config) { + .success(function(data, status, headers, config) { for (let setting = 0; setting < data.body.length; setting++) { $scope.availableInterpreters.push( - data.body[setting].group + '.' + data.body[setting].name) + data.body[setting].group + '.' + data.body[setting].name); } angular.element('#entityname').autocomplete({ source: $scope.availableInterpreters, - select: function (event, selected) { - $scope.entity = selected.item.value - return false - } - }) + select: function(event, selected) { + $scope.entity = selected.item.value; + return false; + }, + }); }) - .error(function (data, status, headers, config) { - showToast(data.message, 'danger') - console.log('Error %o %o', status, data.message) - }) - } + .error(function(data, status, headers, config) { + showToast(data.message, 'danger'); + console.log('Error %o %o', status, data.message); + }); + }; - $scope.toggleAddNewCredentialInfo = function () { + $scope.toggleAddNewCredentialInfo = function() { if ($scope.showAddNewCredentialInfo) { - $scope.showAddNewCredentialInfo = false + $scope.showAddNewCredentialInfo = false; } else { - $scope.showAddNewCredentialInfo = true + $scope.showAddNewCredentialInfo = true; } - } + }; - $scope.cancelCredentialInfo = function () { - $scope.showAddNewCredentialInfo = false - resetCredentialInfo() - } + $scope.cancelCredentialInfo = function() { + $scope.showAddNewCredentialInfo = false; + resetCredentialInfo(); + }; - const resetCredentialInfo = function () { - $scope.entity = '' - $scope.username = '' - $scope.password = '' - } + const resetCredentialInfo = function() { + $scope.entity = ''; + $scope.username = ''; + $scope.password = ''; + }; - $scope.copyOriginCredentialsInfo = function () { - showToast('Since entity is a unique key, you can edit only username & password', 'info') - } + $scope.copyOriginCredentialsInfo = function() { + showToast('Since entity is a unique key, you can edit only username & password', 'info'); + }; - $scope.updateCredentialInfo = function (form, data, entity) { + $scope.updateCredentialInfo = function(form, data, entity) { if (!$scope.isValidCredential()) { - showToast('Username \\ Entity can not be empty.', 'danger') - return + showToast('Username \\ Entity can not be empty.', 'danger'); + return; } let credential = { entity: entity, username: data.username, - password: data.password - } + password: data.password, + }; $http.put(baseUrlSrv.getRestApiBase() + '/credential/', credential) - .success(function (data, status, headers, config) { - const index = $scope.credentialInfo.findIndex(elem => elem.entity === entity) - $scope.credentialInfo[index] = credential - return true - }) - .error(function (data, status, headers, config) { - showToast('We could not save the credential', 'danger') - console.log('Error %o %o', status, data.message) - form.$show() + .success(function(data, status, headers, config) { + const index = $scope.credentialInfo.findIndex((elem) => elem.entity === entity); + $scope.credentialInfo[index] = credential; + return true; }) - return false - } - - $scope.removeCredentialInfo = function (entity) { + .error(function(data, status, headers, config) { + showToast('We could not save the credential', 'danger'); + console.log('Error %o %o', status, data.message); + form.$show(); + }); + return false; + }; + + $scope.removeCredentialInfo = function(entity) { BootstrapDialog.confirm({ closable: false, closeByBackdrop: false, closeByKeyboard: false, title: '', message: 'Do you want to delete this credential information?', - callback: function (result) { + callback: function(result) { if (result) { $http.delete(baseUrlSrv.getRestApiBase() + '/credential/' + entity) - .success(function (data, status, headers, config) { - const index = $scope.credentialInfo.findIndex(elem => elem.entity === entity) - $scope.credentialInfo.splice(index, 1) - console.log('Success %o %o', status, data.message) - }) - .error(function (data, status, headers, config) { - showToast(data.message, 'danger') - console.log('Error %o %o', status, data.message) + .success(function(data, status, headers, config) { + const index = $scope.credentialInfo.findIndex((elem) => elem.entity === entity); + $scope.credentialInfo.splice(index, 1); + console.log('Success %o %o', status, data.message); }) + .error(function(data, status, headers, config) { + showToast(data.message, 'danger'); + console.log('Error %o %o', status, data.message); + }); } - } - }) - } + }, + }); + }; function showToast(message, type) { - const verticalPosition = 'bottom' - const timeout = '3000' + const verticalPosition = 'bottom'; + const timeout = '3000'; if (type === 'success') { - ngToast.success({ content: message, verticalPosition: verticalPosition, timeout: timeout, }) + ngToast.success({content: message, verticalPosition: verticalPosition, timeout: timeout}); } else if (type === 'info') { - ngToast.info({ content: message, verticalPosition: verticalPosition, timeout: timeout, }) + ngToast.info({content: message, verticalPosition: verticalPosition, timeout: timeout}); } else { - ngToast.danger({ content: message, verticalPosition: verticalPosition, timeout: timeout, }) + ngToast.danger({content: message, verticalPosition: verticalPosition, timeout: timeout}); } } - let init = function () { - getAvailableInterpreters() - getCredentialInfo() - } + let init = function() { + getAvailableInterpreters(); + getCredentialInfo(); + }; - init() + init(); } diff --git a/zeppelin-web/src/app/credential/credential.test.js b/zeppelin-web/src/app/credential/credential.test.js index d90567b65e4..2b3c17abb63 100644 --- a/zeppelin-web/src/app/credential/credential.test.js +++ b/zeppelin-web/src/app/credential/credential.test.js @@ -1,114 +1,114 @@ -describe('Controller: Credential', function () { - beforeEach(angular.mock.module('zeppelinWebApp')) +describe('Controller: Credential', function() { + beforeEach(angular.mock.module('zeppelinWebApp')); - let baseUrlSrvMock = { getRestApiBase: () => '' } + let baseUrlSrvMock = {getRestApiBase: () => ''}; - let $scope - let $controller // controller generator - let $httpBackend + let $scope; + let $controller; // controller generator + let $httpBackend; beforeEach(inject((_$controller_, _$rootScope_, _$compile_, _$httpBackend_, _ngToast_) => { - $scope = _$rootScope_.$new() - $controller = _$controller_ - $httpBackend = _$httpBackend_ - })) + $scope = _$rootScope_.$new(); + $controller = _$controller_; + $httpBackend = _$httpBackend_; + })); - const credentialResponse = { 'spark.testCredential': { username: 'user1', password: 'password1' }, } + const credentialResponse = {'spark.testCredential': {username: 'user1', password: 'password1'}}; const interpreterResponse = [ - { 'name': 'spark', 'group': 'spark', }, - { 'name': 'md', 'group': 'md', }, - ] // simplified + {'name': 'spark', 'group': 'spark'}, + {'name': 'md', 'group': 'md'}, + ]; // simplified function setupInitialization(credentialRes, interpreterRes) { // requests should follow the exact order $httpBackend .when('GET', '/interpreter/setting') - .respond(200, { body: interpreterRes, }) - $httpBackend.expectGET('/interpreter/setting') + .respond(200, {body: interpreterRes}); + $httpBackend.expectGET('/interpreter/setting'); $httpBackend .when('GET', '/credential') - .respond(200, { body: { userCredentials: credentialRes, } }) - $httpBackend.expectGET('/credential') + .respond(200, {body: {userCredentials: credentialRes}}); + $httpBackend.expectGET('/credential'); // should flush after calling this function } it('should get available interpreters and credentials initially', () => { - const ctrl = createController() - expect(ctrl).toBeDefined() + const ctrl = createController(); + expect(ctrl).toBeDefined(); - setupInitialization(credentialResponse, interpreterResponse) - $httpBackend.flush() + setupInitialization(credentialResponse, interpreterResponse); + $httpBackend.flush(); expect($scope.credentialInfo).toEqual( - [{ entity: 'spark.testCredential', username: 'user1', password: 'password1'}] - ) + [{entity: 'spark.testCredential', username: 'user1', password: 'password1'}] + ); expect($scope.availableInterpreters).toEqual( ['spark.spark', 'md.md'] - ) + ); - $httpBackend.verifyNoOutstandingExpectation() - $httpBackend.verifyNoOutstandingRequest() - }) + $httpBackend.verifyNoOutstandingExpectation(); + $httpBackend.verifyNoOutstandingRequest(); + }); it('should toggle using toggleAddNewCredentialInfo', () => { - createController() + createController(); - expect($scope.showAddNewCredentialInfo).toBe(false) - $scope.toggleAddNewCredentialInfo() - expect($scope.showAddNewCredentialInfo).toBe(true) - $scope.toggleAddNewCredentialInfo() - expect($scope.showAddNewCredentialInfo).toBe(false) - }) + expect($scope.showAddNewCredentialInfo).toBe(false); + $scope.toggleAddNewCredentialInfo(); + expect($scope.showAddNewCredentialInfo).toBe(true); + $scope.toggleAddNewCredentialInfo(); + expect($scope.showAddNewCredentialInfo).toBe(false); + }); it('should check empty credentials using isInvalidCredential', () => { - createController() + createController(); - $scope.entity = '' - $scope.username = '' - expect($scope.isValidCredential()).toBe(false) + $scope.entity = ''; + $scope.username = ''; + expect($scope.isValidCredential()).toBe(false); - $scope.entity = 'spark1' - $scope.username = '' - expect($scope.isValidCredential()).toBe(false) + $scope.entity = 'spark1'; + $scope.username = ''; + expect($scope.isValidCredential()).toBe(false); - $scope.entity = '' - $scope.username = 'user1' - expect($scope.isValidCredential()).toBe(false) + $scope.entity = ''; + $scope.username = 'user1'; + expect($scope.isValidCredential()).toBe(false); - $scope.entity = 'spark' - $scope.username = 'user1' - expect($scope.isValidCredential()).toBe(true) - }) + $scope.entity = 'spark'; + $scope.username = 'user1'; + expect($scope.isValidCredential()).toBe(true); + }); it('should be able to add credential via addNewCredentialInfo', () => { - const ctrl = createController() - expect(ctrl).toBeDefined() - setupInitialization(credentialResponse, interpreterResponse) + const ctrl = createController(); + expect(ctrl).toBeDefined(); + setupInitialization(credentialResponse, interpreterResponse); // when - const newCredential = { entity: 'spark.sql', username: 'user2', password: 'password2'} + const newCredential = {entity: 'spark.sql', username: 'user2', password: 'password2'}; $httpBackend .when('PUT', '/credential', newCredential) - .respond(200, { }) - $httpBackend.expectPUT('/credential', newCredential) + .respond(200, { }); + $httpBackend.expectPUT('/credential', newCredential); - $scope.entity = newCredential.entity - $scope.username = newCredential.username - $scope.password = newCredential.password - $scope.addNewCredentialInfo() + $scope.entity = newCredential.entity; + $scope.username = newCredential.username; + $scope.password = newCredential.password; + $scope.addNewCredentialInfo(); - $httpBackend.flush() + $httpBackend.flush(); - expect($scope.credentialInfo[1]).toEqual(newCredential) + expect($scope.credentialInfo[1]).toEqual(newCredential); - $httpBackend.verifyNoOutstandingExpectation() - $httpBackend.verifyNoOutstandingRequest() - }) + $httpBackend.verifyNoOutstandingExpectation(); + $httpBackend.verifyNoOutstandingRequest(); + }); function createController() { - return $controller('CredentialCtrl', { $scope: $scope, baseUrlSrv: baseUrlSrvMock, }) + return $controller('CredentialCtrl', {$scope: $scope, baseUrlSrv: baseUrlSrvMock}); } -}) +}); diff --git a/zeppelin-web/src/app/helium/helium-conf.js b/zeppelin-web/src/app/helium/helium-conf.js index 10ca18adf84..05a58cf4f67 100644 --- a/zeppelin-web/src/app/helium/helium-conf.js +++ b/zeppelin-web/src/app/helium/helium-conf.js @@ -16,84 +16,92 @@ export const HeliumConfFieldType = { NUMBER: 'number', JSON: 'json', STRING: 'string', -} +}; /** * @param persisted including `type`, `description`, `defaultValue` for each conf key * @param spec including `value` for each conf key */ -export function mergePersistedConfWithSpec (persisted, spec) { - const confs = [] +export function mergePersistedConfWithSpec(persisted, spec) { + const confs = []; for (let name in spec) { - const specField = spec[name] - const persistedValue = persisted[name] - - const value = (persistedValue) ? persistedValue : specField.defaultValue - const merged = { - name: name, - type: specField.type, - description: specField.description, - value: value, - defaultValue: specField.defaultValue, + if (spec.hasOwnProperty(name)) { + const specField = spec[name]; + const persistedValue = persisted[name]; + + const value = (persistedValue) ? persistedValue : specField.defaultValue; + const merged = { + name: name, + type: specField.type, + description: specField.description, + value: value, + defaultValue: specField.defaultValue, + }; + + confs.push(merged); } - - confs.push(merged) } - return confs + return confs; } -export function createAllPackageConfigs (defaultPackages, persistedConfs) { - let packageConfs = {} +export function createAllPackageConfigs(defaultPackages, persistedConfs) { + let packageConfs = {}; for (let name in defaultPackages) { - const pkgSearchResult = defaultPackages[name] - - const spec = pkgSearchResult.pkg.config - if (!spec) { continue } - - const artifact = pkgSearchResult.pkg.artifact - if (!artifact) { continue } - - let persistedConf = {} - if (persistedConfs[artifact]) { - persistedConf = persistedConfs[artifact] + if (defaultPackages.hasOwnProperty(name)) { + const pkgSearchResult = defaultPackages[name]; + + const spec = pkgSearchResult.pkg.config; + if (!spec) { + continue; + } + + const artifact = pkgSearchResult.pkg.artifact; + if (!artifact) { + continue; + } + + let persistedConf = {}; + if (persistedConfs[artifact]) { + persistedConf = persistedConfs[artifact]; + } + + const confs = mergePersistedConfWithSpec(persistedConf, spec); + packageConfs[name] = confs; } - - const confs = mergePersistedConfWithSpec(persistedConf, spec) - packageConfs[name] = confs } - return packageConfs + return packageConfs; } -export function parseConfigValue (type, stringified) { - let value = stringified +export function parseConfigValue(type, stringified) { + let value = stringified; try { if (HeliumConfFieldType.NUMBER === type) { - value = parseFloat(stringified) + value = parseFloat(stringified); } else if (HeliumConfFieldType.JSON === type) { - value = JSON.parse(stringified) + value = JSON.parse(stringified); } } catch (error) { // return just the stringified one - console.error(`Failed to parse conf type ${type}, value ${value}`) + console.error(`Failed to parse conf type ${type}, value ${value}`); } - return value + return value; } /** * persist key-value only * since other info (e.g type, desc) can be provided by default config */ -export function createPersistableConfig (currentConfs) { +export function createPersistableConfig(currentConfs) { const filtered = currentConfs.reduce((acc, c) => { - acc[c.name] = parseConfigValue(c.type, c.value) - return acc - }, {}) + acc[c.name] = parseConfigValue(c.type, c.value); + return acc; + }, {}); - return filtered + return filtered; } diff --git a/zeppelin-web/src/app/helium/helium-package.js b/zeppelin-web/src/app/helium/helium-package.js index 88d191a7a8e..2fe9bf58964 100644 --- a/zeppelin-web/src/app/helium/helium-package.js +++ b/zeppelin-web/src/app/helium/helium-package.js @@ -12,20 +12,22 @@ * limitations under the License. */ -export function createDefaultPackage (pkgSearchResult, sce) { +export function createDefaultPackage(pkgSearchResult, sce) { for (let pkgIdx in pkgSearchResult) { - const pkg = pkgSearchResult[pkgIdx] - pkg.pkg.icon = sce.trustAsHtml(pkg.pkg.icon) - if (pkg.enabled) { - pkgSearchResult.splice(pkgIdx, 1) - return pkg + if (pkgSearchResult.hasOwnProperty(pkgIdx)) { + const pkg = pkgSearchResult[pkgIdx]; + pkg.pkg.icon = sce.trustAsHtml(pkg.pkg.icon); + if (pkg.enabled) { + pkgSearchResult.splice(pkgIdx, 1); + return pkg; + } } } // show first available version if package is not enabled - const result = pkgSearchResult[0] - pkgSearchResult.splice(0, 1) - return result + const result = pkgSearchResult[0]; + pkgSearchResult.splice(0, 1); + return result; } /** @@ -35,13 +37,15 @@ export function createDefaultPackage (pkgSearchResult, sce) { * @param sce angular `$sce` object * @returns {Object} including {name, pkgInfo} */ -export function createDefaultPackages (pkgSearchResults, sce) { - const defaultPackages = {} +export function createDefaultPackages(pkgSearchResults, sce) { + const defaultPackages = {}; // show enabled version if any version of package is enabled for (let name in pkgSearchResults) { - const pkgSearchResult = pkgSearchResults[name] - defaultPackages[name] = createDefaultPackage(pkgSearchResult, sce) + if (pkgSearchResults.hasOwnProperty(name)) { + const pkgSearchResult = pkgSearchResults[name]; + defaultPackages[name] = createDefaultPackage(pkgSearchResult, sce); + } } - return defaultPackages + return defaultPackages; } diff --git a/zeppelin-web/src/app/helium/helium-type.js b/zeppelin-web/src/app/helium/helium-type.js index 27b34fa6960..0b37a418837 100644 --- a/zeppelin-web/src/app/helium/helium-type.js +++ b/zeppelin-web/src/app/helium/helium-type.js @@ -17,4 +17,4 @@ export const HeliumType = { SPELL: 'SPELL', INTERPRETER: 'INTERPRETER', APPLICATION: 'APPLICATION', -} +}; diff --git a/zeppelin-web/src/app/helium/helium.controller.js b/zeppelin-web/src/app/helium/helium.controller.js index a397aceea34..4728e0896ff 100644 --- a/zeppelin-web/src/app/helium/helium.controller.js +++ b/zeppelin-web/src/app/helium/helium.controller.js @@ -12,92 +12,94 @@ * limitations under the License. */ -import { HeliumType, } from './helium-type' +import {HeliumType} from './helium-type'; -export default function HeliumCtrl ($scope, $rootScope, $sce, +export default function HeliumCtrl($scope, $rootScope, $sce, baseUrlSrv, ngToast, heliumService) { - 'ngInject' - - $scope.pkgSearchResults = {} - $scope.defaultPackages = {} - $scope.showVersions = {} - $scope.bundleOrder = [] - $scope.bundleOrderChanged = false - $scope.vizTypePkg = {} - $scope.spellTypePkg = {} - $scope.intpTypePkg = {} - $scope.appTypePkg = {} - $scope.numberOfEachPackageByType = {} - $scope.allPackageTypes = [HeliumType][0] - $scope.pkgListByType = 'VISUALIZATION' - $scope.defaultPackageConfigs = {} // { pkgName, [{name, type, desc, value, defaultValue}] } - $scope.intpDefaultIcon = $sce.trustAsHtml('') - - function init () { + 'ngInject'; + + $scope.pkgSearchResults = {}; + $scope.defaultPackages = {}; + $scope.showVersions = {}; + $scope.bundleOrder = []; + $scope.bundleOrderChanged = false; + $scope.vizTypePkg = {}; + $scope.spellTypePkg = {}; + $scope.intpTypePkg = {}; + $scope.appTypePkg = {}; + $scope.numberOfEachPackageByType = {}; + $scope.allPackageTypes = [HeliumType][0]; + $scope.pkgListByType = 'VISUALIZATION'; + $scope.defaultPackageConfigs = {}; // { pkgName, [{name, type, desc, value, defaultValue}] } + $scope.intpDefaultIcon = $sce.trustAsHtml(''); + + function init() { // get all package info and set config heliumService.getAllPackageInfoAndDefaultPackages() - .then(({ pkgSearchResults, defaultPackages }) => { + .then(({pkgSearchResults, defaultPackages}) => { // pagination - $scope.itemsPerPage = 10 - $scope.currentPage = 1 - $scope.maxSize = 5 + $scope.itemsPerPage = 10; + $scope.currentPage = 1; + $scope.maxSize = 5; - $scope.pkgSearchResults = pkgSearchResults - $scope.defaultPackages = defaultPackages - classifyPkgType($scope.defaultPackages) + $scope.pkgSearchResults = pkgSearchResults; + $scope.defaultPackages = defaultPackages; + classifyPkgType($scope.defaultPackages); - return heliumService.getAllPackageConfigs() + return heliumService.getAllPackageConfigs(); }) - .then(defaultPackageConfigs => { - $scope.defaultPackageConfigs = defaultPackageConfigs - return heliumService.getVisualizationPackageOrder() - }) - .then(visPackageOrder => { - setVisPackageOrder(visPackageOrder) + .then((defaultPackageConfigs) => { + $scope.defaultPackageConfigs = defaultPackageConfigs; + return heliumService.getVisualizationPackageOrder(); }) + .then((visPackageOrder) => { + setVisPackageOrder(visPackageOrder); + }); } const setVisPackageOrder = function(visPackageOrder) { - $scope.bundleOrder = visPackageOrder - $scope.bundleOrderChanged = false - } + $scope.bundleOrder = visPackageOrder; + $scope.bundleOrderChanged = false; + }; - let orderPackageByPubDate = function (a, b) { + let orderPackageByPubDate = function(a, b) { if (!a.pkg.published) { // Because local registry pkgs don't have 'published' field, put current time instead to show them first - a.pkg.published = new Date().getTime() + a.pkg.published = new Date().getTime(); } - return new Date(a.pkg.published).getTime() - new Date(b.pkg.published).getTime() - } + return new Date(a.pkg.published).getTime() - new Date(b.pkg.published).getTime(); + }; - const classifyPkgType = function (packageInfo) { - let allTypesOfPkg = {} - let vizTypePkg = [] - let spellTypePkg = [] - let intpTypePkg = [] - let appTypePkg = [] + const classifyPkgType = function(packageInfo) { + let allTypesOfPkg = {}; + let vizTypePkg = []; + let spellTypePkg = []; + let intpTypePkg = []; + let appTypePkg = []; - let packageInfoArr = Object.keys(packageInfo).map(key => packageInfo[key]) - packageInfoArr = packageInfoArr.sort(orderPackageByPubDate).reverse() + let packageInfoArr = Object.keys(packageInfo).map((key) => packageInfo[key]); + packageInfoArr = packageInfoArr.sort(orderPackageByPubDate).reverse(); for (let name in packageInfoArr) { - let pkgs = packageInfoArr[name] - let pkgType = pkgs.pkg.type - - switch (pkgType) { - case HeliumType.VISUALIZATION: - vizTypePkg.push(pkgs) - break - case HeliumType.SPELL: - spellTypePkg.push(pkgs) - break - case HeliumType.INTERPRETER: - intpTypePkg.push(pkgs) - break - case HeliumType.APPLICATION: - appTypePkg.push(pkgs) - break + if (packageInfoArr.hasOwnProperty(name)) { + let pkgs = packageInfoArr[name]; + let pkgType = pkgs.pkg.type; + + switch (pkgType) { + case HeliumType.VISUALIZATION: + vizTypePkg.push(pkgs); + break; + case HeliumType.SPELL: + spellTypePkg.push(pkgs); + break; + case HeliumType.INTERPRETER: + intpTypePkg.push(pkgs); + break; + case HeliumType.APPLICATION: + appTypePkg.push(pkgs); + break; + } } } @@ -105,95 +107,99 @@ export default function HeliumCtrl ($scope, $rootScope, $sce, vizTypePkg, spellTypePkg, intpTypePkg, - appTypePkg - ] + appTypePkg, + ]; for (let idx in _.keys(HeliumType)) { - allTypesOfPkg[_.keys(HeliumType)[idx]] = pkgsArr[idx] + if (_.keys(HeliumType).hasOwnProperty(idx)) { + allTypesOfPkg[_.keys(HeliumType)[idx]] = pkgsArr[idx]; + } } - $scope.allTypesOfPkg = allTypesOfPkg - } + $scope.allTypesOfPkg = allTypesOfPkg; + }; $scope.bundleOrderListeners = { - accept: function (sourceItemHandleScope, destSortableScope) { return true }, - itemMoved: function (event) {}, - orderChanged: function (event) { - $scope.bundleOrderChanged = true - } - } - - $scope.saveBundleOrder = function () { + accept: function(sourceItemHandleScope, destSortableScope) { + return true; + }, + itemMoved: function(event) {}, + orderChanged: function(event) { + $scope.bundleOrderChanged = true; + }, + }; + + $scope.saveBundleOrder = function() { const confirm = BootstrapDialog.confirm({ closable: false, closeByBackdrop: false, closeByKeyboard: false, title: '', message: 'Save changes?', - callback: function (result) { + callback: function(result) { if (result) { - confirm.$modalFooter.find('button').addClass('disabled') + confirm.$modalFooter.find('button').addClass('disabled'); confirm.$modalFooter.find('button:contains("OK")') - .html(' Enabling') + .html(' Enabling'); heliumService.setVisualizationPackageOrder($scope.bundleOrder) - .success(function (data, status) { - setVisPackageOrder($scope.bundleOrder) - confirm.close() + .success(function(data, status) { + setVisPackageOrder($scope.bundleOrder); + confirm.close(); }) - .error(function (data, status) { - confirm.close() - console.log('Failed to save order') + .error(function(data, status) { + confirm.close(); + console.log('Failed to save order'); BootstrapDialog.show({ title: 'Error on saving order ', - message: data.message - }) - }) - return false + message: data.message, + }); + }); + return false; } - } - }) - } + }, + }); + }; - let getLicense = function (name, artifact) { - let filteredPkgSearchResults = _.filter($scope.defaultPackages[name], function (p) { - return p.artifact === artifact - }) + let getLicense = function(name, artifact) { + let filteredPkgSearchResults = _.filter($scope.defaultPackages[name], function(p) { + return p.artifact === artifact; + }); - let license + let license; if (filteredPkgSearchResults.length === 0) { - filteredPkgSearchResults = _.filter($scope.pkgSearchResults[name], function (p) { - return p.pkg.artifact === artifact - }) + filteredPkgSearchResults = _.filter($scope.pkgSearchResults[name], function(p) { + return p.pkg.artifact === artifact; + }); if (filteredPkgSearchResults.length > 0) { - license = filteredPkgSearchResults[0].pkg.license + license = filteredPkgSearchResults[0].pkg.license; } } else { - license = filteredPkgSearchResults[0].license + license = filteredPkgSearchResults[0].license; } if (!license) { - license = 'Unknown' + license = 'Unknown'; } - return license - } + return license; + }; - const getHeliumTypeText = function (type) { + const getHeliumTypeText = function(type) { if (type === HeliumType.VISUALIZATION) { - return `${type}` // eslint-disable-line max-len + return `${type}`; // eslint-disable-line max-len } else if (type === HeliumType.SPELL) { - return `${type}` // eslint-disable-line max-len + return `${type}`; // eslint-disable-line max-len } else { - return type + return type; } - } + }; - $scope.enable = function (name, artifact, type, groupId, description) { - let license = getLicense(name, artifact) - let mavenArtifactInfoToHTML = groupId + ':' + artifact.split('@')[0] + ':' + artifact.split('@')[1] - let zeppelinVersion = $rootScope.zeppelinVersion - let url = 'https://zeppelin.apache.org/docs/' + zeppelinVersion + '/manual/interpreterinstallation.html' + $scope.enable = function(name, artifact, type, groupId, description) { + let license = getLicense(name, artifact); + let mavenArtifactInfoToHTML = groupId + ':' + artifact.split('@')[0] + ':' + artifact.split('@')[1]; + let zeppelinVersion = $rootScope.zeppelinVersion; + let url = 'https://zeppelin.apache.org/docs/' + zeppelinVersion + '/manual/interpreterinstallation.html'; - let confirm = '' + let confirm = ''; if (type === HeliumType.INTERPRETER) { confirm = BootstrapDialog.show({ title: '', @@ -206,8 +212,8 @@ export default function HeliumCtrl ($scope, $rootScope, $sce, mavenArtifactInfoToHTML + ' ' + '

After restart Zeppelin, create interpreter setting and bind it with your note. ' + 'For more detailed information, see Interpreter Installation.

' - }) + url + '>Interpreter Installation.

', + }); } else { confirm = BootstrapDialog.confirm({ closable: false, @@ -226,138 +232,138 @@ export default function HeliumCtrl ($scope, $rootScope, $sce, '
' + '
License
' + `
${license}
`, - callback: function (result) { + callback: function(result) { if (result) { - confirm.$modalFooter.find('button').addClass('disabled') + confirm.$modalFooter.find('button').addClass('disabled'); confirm.$modalFooter.find('button:contains("OK")') - .html(' Enabling') - heliumService.enable(name, artifact, type).success(function (data, status) { - init() - confirm.close() - }).error(function (data, status) { - confirm.close() - console.log('Failed to enable package %o %o. %o', name, artifact, data) + .html(' Enabling'); + heliumService.enable(name, artifact, type).success(function(data, status) { + init(); + confirm.close(); + }).error(function(data, status) { + confirm.close(); + console.log('Failed to enable package %o %o. %o', name, artifact, data); BootstrapDialog.show({ title: 'Error on enabling ' + name, - message: data.message - }) - }) - return false + message: data.message, + }); + }); + return false; } - } - }) + }, + }); } - } + }; - $scope.disable = function (name, artifact) { + $scope.disable = function(name, artifact) { const confirm = BootstrapDialog.confirm({ closable: false, closeByBackdrop: false, closeByKeyboard: false, title: '
Do you want to disable Helium Package?
', message: artifact, - callback: function (result) { + callback: function(result) { if (result) { - confirm.$modalFooter.find('button').addClass('disabled') + confirm.$modalFooter.find('button').addClass('disabled'); confirm.$modalFooter.find('button:contains("OK")') - .html(' Disabling') + .html(' Disabling'); heliumService.disable(name) - .success(function (data, status) { - init() - confirm.close() + .success(function(data, status) { + init(); + confirm.close(); }) - .error(function (data, status) { - confirm.close() - console.log('Failed to disable package %o. %o', name, data) + .error(function(data, status) { + confirm.close(); + console.log('Failed to disable package %o. %o', name, data); BootstrapDialog.show({ title: 'Error on disabling ' + name, - message: data.message - }) - }) - return false + message: data.message, + }); + }); + return false; } - } - }) - } + }, + }); + }; - $scope.toggleVersions = function (pkgName) { + $scope.toggleVersions = function(pkgName) { if ($scope.showVersions[pkgName]) { - $scope.showVersions[pkgName] = false + $scope.showVersions[pkgName] = false; } else { - $scope.showVersions[pkgName] = true + $scope.showVersions[pkgName] = true; } - } + }; - $scope.isLocalPackage = function (pkgSearchResult) { - const pkg = pkgSearchResult.pkg - return pkg.artifact && !pkg.artifact.includes('@') - } + $scope.isLocalPackage = function(pkgSearchResult) { + const pkg = pkgSearchResult.pkg; + return pkg.artifact && !pkg.artifact.includes('@'); + }; - $scope.hasNpmLink = function (pkgSearchResult) { - const pkg = pkgSearchResult.pkg + $scope.hasNpmLink = function(pkgSearchResult) { + const pkg = pkgSearchResult.pkg; return (pkg.type === HeliumType.SPELL || pkg.type === HeliumType.VISUALIZATION) && - !$scope.isLocalPackage(pkgSearchResult) - } + !$scope.isLocalPackage(pkgSearchResult); + }; - $scope.hasMavenLink = function (pkgSearchResult) { - const pkg = pkgSearchResult.pkg + $scope.hasMavenLink = function(pkgSearchResult) { + const pkg = pkgSearchResult.pkg; return (pkg.type === HeliumType.APPLICATION || pkg.type === HeliumType.INTERPRETER) && - !$scope.isLocalPackage(pkgSearchResult) - } - - $scope.getPackageSize = function (pkgSearchResult, targetPkgType) { - let result = [] - _.map(pkgSearchResult, function (pkg) { - result.push(_.find(pkg, {type: targetPkgType})) - }) - return _.compact(result).length - } - - $scope.configExists = function (pkgSearchResult) { + !$scope.isLocalPackage(pkgSearchResult); + }; + + $scope.getPackageSize = function(pkgSearchResult, targetPkgType) { + let result = []; + _.map(pkgSearchResult, function(pkg) { + result.push(_.find(pkg, {type: targetPkgType})); + }); + return _.compact(result).length; + }; + + $scope.configExists = function(pkgSearchResult) { // helium package config is persisted per version - return pkgSearchResult.pkg.config && pkgSearchResult.pkg.artifact - } + return pkgSearchResult.pkg.config && pkgSearchResult.pkg.artifact; + }; - $scope.configOpened = function (pkgSearchResult) { - return pkgSearchResult.configOpened && !pkgSearchResult.configFetching - } + $scope.configOpened = function(pkgSearchResult) { + return pkgSearchResult.configOpened && !pkgSearchResult.configFetching; + }; - $scope.getConfigButtonClass = function (pkgSearchResult) { + $scope.getConfigButtonClass = function(pkgSearchResult) { return (pkgSearchResult.configOpened && pkgSearchResult.configFetching) - ? 'disabled' : '' - } + ? 'disabled' : ''; + }; - $scope.toggleConfigButton = function (pkgSearchResult) { + $scope.toggleConfigButton = function(pkgSearchResult) { if (pkgSearchResult.configOpened) { - pkgSearchResult.configOpened = false - return + pkgSearchResult.configOpened = false; + return; } - const pkg = pkgSearchResult.pkg - const pkgName = pkg.name - pkgSearchResult.configFetching = true - pkgSearchResult.configOpened = true + const pkg = pkgSearchResult.pkg; + const pkgName = pkg.name; + pkgSearchResult.configFetching = true; + pkgSearchResult.configOpened = true; heliumService.getSinglePackageConfigs(pkg) - .then(confs => { - $scope.defaultPackageConfigs[pkgName] = confs - pkgSearchResult.configFetching = false - }) - } + .then((confs) => { + $scope.defaultPackageConfigs[pkgName] = confs; + pkgSearchResult.configFetching = false; + }); + }; - $scope.saveConfig = function (pkgSearchResult) { - const pkgName = pkgSearchResult.pkg.name - const currentConf = $scope.defaultPackageConfigs[pkgName] + $scope.saveConfig = function(pkgSearchResult) { + const pkgName = pkgSearchResult.pkg.name; + const currentConf = $scope.defaultPackageConfigs[pkgName]; heliumService.saveConfig(pkgSearchResult.pkg, currentConf, () => { // close after config is saved - pkgSearchResult.configOpened = false - }) - } + pkgSearchResult.configOpened = false; + }); + }; - $scope.getDescriptionText = function (pkgSearchResult) { - return $sce.trustAsHtml(pkgSearchResult.pkg.description) - } + $scope.getDescriptionText = function(pkgSearchResult) { + return $sce.trustAsHtml(pkgSearchResult.pkg.description); + }; - init() + init(); } diff --git a/zeppelin-web/src/app/helium/helium.service.js b/zeppelin-web/src/app/helium/helium.service.js index d2054b320f9..7501fae827f 100644 --- a/zeppelin-web/src/app/helium/helium.service.js +++ b/zeppelin-web/src/app/helium/helium.service.js @@ -12,290 +12,294 @@ * limitations under the License. */ -import { HeliumType, } from './helium-type' +import {HeliumType} from './helium-type'; import { createAllPackageConfigs, createPersistableConfig, mergePersistedConfWithSpec, -} from './helium-conf' +} from './helium-conf'; import { createDefaultPackages, -} from './helium-package' +} from './helium-package'; -angular.module('zeppelinWebApp').service('heliumService', HeliumService) +angular.module('zeppelinWebApp').service('heliumService', HeliumService); export default function HeliumService($http, $sce, baseUrlSrv) { - 'ngInject' + 'ngInject'; - let visualizationBundles = [] - let visualizationPackageOrder = [] + let visualizationBundles = []; + let visualizationPackageOrder = []; // name `heliumBundles` should be same as `HeliumBundleFactory.HELIUM_BUNDLES_VAR` - let heliumBundles = [] + let heliumBundles = []; // map for `{ magic: interpreter }` - let spellPerMagic = {} + let spellPerMagic = {}; // map for `{ magic: package-name }` - let pkgNamePerMagic = {} + let pkgNamePerMagic = {}; /** * @param magic {string} e.g `%flowchart` * @returns {SpellBase} undefined if magic is not registered */ - this.getSpellByMagic = function (magic) { - return spellPerMagic[magic] - } + this.getSpellByMagic = function(magic) { + return spellPerMagic[magic]; + }; - this.executeSpell = function (magic, textWithoutMagic) { + this.executeSpell = function(magic, textWithoutMagic) { const promisedConf = this.getSinglePackageConfigUsingMagic(magic) - .then(confs => createPersistableConfig(confs)) + .then((confs) => createPersistableConfig(confs)); - return promisedConf.then(conf => { - const spell = this.getSpellByMagic(magic) - const spellResult = spell.interpret(textWithoutMagic, conf) + return promisedConf.then((conf) => { + const spell = this.getSpellByMagic(magic); + const spellResult = spell.interpret(textWithoutMagic, conf); const parsed = spellResult.getAllParsedDataWithTypes( - spellPerMagic, magic, textWithoutMagic) + spellPerMagic, magic, textWithoutMagic); - return parsed - }) - } + return parsed; + }); + }; - this.executeSpellAsDisplaySystem = function (magic, textWithoutMagic) { + this.executeSpellAsDisplaySystem = function(magic, textWithoutMagic) { const promisedConf = this.getSinglePackageConfigUsingMagic(magic) - .then(confs => createPersistableConfig(confs)) + .then((confs) => createPersistableConfig(confs)); - return promisedConf.then(conf => { - const spell = this.getSpellByMagic(magic) - const spellResult = spell.interpret(textWithoutMagic.trim(), conf) - const parsed = spellResult.getAllParsedDataWithTypes(spellPerMagic) + return promisedConf.then((conf) => { + const spell = this.getSpellByMagic(magic); + const spellResult = spell.interpret(textWithoutMagic.trim(), conf); + const parsed = spellResult.getAllParsedDataWithTypes(spellPerMagic); - return parsed - }) - } + return parsed; + }); + }; - this.getVisualizationCachedPackages = function () { - return visualizationBundles - } + this.getVisualizationCachedPackages = function() { + return visualizationBundles; + }; - this.getVisualizationCachedPackageOrder = function () { - return visualizationPackageOrder - } + this.getVisualizationCachedPackageOrder = function() { + return visualizationPackageOrder; + }; /** * @returns {Promise} which returns bundleOrder and cache it in `visualizationPackageOrder` */ - this.getVisualizationPackageOrder = function () { + this.getVisualizationPackageOrder = function() { return $http.get(baseUrlSrv.getRestApiBase() + '/helium/order/visualization') - .then(function (response, status) { - const order = response.data.body - visualizationPackageOrder = order - return order - }) - .catch(function (error) { - console.error('Can not get bundle order', error) + .then(function(response, status) { + const order = response.data.body; + visualizationPackageOrder = order; + return order; }) - } + .catch(function(error) { + console.error('Can not get bundle order', error); + }); + }; - this.setVisualizationPackageOrder = function (list) { - return $http.post(baseUrlSrv.getRestApiBase() + '/helium/order/visualization', list) - } + this.setVisualizationPackageOrder = function(list) { + return $http.post(baseUrlSrv.getRestApiBase() + '/helium/order/visualization', list); + }; - this.enable = function (name, artifact) { - return $http.post(baseUrlSrv.getRestApiBase() + '/helium/enable/' + name, artifact) - } + this.enable = function(name, artifact) { + return $http.post(baseUrlSrv.getRestApiBase() + '/helium/enable/' + name, artifact); + }; - this.disable = function (name) { - return $http.post(baseUrlSrv.getRestApiBase() + '/helium/disable/' + name) - } + this.disable = function(name) { + return $http.post(baseUrlSrv.getRestApiBase() + '/helium/disable/' + name); + }; - this.saveConfig = function (pkg, defaultPackageConfig, closeConfigPanelCallback) { + this.saveConfig = function(pkg, defaultPackageConfig, closeConfigPanelCallback) { // in case of local package, it will include `/` - const pkgArtifact = encodeURIComponent(pkg.artifact) - const pkgName = pkg.name - const filtered = createPersistableConfig(defaultPackageConfig) + const pkgArtifact = encodeURIComponent(pkg.artifact); + const pkgName = pkg.name; + const filtered = createPersistableConfig(defaultPackageConfig); if (!pkgName || !pkgArtifact || !filtered) { console.error( - `Can't save config for helium package '${pkgArtifact}'`, filtered) - return + `Can't save config for helium package '${pkgArtifact}'`, filtered); + return; } - const url = `${baseUrlSrv.getRestApiBase()}/helium/config/${pkgName}/${pkgArtifact}` + const url = `${baseUrlSrv.getRestApiBase()}/helium/config/${pkgName}/${pkgArtifact}`; return $http.post(url, filtered) .then(() => { - if (closeConfigPanelCallback) { closeConfigPanelCallback() } + if (closeConfigPanelCallback) { + closeConfigPanelCallback(); + } }).catch((error) => { - console.error(`Failed to save config for ${pkgArtifact}`, error) - }) - } + console.error(`Failed to save config for ${pkgArtifact}`, error); + }); + }; /** * @returns {Promise} which including {name, Array} */ - this.getAllPackageInfo = function () { + this.getAllPackageInfo = function() { return $http.get(`${baseUrlSrv.getRestApiBase()}/helium/package`) - .then(function (response, status) { - return response.data.body - }) - .catch(function (error) { - console.error('Failed to get all package infos', error) + .then(function(response, status) { + return response.data.body; }) - } + .catch(function(error) { + console.error('Failed to get all package infos', error); + }); + }; - this.getAllEnabledPackages = function () { + this.getAllEnabledPackages = function() { return $http.get(`${baseUrlSrv.getRestApiBase()}/helium/enabledPackage`) - .then(function (response, status) { - return response.data.body + .then(function(response, status) { + return response.data.body; }) - .catch(function (error) { - console.error('Failed to get all enabled package infos', error) - }) - } + .catch(function(error) { + console.error('Failed to get all enabled package infos', error); + }); + }; - this.getSingleBundle = function (pkgName) { - let url = `${baseUrlSrv.getRestApiBase()}/helium/bundle/load/${pkgName}` + this.getSingleBundle = function(pkgName) { + let url = `${baseUrlSrv.getRestApiBase()}/helium/bundle/load/${pkgName}`; if (process.env.HELIUM_BUNDLE_DEV) { - url = url + '?refresh=true' + url = url + '?refresh=true'; } return $http.get(url) - .then(function (response, status) { - const bundle = response.data + .then(function(response, status) { + const bundle = response.data; if (bundle.substring(0, 'ERROR:'.length) === 'ERROR:') { - console.error(`Failed to get bundle: ${pkgName}`, bundle) - return '' // empty bundle will be filtered later + console.error(`Failed to get bundle: ${pkgName}`, bundle); + return ''; // empty bundle will be filtered later } - return bundle - }) - .catch(function (error) { - console.error(`Failed to get single bundle: ${pkgName}`, error) + return bundle; }) - } + .catch(function(error) { + console.error(`Failed to get single bundle: ${pkgName}`, error); + }); + }; - this.getDefaultPackages = function () { + this.getDefaultPackages = function() { return this.getAllPackageInfo() - .then(pkgSearchResults => { - return createDefaultPackages(pkgSearchResults, $sce) - }) - } + .then((pkgSearchResults) => { + return createDefaultPackages(pkgSearchResults, $sce); + }); + }; - this.getAllPackageInfoAndDefaultPackages = function () { + this.getAllPackageInfoAndDefaultPackages = function() { return this.getAllPackageInfo() - .then(pkgSearchResults => { + .then((pkgSearchResults) => { return { pkgSearchResults: pkgSearchResults, defaultPackages: createDefaultPackages(pkgSearchResults, $sce), - } - }) - } + }; + }); + }; /** * get all package configs. * @return { Promise<{name, Array}> } */ - this.getAllPackageConfigs = function () { - const promisedDefaultPackages = this.getDefaultPackages() + this.getAllPackageConfigs = function() { + const promisedDefaultPackages = this.getDefaultPackages(); const promisedPersistedConfs = $http.get(`${baseUrlSrv.getRestApiBase()}/helium/config`) - .then(function (response, status) { - return response.data.body - }) + .then(function(response, status) { + return response.data.body; + }); return Promise.all([promisedDefaultPackages, promisedPersistedConfs]) - .then(values => { - const defaultPackages = values[0] - const persistedConfs = values[1] + .then((values) => { + const defaultPackages = values[0]; + const persistedConfs = values[1]; - return createAllPackageConfigs(defaultPackages, persistedConfs) - }) - .catch(function (error) { - console.error('Failed to get all package configs', error) + return createAllPackageConfigs(defaultPackages, persistedConfs); }) - } + .catch(function(error) { + console.error('Failed to get all package configs', error); + }); + }; /** * get the package config which is persisted in server. * @return { Promise> } */ - this.getSinglePackageConfigs = function (pkg) { - const pkgName = pkg.name + this.getSinglePackageConfigs = function(pkg) { + const pkgName = pkg.name; // in case of local package, it will include `/` - const pkgArtifact = encodeURIComponent(pkg.artifact) + const pkgArtifact = encodeURIComponent(pkg.artifact); if (!pkgName || !pkgArtifact) { - console.error('Failed to fetch config for\n', pkg) - return Promise.resolve([]) + console.error('Failed to fetch config for\n', pkg); + return Promise.resolve([]); } - const confUrl = `${baseUrlSrv.getRestApiBase()}/helium/config/${pkgName}/${pkgArtifact}` + const confUrl = `${baseUrlSrv.getRestApiBase()}/helium/config/${pkgName}/${pkgArtifact}`; const promisedConf = $http.get(confUrl) - .then(function (response, status) { - return response.data.body - }) + .then(function(response, status) { + return response.data.body; + }); return promisedConf.then(({confSpec, confPersisted}) => { - const merged = mergePersistedConfWithSpec(confPersisted, confSpec) - return merged - }) - } + const merged = mergePersistedConfWithSpec(confPersisted, confSpec); + return merged; + }); + }; - this.getSinglePackageConfigUsingMagic = function (magic) { - const pkgName = pkgNamePerMagic[magic] + this.getSinglePackageConfigUsingMagic = function(magic) { + const pkgName = pkgNamePerMagic[magic]; - const confUrl = `${baseUrlSrv.getRestApiBase()}/helium/spell/config/${pkgName}` + const confUrl = `${baseUrlSrv.getRestApiBase()}/helium/spell/config/${pkgName}`; const promisedConf = $http.get(confUrl) - .then(function (response, status) { - return response.data.body - }) + .then(function(response, status) { + return response.data.body; + }); return promisedConf.then(({confSpec, confPersisted}) => { - const merged = mergePersistedConfWithSpec(confPersisted, confSpec) - return merged - }) - } + const merged = mergePersistedConfWithSpec(confPersisted, confSpec); + return merged; + }); + }; const p = this.getAllEnabledPackages() - .then(enabledPackageSearchResults => { - const promises = enabledPackageSearchResults.map(packageSearchResult => { - const pkgName = packageSearchResult.pkg.name - return this.getSingleBundle(pkgName) - }) + .then((enabledPackageSearchResults) => { + const promises = enabledPackageSearchResults.map((packageSearchResult) => { + const pkgName = packageSearchResult.pkg.name; + return this.getSingleBundle(pkgName); + }); - return Promise.all(promises) + return Promise.all(promises); }) - .then(bundles => { + .then((bundles) => { return bundles.reduce((acc, b) => { // filter out empty bundle - if (b === '') { return acc } - acc.push(b) - return acc - }, []) - }) + if (b === '') { + return acc; + } + acc.push(b); + return acc; + }, []); + }); // load should be promise - this.load = p.then(availableBundles => { + this.load = p.then((availableBundles) => { // evaluate bundles - availableBundles.map(b => { + availableBundles.map((b) => { // eslint-disable-next-line no-eval - eval(b) - }) + eval(b); + }); // extract bundles by type - heliumBundles.map(b => { + heliumBundles.map((b) => { if (b.type === HeliumType.SPELL) { - const spell = new b.class() // eslint-disable-line new-cap - const pkgName = b.id - spellPerMagic[spell.getMagic()] = spell - pkgNamePerMagic[spell.getMagic()] = pkgName + const spell = new b.class(); // eslint-disable-line new-cap + const pkgName = b.id; + spellPerMagic[spell.getMagic()] = spell; + pkgNamePerMagic[spell.getMagic()] = pkgName; } else if (b.type === HeliumType.VISUALIZATION) { - visualizationBundles.push(b) + visualizationBundles.push(b); } - }) - }) + }); + }); this.init = function() { - this.getVisualizationPackageOrder() - } + this.getVisualizationPackageOrder(); + }; // init - this.init() + this.init(); } diff --git a/zeppelin-web/src/app/helium/index.js b/zeppelin-web/src/app/helium/index.js index 2b27d6036ce..754c9499f4f 100644 --- a/zeppelin-web/src/app/helium/index.js +++ b/zeppelin-web/src/app/helium/index.js @@ -12,7 +12,7 @@ * limitations under the License. */ -import HeliumController from './helium.controller' +import HeliumController from './helium.controller'; angular.module('zeppelinWebApp') - .controller('HeliumCtrl', HeliumController) + .controller('HeliumCtrl', HeliumController); diff --git a/zeppelin-web/src/app/home/home.controller.js b/zeppelin-web/src/app/home/home.controller.js index d2823dd6f6f..7ae5e44d70c 100644 --- a/zeppelin-web/src/app/home/home.controller.js +++ b/zeppelin-web/src/app/home/home.controller.js @@ -12,145 +12,145 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').controller('HomeCtrl', HomeCtrl) +angular.module('zeppelinWebApp').controller('HomeCtrl', HomeCtrl); -function HomeCtrl ($scope, noteListFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv, +function HomeCtrl($scope, noteListFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv, ngToast, noteActionService, TRASH_FOLDER_ID) { - 'ngInject' - - ngToast.dismiss() - let vm = this - vm.notes = noteListFactory - vm.websocketMsgSrv = websocketMsgSrv - vm.arrayOrderingSrv = arrayOrderingSrv - vm.noteActionService = noteActionService - vm.numberOfNotesDisplayed = window.innerHeight / 20 - - vm.notebookHome = false - vm.noteCustomHome = true + 'ngInject'; + + ngToast.dismiss(); + let vm = this; + vm.notes = noteListFactory; + vm.websocketMsgSrv = websocketMsgSrv; + vm.arrayOrderingSrv = arrayOrderingSrv; + vm.noteActionService = noteActionService; + vm.numberOfNotesDisplayed = window.innerHeight / 20; + + vm.notebookHome = false; + vm.noteCustomHome = true; if ($rootScope.ticket !== undefined) { - vm.staticHome = false + vm.staticHome = false; } else { - vm.staticHome = true + vm.staticHome = true; } - $scope.isReloading = false - $scope.TRASH_FOLDER_ID = TRASH_FOLDER_ID - $scope.query = {q: ''} + $scope.isReloading = false; + $scope.TRASH_FOLDER_ID = TRASH_FOLDER_ID; + $scope.query = {q: ''}; - $scope.initHome = function () { - websocketMsgSrv.getHomeNote() - vm.noteCustomHome = false - } + $scope.initHome = function() { + websocketMsgSrv.getHomeNote(); + vm.noteCustomHome = false; + }; - $scope.reloadNoteList = function () { - websocketMsgSrv.reloadAllNotesFromRepo() - $scope.isReloadingNotes = true - } + $scope.reloadNoteList = function() { + websocketMsgSrv.reloadAllNotesFromRepo(); + $scope.isReloadingNotes = true; + }; - $scope.toggleFolderNode = function (node) { - node.hidden = !node.hidden - } + $scope.toggleFolderNode = function(node) { + node.hidden = !node.hidden; + }; - angular.element('#loginModal').on('hidden.bs.modal', function (e) { - $rootScope.$broadcast('initLoginValues') - }) + angular.element('#loginModal').on('hidden.bs.modal', function(e) { + $rootScope.$broadcast('initLoginValues'); + }); /* ** $scope.$on functions below */ - $scope.$on('setNoteMenu', function (event, notes) { - $scope.isReloadingNotes = false - }) + $scope.$on('setNoteMenu', function(event, notes) { + $scope.isReloadingNotes = false; + }); - $scope.$on('setNoteContent', function (event, note) { + $scope.$on('setNoteContent', function(event, note) { if (vm.noteCustomHome) { - return + return; } if (note) { - vm.note = note + vm.note = note; // initialize look And Feel - $rootScope.$broadcast('setLookAndFeel', 'home') + $rootScope.$broadcast('setLookAndFeel', 'home'); // make it read only - vm.viewOnly = true + vm.viewOnly = true; - vm.notebookHome = true - vm.staticHome = false + vm.notebookHome = true; + vm.staticHome = false; } else { - vm.staticHome = true - vm.notebookHome = false + vm.staticHome = true; + vm.notebookHome = false; } - }) + }); - $scope.loadMoreNotes = function () { - vm.numberOfNotesDisplayed += 10 - } + $scope.loadMoreNotes = function() { + vm.numberOfNotesDisplayed += 10; + }; - $scope.renameNote = function (nodeId, nodePath) { - vm.noteActionService.renameNote(nodeId, nodePath) - } + $scope.renameNote = function(nodeId, nodePath) { + vm.noteActionService.renameNote(nodeId, nodePath); + }; - $scope.moveNoteToTrash = function (noteId) { - vm.noteActionService.moveNoteToTrash(noteId, false) - } + $scope.moveNoteToTrash = function(noteId) { + vm.noteActionService.moveNoteToTrash(noteId, false); + }; - $scope.moveFolderToTrash = function (folderId) { - vm.noteActionService.moveFolderToTrash(folderId) - } + $scope.moveFolderToTrash = function(folderId) { + vm.noteActionService.moveFolderToTrash(folderId); + }; - $scope.restoreNote = function (noteId) { - websocketMsgSrv.restoreNote(noteId) - } + $scope.restoreNote = function(noteId) { + websocketMsgSrv.restoreNote(noteId); + }; - $scope.restoreFolder = function (folderId) { - websocketMsgSrv.restoreFolder(folderId) - } + $scope.restoreFolder = function(folderId) { + websocketMsgSrv.restoreFolder(folderId); + }; - $scope.restoreAll = function () { - vm.noteActionService.restoreAll() - } + $scope.restoreAll = function() { + vm.noteActionService.restoreAll(); + }; - $scope.renameFolder = function (node) { - vm.noteActionService.renameFolder(node.id) - } + $scope.renameFolder = function(node) { + vm.noteActionService.renameFolder(node.id); + }; - $scope.removeNote = function (noteId) { - vm.noteActionService.removeNote(noteId, false) - } + $scope.removeNote = function(noteId) { + vm.noteActionService.removeNote(noteId, false); + }; - $scope.removeFolder = function (folderId) { - vm.noteActionService.removeFolder(folderId) - } + $scope.removeFolder = function(folderId) { + vm.noteActionService.removeFolder(folderId); + }; - $scope.emptyTrash = function () { - vm.noteActionService.emptyTrash() - } + $scope.emptyTrash = function() { + vm.noteActionService.emptyTrash(); + }; - $scope.clearAllParagraphOutput = function (noteId) { - vm.noteActionService.clearAllParagraphOutput(noteId) - } + $scope.clearAllParagraphOutput = function(noteId) { + vm.noteActionService.clearAllParagraphOutput(noteId); + }; - $scope.isFilterNote = function (note) { + $scope.isFilterNote = function(note) { if (!$scope.query.q) { - return true + return true; } - let noteName = note.name + let noteName = note.name; if (noteName.toLowerCase().indexOf($scope.query.q.toLowerCase()) > -1) { - return true + return true; } - return false - } + return false; + }; - $scope.getNoteName = function (note) { - return arrayOrderingSrv.getNoteName(note) - } + $scope.getNoteName = function(note) { + return arrayOrderingSrv.getNoteName(note); + }; - $scope.noteComparator = function (note1, note2) { - return arrayOrderingSrv.noteComparator(note1, note2) - } + $scope.noteComparator = function(note1, note2) { + return arrayOrderingSrv.noteComparator(note1, note2); + }; } diff --git a/zeppelin-web/src/app/interpreter/interpreter-item.directive.js b/zeppelin-web/src/app/interpreter/interpreter-item.directive.js index 4bde44d16c1..cfb109a12fc 100644 --- a/zeppelin-web/src/app/interpreter/interpreter-item.directive.js +++ b/zeppelin-web/src/app/interpreter/interpreter-item.directive.js @@ -12,20 +12,20 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').directive('interpreterItem', InterpreterItemDirective) +angular.module('zeppelinWebApp').directive('interpreterItem', InterpreterItemDirective); -function InterpreterItemDirective ($timeout) { - 'ngInject' +function InterpreterItemDirective($timeout) { + 'ngInject'; return { restrict: 'A', - link: function (scope, element, attr) { + link: function(scope, element, attr) { if (scope.$last === true) { - $timeout(function () { - let id = 'ngRenderFinished' - scope.$emit(id) - }) + $timeout(function() { + let id = 'ngRenderFinished'; + scope.$emit(id); + }); } - } - } + }, + }; } diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index ef8840240e7..060c6b6132b 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -12,548 +12,552 @@ * limitations under the License. */ -import { ParagraphStatus, } from '../notebook/paragraph/paragraph.status' +import {ParagraphStatus} from '../notebook/paragraph/paragraph.status'; -angular.module('zeppelinWebApp').controller('InterpreterCtrl', InterpreterCtrl) +angular.module('zeppelinWebApp').controller('InterpreterCtrl', InterpreterCtrl); function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeout, $route) { - 'ngInject' - - let interpreterSettingsTmp = [] - $scope.interpreterSettings = [] - $scope.availableInterpreters = {} - $scope.showAddNewSetting = false - $scope.showRepositoryInfo = false - $scope.searchInterpreter = '' - $scope._ = _ - $scope.interpreterPropertyTypes = [] - ngToast.dismiss() - - $scope.openPermissions = function () { - $scope.showInterpreterAuth = true - } - - $scope.closePermissions = function () { - $scope.showInterpreterAuth = false - } - - let getSelectJson = function () { + 'ngInject'; + + let interpreterSettingsTmp = []; + $scope.interpreterSettings = []; + $scope.availableInterpreters = {}; + $scope.showAddNewSetting = false; + $scope.showRepositoryInfo = false; + $scope.searchInterpreter = ''; + $scope._ = _; + $scope.interpreterPropertyTypes = []; + ngToast.dismiss(); + + $scope.openPermissions = function() { + $scope.showInterpreterAuth = true; + }; + + $scope.closePermissions = function() { + $scope.showInterpreterAuth = false; + }; + + let getSelectJson = function() { let selectJson = { tags: true, minimumInputLength: 3, multiple: true, tokenSeparators: [',', ' '], ajax: { - url: function (params) { + url: function(params) { if (!params.term) { - return false + return false; } - return baseUrlSrv.getRestApiBase() + '/security/userlist/' + params.term + return baseUrlSrv.getRestApiBase() + '/security/userlist/' + params.term; }, delay: 250, - processResults: function (data, params) { - let results = [] + processResults: function(data, params) { + let results = []; if (data.body.users.length !== 0) { - let users = [] + let users = []; for (let len = 0; len < data.body.users.length; len++) { users.push({ 'id': data.body.users[len], - 'text': data.body.users[len] - }) + 'text': data.body.users[len], + }); } results.push({ 'text': 'Users :', - 'children': users - }) + 'children': users, + }); } if (data.body.roles.length !== 0) { - let roles = [] + let roles = []; for (let len = 0; len < data.body.roles.length; len++) { roles.push({ 'id': data.body.roles[len], - 'text': data.body.roles[len] - }) + 'text': data.body.roles[len], + }); } results.push({ 'text': 'Roles :', - 'children': roles - }) + 'children': roles, + }); } return { results: results, pagination: { - more: false - } - } + more: false, + }, + }; }, - cache: false - } - } - return selectJson - } - - $scope.togglePermissions = function (intpName) { - angular.element('#' + intpName + 'Owners').select2(getSelectJson()) + cache: false, + }, + }; + return selectJson; + }; + + $scope.togglePermissions = function(intpName) { + angular.element('#' + intpName + 'Owners').select2(getSelectJson()); if ($scope.showInterpreterAuth) { - $scope.closePermissions() + $scope.closePermissions(); } else { - $scope.openPermissions() + $scope.openPermissions(); } - } + }; - $scope.$on('ngRenderFinished', function (event, data) { + $scope.$on('ngRenderFinished', function(event, data) { for (let setting = 0; setting < $scope.interpreterSettings.length; setting++) { - angular.element('#' + $scope.interpreterSettings[setting].name + 'Owners').select2(getSelectJson()) + angular.element('#' + $scope.interpreterSettings[setting].name + 'Owners').select2(getSelectJson()); } - }) + }); - let getInterpreterSettings = function () { + let getInterpreterSettings = function() { $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/setting') - .then(function (res) { - $scope.interpreterSettings = res.data.body - checkDownloadingDependencies() - }).catch(function (res) { + .then(function(res) { + $scope.interpreterSettings = res.data.body; + checkDownloadingDependencies(); + }).catch(function(res) { if (res.status === 401) { ngToast.danger({ content: 'You don\'t have permission on this page', verticalPosition: 'bottom', - timeout: '3000' - }) - setTimeout(function () { - window.location = baseUrlSrv.getBase() - }, 3000) + timeout: '3000', + }); + setTimeout(function() { + window.location = baseUrlSrv.getBase(); + }, 3000); } - console.log('Error %o %o', res.status, res.data ? res.data.message : '') - }) - } + console.log('Error %o %o', res.status, res.data ? res.data.message : ''); + }); + }; - const checkDownloadingDependencies = function () { - let isDownloading = false + const checkDownloadingDependencies = function() { + let isDownloading = false; for (let index = 0; index < $scope.interpreterSettings.length; index++) { - let setting = $scope.interpreterSettings[index] + let setting = $scope.interpreterSettings[index]; if (setting.status === 'DOWNLOADING_DEPENDENCIES') { - isDownloading = true + isDownloading = true; } if (setting.status === ParagraphStatus.ERROR || setting.errorReason) { ngToast.danger({content: 'Error setting properties for interpreter \'' + setting.group + '.' + setting.name + '\': ' + setting.errorReason, verticalPosition: 'top', - dismissOnTimeout: false - }) + dismissOnTimeout: false, + }); } } if (isDownloading) { - $timeout(function () { + $timeout(function() { if ($route.current.$$route.originalPath === '/interpreter') { - getInterpreterSettings() + getInterpreterSettings(); } - }, 2000) + }, 2000); } - } + }; - let getAvailableInterpreters = function () { - $http.get(baseUrlSrv.getRestApiBase() + '/interpreter').then(function (res) { - $scope.availableInterpreters = res.data.body - }).catch(function (res) { - console.log('Error %o %o', res.status, res.data ? res.data.message : '') - }) - } + let getAvailableInterpreters = function() { + $http.get(baseUrlSrv.getRestApiBase() + '/interpreter').then(function(res) { + $scope.availableInterpreters = res.data.body; + }).catch(function(res) { + console.log('Error %o %o', res.status, res.data ? res.data.message : ''); + }); + }; - let getAvailableInterpreterPropertyWidgets = function () { + let getAvailableInterpreterPropertyWidgets = function() { $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/property/types') - .then(function (res) { - $scope.interpreterPropertyTypes = res.data.body - }).catch(function (res) { - console.log('Error %o %o', res.status, res.data ? res.data.message : '') - }) - } + .then(function(res) { + $scope.interpreterPropertyTypes = res.data.body; + }).catch(function(res) { + console.log('Error %o %o', res.status, res.data ? res.data.message : ''); + }); + }; let emptyNewProperty = function(object) { - angular.extend(object, {propertyValue: '', propertyKey: '', propertyType: $scope.interpreterPropertyTypes[0]}) - } + angular.extend(object, {propertyValue: '', propertyKey: '', propertyType: $scope.interpreterPropertyTypes[0]}); + }; - let emptyNewDependency = function (object) { - angular.extend(object, {depArtifact: '', depExclude: ''}) - } + let emptyNewDependency = function(object) { + angular.extend(object, {depArtifact: '', depExclude: ''}); + }; - let removeTMPSettings = function (index) { - interpreterSettingsTmp.splice(index, 1) - } + let removeTMPSettings = function(index) { + interpreterSettingsTmp.splice(index, 1); + }; - $scope.copyOriginInterpreterSettingProperties = function (settingId) { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - interpreterSettingsTmp[index] = angular.copy($scope.interpreterSettings[index]) - } + $scope.copyOriginInterpreterSettingProperties = function(settingId) { + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + interpreterSettingsTmp[index] = angular.copy($scope.interpreterSettings[index]); + }; - $scope.setPerNoteOption = function (settingId, sessionOption) { - let option + $scope.setPerNoteOption = function(settingId, sessionOption) { + let option; if (settingId === undefined) { - option = $scope.newInterpreterSetting.option + option = $scope.newInterpreterSetting.option; } else { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - let setting = $scope.interpreterSettings[index] - option = setting.option + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + let setting = $scope.interpreterSettings[index]; + option = setting.option; } if (sessionOption === 'isolated') { - option.perNote = sessionOption - option.session = false - option.process = true + option.perNote = sessionOption; + option.session = false; + option.process = true; } else if (sessionOption === 'scoped') { - option.perNote = sessionOption - option.session = true - option.process = false + option.perNote = sessionOption; + option.session = true; + option.process = false; } else { - option.perNote = 'shared' - option.session = false - option.process = false + option.perNote = 'shared'; + option.session = false; + option.process = false; } - } + }; - $scope.defaultValueByType = function (setting) { + $scope.defaultValueByType = function(setting) { if (setting.propertyType === 'checkbox') { - setting.propertyValue = false - return + setting.propertyValue = false; + return; } - setting.propertyValue = '' - } + setting.propertyValue = ''; + }; - $scope.setPerUserOption = function (settingId, sessionOption) { - let option + $scope.setPerUserOption = function(settingId, sessionOption) { + let option; if (settingId === undefined) { - option = $scope.newInterpreterSetting.option + option = $scope.newInterpreterSetting.option; } else { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - let setting = $scope.interpreterSettings[index] - option = setting.option + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + let setting = $scope.interpreterSettings[index]; + option = setting.option; } if (sessionOption === 'isolated') { - option.perUser = sessionOption - option.session = false - option.process = true + option.perUser = sessionOption; + option.session = false; + option.process = true; } else if (sessionOption === 'scoped') { - option.perUser = sessionOption - option.session = true - option.process = false + option.perUser = sessionOption; + option.session = true; + option.process = false; } else { - option.perUser = 'shared' - option.session = false - option.process = false + option.perUser = 'shared'; + option.session = false; + option.process = false; } - } + }; - $scope.getPerNoteOption = function (settingId) { - let option + $scope.getPerNoteOption = function(settingId) { + let option; if (settingId === undefined) { - option = $scope.newInterpreterSetting.option + option = $scope.newInterpreterSetting.option; } else { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - let setting = $scope.interpreterSettings[index] - option = setting.option + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + let setting = $scope.interpreterSettings[index]; + option = setting.option; } if (option.perNote === 'scoped') { - return 'scoped' + return 'scoped'; } else if (option.perNote === 'isolated') { - return 'isolated' + return 'isolated'; } else { - return 'shared' + return 'shared'; } - } + }; - $scope.getPerUserOption = function (settingId) { - let option + $scope.getPerUserOption = function(settingId) { + let option; if (settingId === undefined) { - option = $scope.newInterpreterSetting.option + option = $scope.newInterpreterSetting.option; } else { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - let setting = $scope.interpreterSettings[index] - option = setting.option + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + let setting = $scope.interpreterSettings[index]; + option = setting.option; } if (option.perUser === 'scoped') { - return 'scoped' + return 'scoped'; } else if (option.perUser === 'isolated') { - return 'isolated' + return 'isolated'; } else { - return 'shared' + return 'shared'; } - } + }; - $scope.getInterpreterRunningOption = function (settingId) { - let sharedModeName = 'shared' + $scope.getInterpreterRunningOption = function(settingId) { + let sharedModeName = 'shared'; - let globallyModeName = 'Globally' - let perNoteModeName = 'Per Note' - let perUserModeName = 'Per User' + let globallyModeName = 'Globally'; + let perNoteModeName = 'Per Note'; + let perUserModeName = 'Per User'; - let option + let option; if (settingId === undefined) { - option = $scope.newInterpreterSetting.option + option = $scope.newInterpreterSetting.option; } else { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - let setting = $scope.interpreterSettings[index] - option = setting.option + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + let setting = $scope.interpreterSettings[index]; + option = setting.option; } - let perNote = option.perNote - let perUser = option.perUser + let perNote = option.perNote; + let perUser = option.perUser; // Globally == shared_perNote + shared_perUser if (perNote === sharedModeName && perUser === sharedModeName) { - return globallyModeName + return globallyModeName; } if ($rootScope.ticket.ticket === 'anonymous' && $rootScope.ticket.roles === '[]') { if (perNote !== undefined && typeof perNote === 'string' && perNote !== '') { - return perNoteModeName + return perNoteModeName; } } else if ($rootScope.ticket.ticket !== 'anonymous') { if (perNote !== undefined && typeof perNote === 'string' && perNote !== '') { if (perUser !== undefined && typeof perUser === 'string' && perUser !== '') { - return perUserModeName + return perUserModeName; } - return perNoteModeName + return perNoteModeName; } } - option.perNote = sharedModeName - option.perUser = sharedModeName - return globallyModeName - } + option.perNote = sharedModeName; + option.perUser = sharedModeName; + return globallyModeName; + }; - $scope.setInterpreterRunningOption = function (settingId, isPerNoteMode, isPerUserMode) { - let option + $scope.setInterpreterRunningOption = function(settingId, isPerNoteMode, isPerUserMode) { + let option; if (settingId === undefined) { - option = $scope.newInterpreterSetting.option + option = $scope.newInterpreterSetting.option; } else { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - let setting = $scope.interpreterSettings[index] - option = setting.option + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + let setting = $scope.interpreterSettings[index]; + option = setting.option; } - option.perNote = isPerNoteMode - option.perUser = isPerUserMode - } + option.perNote = isPerNoteMode; + option.perUser = isPerUserMode; + }; - $scope.updateInterpreterSetting = function (form, settingId) { + $scope.updateInterpreterSetting = function(form, settingId) { const thisConfirm = BootstrapDialog.confirm({ closable: false, closeByBackdrop: false, closeByKeyboard: false, title: '', message: 'Do you want to update this interpreter and restart with new settings?', - callback: function (result) { + callback: function(result) { if (result) { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - let setting = $scope.interpreterSettings[index] + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + let setting = $scope.interpreterSettings[index]; if (setting.propertyKey !== '' || setting.propertyKey) { - $scope.addNewInterpreterProperty(settingId) + $scope.addNewInterpreterProperty(settingId); } if (setting.depArtifact !== '' || setting.depArtifact) { - $scope.addNewInterpreterDependency(settingId) + $scope.addNewInterpreterDependency(settingId); } // add missing field of option if (!setting.option) { - setting.option = {} + setting.option = {}; } if (setting.option.isExistingProcess === undefined) { - setting.option.isExistingProcess = false + setting.option.isExistingProcess = false; } if (setting.option.setPermission === undefined) { - setting.option.setPermission = false + setting.option.setPermission = false; } if (setting.option.isUserImpersonate === undefined) { - setting.option.isUserImpersonate = false + setting.option.isUserImpersonate = false; } if (!($scope.getInterpreterRunningOption(settingId) === 'Per User' && $scope.getPerUserOption(settingId) === 'isolated')) { - setting.option.isUserImpersonate = false + setting.option.isUserImpersonate = false; } if (setting.option.remote === undefined) { // remote always true for now - setting.option.remote = true + setting.option.remote = true; } - setting.option.owners = angular.element('#' + setting.name + 'Owners').val() + setting.option.owners = angular.element('#' + setting.name + 'Owners').val(); let request = { option: angular.copy(setting.option), properties: angular.copy(setting.properties), - dependencies: angular.copy(setting.dependencies) - } + dependencies: angular.copy(setting.dependencies), + }; - thisConfirm.$modalFooter.find('button').addClass('disabled') + thisConfirm.$modalFooter.find('button').addClass('disabled'); thisConfirm.$modalFooter.find('button:contains("OK")') - .html(' Saving Setting') + .html(' Saving Setting'); $http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId, request) - .then(function (res) { - $scope.interpreterSettings[index] = res.data.body - removeTMPSettings(index) - checkDownloadingDependencies() - thisConfirm.close() - }) - .catch(function (res) { - const message = res.data ? res.data.message : 'Could not connect to server.' - console.log('Error %o %o', res.status, message) - ngToast.danger({content: message, verticalPosition: 'bottom'}) - form.$show() - thisConfirm.close() + .then(function(res) { + $scope.interpreterSettings[index] = res.data.body; + removeTMPSettings(index); + checkDownloadingDependencies(); + thisConfirm.close(); }) - return false + .catch(function(res) { + const message = res.data ? res.data.message : 'Could not connect to server.'; + console.log('Error %o %o', res.status, message); + ngToast.danger({content: message, verticalPosition: 'bottom'}); + form.$show(); + thisConfirm.close(); + }); + return false; } else { - form.$show() + form.$show(); } - } - }) - } + }, + }); + }; - $scope.resetInterpreterSetting = function (settingId) { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) + $scope.resetInterpreterSetting = function(settingId) { + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); // Set the old settings back - $scope.interpreterSettings[index] = angular.copy(interpreterSettingsTmp[index]) - removeTMPSettings(index) - } + $scope.interpreterSettings[index] = angular.copy(interpreterSettingsTmp[index]); + removeTMPSettings(index); + }; - $scope.removeInterpreterSetting = function (settingId) { + $scope.removeInterpreterSetting = function(settingId) { BootstrapDialog.confirm({ closable: true, title: '', message: 'Do you want to delete this interpreter setting?', - callback: function (result) { + callback: function(result) { if (result) { $http.delete(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId) - .then(function (res) { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - $scope.interpreterSettings.splice(index, 1) - }).catch(function (res) { - console.log('Error %o %o', res.status, res.data ? res.data.message : '') - }) + .then(function(res) { + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + $scope.interpreterSettings.splice(index, 1); + }).catch(function(res) { + console.log('Error %o %o', res.status, res.data ? res.data.message : ''); + }); } - } - }) - } + }, + }); + }; - $scope.newInterpreterGroupChange = function () { + $scope.newInterpreterGroupChange = function() { let el = _.pluck(_.filter($scope.availableInterpreters, {'name': $scope.newInterpreterSetting.group}), - 'properties') - let properties = {} + 'properties'); + let properties = {}; for (let i = 0; i < el.length; i++) { - let intpInfo = el[i] + let intpInfo = el[i]; for (let key in intpInfo) { - properties[key] = { - value: intpInfo[key].defaultValue, - description: intpInfo[key].description, - type: intpInfo[key].type + if (intpInfo.hasOwnProperty(key)) { + properties[key] = { + value: intpInfo[key].defaultValue, + description: intpInfo[key].description, + type: intpInfo[key].type, + }; } } } - $scope.newInterpreterSetting.properties = properties - } + $scope.newInterpreterSetting.properties = properties; + }; - $scope.restartInterpreterSetting = function (settingId) { + $scope.restartInterpreterSetting = function(settingId) { BootstrapDialog.confirm({ closable: true, title: '', message: 'Do you want to restart this interpreter?', - callback: function (result) { + callback: function(result) { if (result) { $http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/restart/' + settingId) - .then(function (res) { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - $scope.interpreterSettings[index] = res.data.body - ngToast.info('Interpreter stopped. Will be lazily started on next run.') - }).catch(function (res) { - let errorMsg = (res.data !== null) ? res.data.message : 'Could not connect to server.' - console.log('Error %o %o', res.status, errorMsg) - ngToast.danger(errorMsg) - }) + .then(function(res) { + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + $scope.interpreterSettings[index] = res.data.body; + ngToast.info('Interpreter stopped. Will be lazily started on next run.'); + }).catch(function(res) { + let errorMsg = (res.data !== null) ? res.data.message : 'Could not connect to server.'; + console.log('Error %o %o', res.status, errorMsg); + ngToast.danger(errorMsg); + }); } - } - }) - } + }, + }); + }; - $scope.addNewInterpreterSetting = function () { + $scope.addNewInterpreterSetting = function() { // user input validation on interpreter creation if (!$scope.newInterpreterSetting.name || !$scope.newInterpreterSetting.name.trim() || !$scope.newInterpreterSetting.group) { BootstrapDialog.alert({ closable: true, title: 'Add interpreter', - message: 'Please fill in interpreter name and choose a group' - }) - return + message: 'Please fill in interpreter name and choose a group', + }); + return; } if ($scope.newInterpreterSetting.name.indexOf('.') >= 0) { BootstrapDialog.alert({ closable: true, title: 'Add interpreter', - message: '\'.\' is invalid for interpreter name' - }) - return + message: '\'.\' is invalid for interpreter name', + }); + return; } if (_.findIndex($scope.interpreterSettings, {'name': $scope.newInterpreterSetting.name}) >= 0) { BootstrapDialog.alert({ closable: true, title: 'Add interpreter', - message: 'Name ' + $scope.newInterpreterSetting.name + ' already exists' - }) - return + message: 'Name ' + $scope.newInterpreterSetting.name + ' already exists', + }); + return; } - let newSetting = $scope.newInterpreterSetting + let newSetting = $scope.newInterpreterSetting; if (newSetting.propertyKey !== '' || newSetting.propertyKey) { - $scope.addNewInterpreterProperty() + $scope.addNewInterpreterProperty(); } if (newSetting.depArtifact !== '' || newSetting.depArtifact) { - $scope.addNewInterpreterDependency() + $scope.addNewInterpreterDependency(); } if (newSetting.option.setPermission === undefined) { - newSetting.option.setPermission = false + newSetting.option.setPermission = false; } - newSetting.option.owners = angular.element('#newInterpreterOwners').val() + newSetting.option.owners = angular.element('#newInterpreterOwners').val(); - let request = angular.copy($scope.newInterpreterSetting) + let request = angular.copy($scope.newInterpreterSetting); // Change properties to proper request format - let newProperties = {} + let newProperties = {}; for (let p in newSetting.properties) { - newProperties[p] = { - value: newSetting.properties[p].value, - type: newSetting.properties[p].type, - name: p + if (newSetting.properties.hasOwnProperty(p)) { + newProperties[p] = { + value: newSetting.properties[p].value, + type: newSetting.properties[p].type, + name: p, + }; } } - request.properties = newProperties + request.properties = newProperties; $http.post(baseUrlSrv.getRestApiBase() + '/interpreter/setting', request) - .then(function (res) { - $scope.resetNewInterpreterSetting() - getInterpreterSettings() - $scope.showAddNewSetting = false - checkDownloadingDependencies() - }).catch(function (res) { - const errorMsg = res.data ? res.data.message : 'Could not connect to server.' - console.log('Error %o %o', res.status, errorMsg) - ngToast.danger({content: errorMsg, verticalPosition: 'bottom'}) - }) - } - - $scope.cancelInterpreterSetting = function () { - $scope.showAddNewSetting = false - $scope.resetNewInterpreterSetting() - } - - $scope.resetNewInterpreterSetting = function () { + .then(function(res) { + $scope.resetNewInterpreterSetting(); + getInterpreterSettings(); + $scope.showAddNewSetting = false; + checkDownloadingDependencies(); + }).catch(function(res) { + const errorMsg = res.data ? res.data.message : 'Could not connect to server.'; + console.log('Error %o %o', res.status, errorMsg); + ngToast.danger({content: errorMsg, verticalPosition: 'bottom'}); + }); + }; + + $scope.cancelInterpreterSetting = function() { + $scope.showAddNewSetting = false; + $scope.resetNewInterpreterSetting(); + }; + + $scope.resetNewInterpreterSetting = function() { $scope.newInterpreterSetting = { name: undefined, group: undefined, @@ -564,94 +568,94 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeou isExistingProcess: false, setPermission: false, session: false, - process: false + process: false, - } - } - emptyNewProperty($scope.newInterpreterSetting) - } + }, + }; + emptyNewProperty($scope.newInterpreterSetting); + }; - $scope.removeInterpreterProperty = function (key, settingId) { + $scope.removeInterpreterProperty = function(key, settingId) { if (settingId === undefined) { - delete $scope.newInterpreterSetting.properties[key] + delete $scope.newInterpreterSetting.properties[key]; } else { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - delete $scope.interpreterSettings[index].properties[key] + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + delete $scope.interpreterSettings[index].properties[key]; } - } + }; - $scope.removeInterpreterDependency = function (artifact, settingId) { + $scope.removeInterpreterDependency = function(artifact, settingId) { if (settingId === undefined) { $scope.newInterpreterSetting.dependencies = _.reject($scope.newInterpreterSetting.dependencies, - function (el) { - return el.groupArtifactVersion === artifact - }) + function(el) { + return el.groupArtifactVersion === artifact; + }); } else { - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); $scope.interpreterSettings[index].dependencies = _.reject($scope.interpreterSettings[index].dependencies, - function (el) { - return el.groupArtifactVersion === artifact - }) + function(el) { + return el.groupArtifactVersion === artifact; + }); } - } + }; - $scope.addNewInterpreterProperty = function (settingId) { + $scope.addNewInterpreterProperty = function(settingId) { if (settingId === undefined) { // Add new property from create form if (!$scope.newInterpreterSetting.propertyKey || $scope.newInterpreterSetting.propertyKey === '') { - return + return; } $scope.newInterpreterSetting.properties[$scope.newInterpreterSetting.propertyKey] = { value: $scope.newInterpreterSetting.propertyValue, - type: $scope.newInterpreterSetting.propertyType - } - emptyNewProperty($scope.newInterpreterSetting) + type: $scope.newInterpreterSetting.propertyType, + }; + emptyNewProperty($scope.newInterpreterSetting); } else { // Add new property from edit form - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - let setting = $scope.interpreterSettings[index] + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + let setting = $scope.interpreterSettings[index]; if (!setting.propertyKey || setting.propertyKey === '') { - return + return; } setting.properties[setting.propertyKey] = - {value: setting.propertyValue, type: setting.propertyType} + {value: setting.propertyValue, type: setting.propertyType}; - emptyNewProperty(setting) + emptyNewProperty(setting); } - } + }; - $scope.addNewInterpreterDependency = function (settingId) { + $scope.addNewInterpreterDependency = function(settingId) { if (settingId === undefined) { // Add new dependency from create form if (!$scope.newInterpreterSetting.depArtifact || $scope.newInterpreterSetting.depArtifact === '') { - return + return; } // overwrite if artifact already exists - let newSetting = $scope.newInterpreterSetting + let newSetting = $scope.newInterpreterSetting; for (let d in newSetting.dependencies) { if (newSetting.dependencies[d].groupArtifactVersion === newSetting.depArtifact) { newSetting.dependencies[d] = { 'groupArtifactVersion': newSetting.depArtifact, - 'exclusions': newSetting.depExclude - } - newSetting.dependencies.splice(d, 1) + 'exclusions': newSetting.depExclude, + }; + newSetting.dependencies.splice(d, 1); } } newSetting.dependencies.push({ 'groupArtifactVersion': newSetting.depArtifact, - 'exclusions': (newSetting.depExclude === '') ? [] : newSetting.depExclude - }) - emptyNewDependency(newSetting) + 'exclusions': (newSetting.depExclude === '') ? [] : newSetting.depExclude, + }); + emptyNewDependency(newSetting); } else { // Add new dependency from edit form - let index = _.findIndex($scope.interpreterSettings, {'id': settingId}) - let setting = $scope.interpreterSettings[index] + let index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + let setting = $scope.interpreterSettings[index]; if (!setting.depArtifact || setting.depArtifact === '') { - return + return; } // overwrite if artifact already exists @@ -659,21 +663,21 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeou if (setting.dependencies[dep].groupArtifactVersion === setting.depArtifact) { setting.dependencies[dep] = { 'groupArtifactVersion': setting.depArtifact, - 'exclusions': setting.depExclude - } - setting.dependencies.splice(dep, 1) + 'exclusions': setting.depExclude, + }; + setting.dependencies.splice(dep, 1); } } setting.dependencies.push({ 'groupArtifactVersion': setting.depArtifact, - 'exclusions': (setting.depExclude === '') ? [] : setting.depExclude - }) - emptyNewDependency(setting) + 'exclusions': (setting.depExclude === '') ? [] : setting.depExclude, + }); + emptyNewDependency(setting); } - } + }; - $scope.resetNewRepositorySetting = function () { + $scope.resetNewRepositorySetting = function() { $scope.newRepoSetting = { id: '', url: '', @@ -684,102 +688,102 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeou proxyHost: '', proxyPort: null, proxyLogin: '', - proxyPassword: '' - } - } + proxyPassword: '', + }; + }; - let getRepositories = function () { + let getRepositories = function() { $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/repository') - .success(function (data, status, headers, config) { - $scope.repositories = data.body - }).error(function (data, status, headers, config) { - console.log('Error %o %o', status, data.message) - }) - } + .success(function(data, status, headers, config) { + $scope.repositories = data.body; + }).error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); + }; - $scope.addNewRepository = function () { - let request = angular.copy($scope.newRepoSetting) + $scope.addNewRepository = function() { + let request = angular.copy($scope.newRepoSetting); $http.post(baseUrlSrv.getRestApiBase() + '/interpreter/repository', request) - .then(function (res) { - getRepositories() - $scope.resetNewRepositorySetting() - angular.element('#repoModal').modal('hide') - }).catch(function (res) { - console.log('Error %o %o', res.headers, res.config) - }) - } - - $scope.removeRepository = function (repoId) { + .then(function(res) { + getRepositories(); + $scope.resetNewRepositorySetting(); + angular.element('#repoModal').modal('hide'); + }).catch(function(res) { + console.log('Error %o %o', res.headers, res.config); + }); + }; + + $scope.removeRepository = function(repoId) { BootstrapDialog.confirm({ closable: true, title: '', message: 'Do you want to delete this repository?', - callback: function (result) { + callback: function(result) { if (result) { $http.delete(baseUrlSrv.getRestApiBase() + '/interpreter/repository/' + repoId) - .then(function (res) { - let index = _.findIndex($scope.repositories, {'id': repoId}) - $scope.repositories.splice(index, 1) - }).catch(function (res) { - console.log('Error %o %o', res.status, res.data ? res.data.message : '') - }) + .then(function(res) { + let index = _.findIndex($scope.repositories, {'id': repoId}); + $scope.repositories.splice(index, 1); + }).catch(function(res) { + console.log('Error %o %o', res.status, res.data ? res.data.message : ''); + }); } - } - }) - } + }, + }); + }; - $scope.isDefaultRepository = function (repoId) { + $scope.isDefaultRepository = function(repoId) { if (repoId === 'central' || repoId === 'local') { - return true + return true; } else { - return false + return false; } - } + }; - $scope.showErrorMessage = function (setting) { + $scope.showErrorMessage = function(setting) { BootstrapDialog.show({ title: 'Error downloading dependencies', - message: setting.errorReason - }) - } + message: setting.errorReason, + }); + }; let init = function() { - getAvailableInterpreterPropertyWidgets() + getAvailableInterpreterPropertyWidgets(); - $scope.resetNewInterpreterSetting() - $scope.resetNewRepositorySetting() + $scope.resetNewInterpreterSetting(); + $scope.resetNewRepositorySetting(); - getInterpreterSettings() - getAvailableInterpreters() - getRepositories() - } + getInterpreterSettings(); + getAvailableInterpreters(); + getRepositories(); + }; - $scope.showSparkUI = function (settingId) { + $scope.showSparkUI = function(settingId) { $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/metadata/' + settingId) - .then(function (res) { + .then(function(res) { if (res.data.body === undefined) { BootstrapDialog.alert({ - message: 'No spark application running' - }) - return + message: 'No spark application running', + }); + return; } if (res.data.body.url) { - window.open(res.data.body.url, '_blank') + window.open(res.data.body.url, '_blank'); } else { BootstrapDialog.alert({ - message: res.data.body.message - }) + message: res.data.body.message, + }); } - }).catch(function (res) { - console.log('Error %o %o', res.status, res.data ? res.data.message : '') - }) - } + }).catch(function(res) { + console.log('Error %o %o', res.status, res.data ? res.data.message : ''); + }); + }; $scope.getInterpreterBindingModeDocsLink = function() { - const currentVersion = $rootScope.zeppelinVersion - return `https://zeppelin.apache.org/docs/${currentVersion}/usage/interpreter/interpreter_binding_mode.html` - } + const currentVersion = $rootScope.zeppelinVersion; + return `https://zeppelin.apache.org/docs/${currentVersion}/usage/interpreter/interpreter_binding_mode.html`; + }; - init() + init(); } diff --git a/zeppelin-web/src/app/interpreter/interpreter.filter.js b/zeppelin-web/src/app/interpreter/interpreter.filter.js index 3f42572015a..7b5ace0298a 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.filter.js +++ b/zeppelin-web/src/app/interpreter/interpreter.filter.js @@ -12,11 +12,11 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').filter('sortByKey', sortByKey) +angular.module('zeppelinWebApp').filter('sortByKey', sortByKey); -function sortByKey () { - return function (properties) { - let sortedKeys = properties ? Object.keys(properties) : [] - return sortedKeys.sort() - } +function sortByKey() { + return function(properties) { + let sortedKeys = properties ? Object.keys(properties) : []; + return sortedKeys.sort(); + }; } diff --git a/zeppelin-web/src/app/interpreter/widget/number-widget.directive.js b/zeppelin-web/src/app/interpreter/widget/number-widget.directive.js index 2046b94d924..6ea129ad903 100644 --- a/zeppelin-web/src/app/interpreter/widget/number-widget.directive.js +++ b/zeppelin-web/src/app/interpreter/widget/number-widget.directive.js @@ -12,20 +12,20 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').directive('numberWidget', InterpreterNumberDirective) +angular.module('zeppelinWebApp').directive('numberWidget', InterpreterNumberDirective); function InterpreterNumberDirective() { return { require: 'ngModel', - link: function (scope, element, attrs, modelCtrl) { - modelCtrl.$parsers.push(function (inputValue) { - let transformedInput = inputValue ? inputValue.replace(/[^\d.-]/g, '') : null + link: function(scope, element, attrs, modelCtrl) { + modelCtrl.$parsers.push(function(inputValue) { + let transformedInput = inputValue ? inputValue.replace(/[^\d.-]/g, '') : null; if (transformedInput !== inputValue) { - modelCtrl.$setViewValue(transformedInput) - modelCtrl.$render() + modelCtrl.$setViewValue(transformedInput); + modelCtrl.$render(); } - return transformedInput - }) - } - } + return transformedInput; + }); + }, + }; } diff --git a/zeppelin-web/src/app/jobmanager/job-status.js b/zeppelin-web/src/app/jobmanager/job-status.js index eda41b1a38f..d918299f724 100644 --- a/zeppelin-web/src/app/jobmanager/job-status.js +++ b/zeppelin-web/src/app/jobmanager/job-status.js @@ -19,36 +19,36 @@ export const JobStatus = { ERROR: 'ERROR', PENDING: 'PENDING', RUNNING: 'RUNNING', -} +}; export function getJobIconByStatus(jobStatus) { if (jobStatus === JobStatus.READY) { - return 'fa fa-circle-o' + return 'fa fa-circle-o'; } else if (jobStatus === JobStatus.FINISHED) { - return 'fa fa-circle' + return 'fa fa-circle'; } else if (jobStatus === JobStatus.ABORT) { - return 'fa fa-circle' + return 'fa fa-circle'; } else if (jobStatus === JobStatus.ERROR) { - return 'fa fa-circle' + return 'fa fa-circle'; } else if (jobStatus === JobStatus.PENDING) { - return 'fa fa-circle' + return 'fa fa-circle'; } else if (jobStatus === JobStatus.RUNNING) { - return 'fa fa-spinner' + return 'fa fa-spinner'; } } export function getJobColorByStatus(jobStatus) { if (jobStatus === JobStatus.READY) { - return 'green' + return 'green'; } else if (jobStatus === JobStatus.FINISHED) { - return 'green' + return 'green'; } else if (jobStatus === JobStatus.ABORT) { - return 'orange' + return 'orange'; } else if (jobStatus === JobStatus.ERROR) { - return 'red' + return 'red'; } else if (jobStatus === JobStatus.PENDING) { - return 'gray' + return 'gray'; } else if (jobStatus === JobStatus.RUNNING) { - return 'blue' + return 'blue'; } } diff --git a/zeppelin-web/src/app/jobmanager/job/job.component.js b/zeppelin-web/src/app/jobmanager/job/job.component.js index c4d4f514307..e6f102f2387 100644 --- a/zeppelin-web/src/app/jobmanager/job/job.component.js +++ b/zeppelin-web/src/app/jobmanager/job/job.component.js @@ -12,35 +12,35 @@ * limitations under the License. */ -import moment from 'moment' +import moment from 'moment'; -import { ParagraphStatus, } from '../../notebook/paragraph/paragraph.status' -import { getJobColorByStatus, getJobIconByStatus } from '../job-status' +import {ParagraphStatus} from '../../notebook/paragraph/paragraph.status'; +import {getJobColorByStatus, getJobIconByStatus} from '../job-status'; -import jobTemplate from './job.html' -import './job.css' +import jobTemplate from './job.html'; +import './job.css'; class JobController { constructor($http, JobManagerService) { - 'ngInject' - this.$http = $http - this.JobManagerService = JobManagerService + 'ngInject'; + this.$http = $http; + this.JobManagerService = JobManagerService; } isRunning() { - return this.note.isRunningJob + return this.note.isRunningJob; } getParagraphs() { - return this.note.paragraphs + return this.note.paragraphs; } getNoteId() { - return this.note.noteId + return this.note.noteId; } getNoteName() { - return this.note.noteName + return this.note.noteName; } runJob() { @@ -48,19 +48,21 @@ class JobController { closable: true, title: 'Job Dialog', message: 'Run all paragraphs?', - callback: clickOk => { - if (!clickOk) { return } + callback: (clickOk) => { + if (!clickOk) { + return; + } - const noteId = this.getNoteId() + const noteId = this.getNoteId(); // if the request is handled successfully, the job page will get updated using websocket this.JobManagerService.sendRunJobRequest(noteId) - .catch(response => { + .catch((response) => { let message = (response.data && response.data.message) - ? response.data.message : 'SERVER ERROR' - this.showErrorDialog('Execution Failure', message) - }) - } - }) + ? response.data.message : 'SERVER ERROR'; + this.showErrorDialog('Execution Failure', message); + }); + }, + }); } stopJob() { @@ -68,81 +70,85 @@ class JobController { closable: true, title: 'Job Dialog', message: 'Stop all paragraphs?', - callback: clickOk => { - if (!clickOk) { return } + callback: (clickOk) => { + if (!clickOk) { + return; + } - const noteId = this.getNoteId() + const noteId = this.getNoteId(); // if the request is handled successfully, the job page will get updated using websocket this.JobManagerService.sendStopJobRequest(noteId) - .catch(response => { + .catch((response) => { let message = (response.data && response.data.message) - ? response.data.message : 'SERVER ERROR' - this.showErrorDialog('Stop Failure', message) - }) - } - }) + ? response.data.message : 'SERVER ERROR'; + this.showErrorDialog('Stop Failure', message); + }); + }, + }); } showErrorDialog(title, errorMessage) { - if (!errorMessage) { errorMessage = 'SERVER ERROR' } + if (!errorMessage) { + errorMessage = 'SERVER ERROR'; + } BootstrapDialog.alert({ closable: true, title: title, - message: errorMessage - }) + message: errorMessage, + }); } lastExecuteTime() { - const timestamp = this.note.unixTimeLastRun - return moment.unix(timestamp / 1000).fromNow() + const timestamp = this.note.unixTimeLastRun; + return moment.unix(timestamp / 1000).fromNow(); } getInterpreterName() { return typeof this.note.interpreter === 'undefined' - ? 'interpreter is not set' : this.note.interpreter + ? 'interpreter is not set' : this.note.interpreter; } getInterpreterNameStyle() { return typeof this.note.interpreter === 'undefined' - ? { color: 'gray' } : { color: 'black' } + ? {color: 'gray'} : {color: 'black'}; } getJobTypeIcon() { - const noteType = this.note.noteType + const noteType = this.note.noteType; if (noteType === 'normal') { - return 'icon-doc' + return 'icon-doc'; } else if (noteType === 'cron') { - return 'icon-clock' + return 'icon-clock'; } else { - return 'icon-question' + return 'icon-question'; } } getJobColorByStatus(status) { - return getJobColorByStatus(status) + return getJobColorByStatus(status); } getJobIconByStatus(status) { - return getJobIconByStatus(status) + return getJobIconByStatus(status); } getProgress() { - const paragraphs = this.getParagraphs() - let paragraphStatuses = paragraphs.map(p => p.status) - let runningOrFinishedParagraphs = paragraphStatuses.filter(status => { - return status === ParagraphStatus.RUNNING || status === ParagraphStatus.FINISHED - }) + const paragraphs = this.getParagraphs(); + let paragraphStatuses = paragraphs.map((p) => p.status); + let runningOrFinishedParagraphs = paragraphStatuses.filter((status) => { + return status === ParagraphStatus.RUNNING || status === ParagraphStatus.FINISHED; + }); - let totalCount = paragraphStatuses.length - let runningCount = runningOrFinishedParagraphs.length - let result = Math.ceil(runningCount / totalCount * 100) - result = isNaN(result) ? 0 : result + let totalCount = paragraphStatuses.length; + let runningCount = runningOrFinishedParagraphs.length; + let result = Math.ceil(runningCount / totalCount * 100); + result = isNaN(result) ? 0 : result; - return `${result}%` + return `${result}%`; } showPercentProgressBar() { - return this.getProgress() > 0 && this.getProgress() < 100 + return this.getProgress() > 0 && this.getProgress() < 100; } } @@ -152,9 +158,9 @@ export const JobComponent = { }, template: jobTemplate, controller: JobController, -} +}; export const JobModule = angular .module('zeppelinWebApp') .component('job', JobComponent) - .name + .name; diff --git a/zeppelin-web/src/app/jobmanager/job/job.component.test.js b/zeppelin-web/src/app/jobmanager/job/job.component.test.js index 6ca285cc89d..5b6bec1a6d4 100644 --- a/zeppelin-web/src/app/jobmanager/job/job.component.test.js +++ b/zeppelin-web/src/app/jobmanager/job/job.component.test.js @@ -1,55 +1,55 @@ -import { ParagraphStatus } from '../../notebook/paragraph/paragraph.status' +import {ParagraphStatus} from '../../notebook/paragraph/paragraph.status'; describe('JobComponent', () => { - let $componentController + let $componentController; - beforeEach(angular.mock.module('zeppelinWebApp')) + beforeEach(angular.mock.module('zeppelinWebApp')); beforeEach(angular.mock.inject((_$componentController_) => { - $componentController = _$componentController_ - })) + $componentController = _$componentController_; + })); it('should get progress when there is a finished paragraph', () => { const paragraphs = [ - { status: ParagraphStatus.FINISHED }, - ] - const mockNote = createMockNote(paragraphs) - const bindings = { note: mockNote, } + {status: ParagraphStatus.FINISHED}, + ]; + const mockNote = createMockNote(paragraphs); + const bindings = {note: mockNote}; - const ctrl = $componentController('job', null, bindings) - expect(ctrl).toBeDefined() + const ctrl = $componentController('job', null, bindings); + expect(ctrl).toBeDefined(); - const progress1 = ctrl.getProgress() - expect(progress1).toBe('100%') - }) + const progress1 = ctrl.getProgress(); + expect(progress1).toBe('100%'); + }); it('should get progress when there is pending and finished paragraphs', () => { const paragraphs = [ - { status: ParagraphStatus.PENDING }, - { status: ParagraphStatus.FINISHED}, - ] - const mockNote = createMockNote(paragraphs) - const bindings = { note: mockNote, } + {status: ParagraphStatus.PENDING}, + {status: ParagraphStatus.FINISHED}, + ]; + const mockNote = createMockNote(paragraphs); + const bindings = {note: mockNote}; - const ctrl = $componentController('job', null, bindings) + const ctrl = $componentController('job', null, bindings); - const progress1 = ctrl.getProgress() - expect(progress1).toBe('50%') - }) + const progress1 = ctrl.getProgress(); + expect(progress1).toBe('50%'); + }); it('should get proper job type icons', () => { - const paragraphs = [ { status: ParagraphStatus.PENDING }, ] - const mockNote = createMockNote(paragraphs) - const bindings = { note: mockNote, } + const paragraphs = [{status: ParagraphStatus.PENDING}]; + const mockNote = createMockNote(paragraphs); + const bindings = {note: mockNote}; - const ctrl = $componentController('job', null, bindings) + const ctrl = $componentController('job', null, bindings); - let icon = ctrl.getJobTypeIcon() - expect(icon).toBe('icon-doc') + let icon = ctrl.getJobTypeIcon(); + expect(icon).toBe('icon-doc'); - mockNote.noteType = 'cron' - icon = ctrl.getJobTypeIcon() - expect(icon).toBe('icon-clock') - }) + mockNote.noteType = 'cron'; + icon = ctrl.getJobTypeIcon(); + expect(icon).toBe('icon-clock'); + }); function createMockNote(paragraphs) { return { @@ -58,6 +58,6 @@ describe('JobComponent', () => { noteId: 'NT01', noteName: 'TestNote01', noteType: 'normal', - } + }; } -}) +}); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.component.js b/zeppelin-web/src/app/jobmanager/jobmanager.component.js index 364cc45a0cc..c883a11effc 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.component.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.component.js @@ -12,129 +12,132 @@ * limitations under the License. */ -import './job/job.component' -import { JobManagerFilter } from './jobmanager.filter' -import { JobManagerService} from './jobmanager.service' +import './job/job.component'; +import {JobManagerFilter} from './jobmanager.filter'; +import {JobManagerService} from './jobmanager.service'; -import { getJobIconByStatus, getJobColorByStatus } from './job-status' +import {getJobIconByStatus, getJobColorByStatus} from './job-status'; angular.module('zeppelinWebApp') .controller('JobManagerCtrl', JobManagerController) .filter('JobManager', JobManagerFilter) - .service('JobManagerService', JobManagerService) + .service('JobManagerService', JobManagerService); const JobDateSorter = { RECENTLY_UPDATED: 'Recently Update', OLDEST_UPDATED: 'Oldest Updated', -} +}; function JobManagerController($scope, ngToast, JobManagerFilter, JobManagerService) { - 'ngInject' + 'ngInject'; - $scope.isFilterLoaded = false - $scope.jobs = [] + $scope.isFilterLoaded = false; + $scope.jobs = []; $scope.sorter = { - availableDateSorter: Object.keys(JobDateSorter).map(key => { return JobDateSorter[key] }), + availableDateSorter: Object.keys(JobDateSorter).map((key) => { + return JobDateSorter[key]; + }), currentDateSorter: JobDateSorter.RECENTLY_UPDATED, - } - $scope.filteredJobs = $scope.jobs + }; + $scope.filteredJobs = $scope.jobs; $scope.filterConfig = { isRunningAlwaysTop: true, noteNameFilterValue: '', interpreterFilterValue: '*', isSortByAsc: true, - } + }; $scope.pagination = { currentPage: 1, itemsPerPage: 10, maxPageCount: 5, - } + }; - ngToast.dismiss() - init() + ngToast.dismiss(); + init(); /** functions */ $scope.setJobDateSorter = function(dateSorter) { - $scope.sorter.currentDateSorter = dateSorter - } + $scope.sorter.currentDateSorter = dateSorter; + }; $scope.getJobsInCurrentPage = function(jobs) { - const cp = $scope.pagination.currentPage - const itp = $scope.pagination.itemsPerPage - return jobs.slice((cp - 1) * itp, (cp * itp)) - } + const cp = $scope.pagination.currentPage; + const itp = $scope.pagination.itemsPerPage; + return jobs.slice((cp - 1) * itp, (cp * itp)); + }; - let asyncNotebookJobFilter = function (jobs, filterConfig) { + let asyncNotebookJobFilter = function(jobs, filterConfig) { return new Promise((resolve, reject) => { - $scope.filteredJobs = JobManagerFilter(jobs, filterConfig) - resolve($scope.filteredJobs) - }) - } + // eslint-disable-next-line new-cap + $scope.filteredJobs = JobManagerFilter(jobs, filterConfig); + resolve($scope.filteredJobs); + }); + }; $scope.$watch('sorter.currentDateSorter', function() { $scope.filterConfig.isSortByAsc = - $scope.sorter.currentDateSorter === JobDateSorter.OLDEST_UPDATED - asyncNotebookJobFilter($scope.jobs, $scope.filterConfig) - }) + $scope.sorter.currentDateSorter === JobDateSorter.OLDEST_UPDATED; + asyncNotebookJobFilter($scope.jobs, $scope.filterConfig); + }); - $scope.getJobIconByStatus = getJobIconByStatus - $scope.getJobColorByStatus = getJobColorByStatus + $scope.getJobIconByStatus = getJobIconByStatus; + $scope.getJobColorByStatus = getJobColorByStatus; - $scope.filterJobs = function (jobs, filterConfig) { + $scope.filterJobs = function(jobs, filterConfig) { asyncNotebookJobFilter(jobs, filterConfig) .then(() => { - $scope.isFilterLoaded = true - }) - .catch(error => { - console.error('Failed to search jobs from server', error) + $scope.isFilterLoaded = true; }) - } + .catch((error) => { + console.error('Failed to search jobs from server', error); + }); + }; - $scope.filterValueToName = function (filterValue, maxStringLength) { + $scope.filterValueToName = function(filterValue, maxStringLength) { if (typeof $scope.defaultInterpreters === 'undefined') { - return + return; } - let index = $scope.defaultInterpreters.findIndex(intp => intp.value === filterValue) + let index = $scope.defaultInterpreters.findIndex((intp) => intp.value === filterValue); if (typeof $scope.defaultInterpreters[index].name !== 'undefined') { if (typeof maxStringLength !== 'undefined' && maxStringLength > $scope.defaultInterpreters[index].name) { - return $scope.defaultInterpreters[index].name.substr(0, maxStringLength - 3) + '...' + return $scope.defaultInterpreters[index].name.substr(0, maxStringLength - 3) + '...'; } - return $scope.defaultInterpreters[index].name + return $scope.defaultInterpreters[index].name; } else { - return 'NONE' + return 'NONE'; } - } + }; - $scope.setFilterValue = function (filterValue) { - $scope.filterConfig.interpreterFilterValue = filterValue - $scope.filterJobs($scope.jobs, $scope.filterConfig) - } + $scope.setFilterValue = function(filterValue) { + $scope.filterConfig.interpreterFilterValue = filterValue; + $scope.filterJobs($scope.jobs, $scope.filterConfig); + }; $scope.setJobs = function(jobs) { - $scope.jobs = jobs + $scope.jobs = jobs; let interpreters = $scope.jobs - .filter(j => typeof j.interpreter !== 'undefined') - .map(j => j.interpreter) - interpreters = [...new Set(interpreters)] // remove duplicated interpreters + .filter((j) => typeof j.interpreter !== 'undefined') + .map((j) => j.interpreter); + interpreters = [...new Set(interpreters)]; // remove duplicated interpreters - $scope.defaultInterpreters = [ { name: 'ALL', value: '*' } ] + $scope.defaultInterpreters = [{name: 'ALL', value: '*'}]; for (let i = 0; i < interpreters.length; i++) { - $scope.defaultInterpreters.push({ name: interpreters[i], value: interpreters[i] }) + $scope.defaultInterpreters.push({name: interpreters[i], value: interpreters[i]}); } - } + }; function init() { - JobManagerService.getJobs() - JobManagerService.subscribeSetJobs($scope, setJobsCallback) - JobManagerService.subscribeUpdateJobs($scope, updateJobsCallback) + JobManagerService.getJobs(); + JobManagerService.subscribeSetJobs($scope, setJobsCallback); + JobManagerService.subscribeUpdateJobs($scope, updateJobsCallback); - $scope.$on('$destroy', function () { - JobManagerService.disconnect() - }) + $scope.$on('$destroy', function() { + JobManagerService.disconnect(); + }); } /* @@ -142,45 +145,45 @@ function JobManagerController($scope, ngToast, JobManagerFilter, JobManagerServi */ function setJobsCallback(event, response) { - const jobs = response.jobs - $scope.setJobs(jobs) - $scope.filterJobs($scope.jobs, $scope.filterConfig) + const jobs = response.jobs; + $scope.setJobs(jobs); + $scope.filterJobs($scope.jobs, $scope.filterConfig); } function updateJobsCallback(event, response) { - let jobs = $scope.jobs + let jobs = $scope.jobs; let jobByNoteId = jobs.reduce((acc, j) => { - const noteId = j.noteId - acc[noteId] = j - return acc - }, {}) + const noteId = j.noteId; + acc[noteId] = j; + return acc; + }, {}); - let updatedJobs = response.jobs - updatedJobs.map(updatedJob => { + let updatedJobs = response.jobs; + updatedJobs.map((updatedJob) => { if (typeof jobByNoteId[updatedJob.noteId] === 'undefined') { - let newItem = angular.copy(updatedJob) - jobs.push(newItem) - jobByNoteId[updatedJob.noteId] = newItem + let newItem = angular.copy(updatedJob); + jobs.push(newItem); + jobByNoteId[updatedJob.noteId] = newItem; } else { - let job = jobByNoteId[updatedJob.noteId] + let job = jobByNoteId[updatedJob.noteId]; if (updatedJob.isRemoved === true) { - delete jobByNoteId[updatedJob.noteId] - let removeIndex = jobs.findIndex(j => j.noteId === updatedJob.noteId) + delete jobByNoteId[updatedJob.noteId]; + let removeIndex = jobs.findIndex((j) => j.noteId === updatedJob.noteId); if (removeIndex) { - jobs.splice(removeIndex, 1) + jobs.splice(removeIndex, 1); } } else { // update the job - job.isRunningJob = updatedJob.isRunningJob - job.noteName = updatedJob.noteName - job.noteType = updatedJob.noteType - job.interpreter = updatedJob.interpreter - job.unixTimeLastRun = updatedJob.unixTimeLastRun - job.paragraphs = updatedJob.paragraphs + job.isRunningJob = updatedJob.isRunningJob; + job.noteName = updatedJob.noteName; + job.noteType = updatedJob.noteType; + job.interpreter = updatedJob.interpreter; + job.unixTimeLastRun = updatedJob.unixTimeLastRun; + job.paragraphs = updatedJob.paragraphs; } } - }) - $scope.filterJobs(jobs, $scope.filterConfig) + }); + $scope.filterJobs(jobs, $scope.filterConfig); } } diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.component.test.js b/zeppelin-web/src/app/jobmanager/jobmanager.component.test.js index a4b858b95e2..760414244ce 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.component.test.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.component.test.js @@ -1,26 +1,26 @@ describe('JobManagerComponent', () => { - let $scope - let $controller + let $scope; + let $controller; - beforeEach(angular.mock.module('zeppelinWebApp')) + beforeEach(angular.mock.module('zeppelinWebApp')); beforeEach(angular.mock.inject((_$rootScope_, _$controller_) => { - $scope = _$rootScope_.$new() - $controller = _$controller_ - })) + $scope = _$rootScope_.$new(); + $controller = _$controller_; + })); it('should set jobs using `setJobs`', () => { - let ctrl = $controller('JobManagerCtrl', { $scope: $scope, }) - expect(ctrl).toBeDefined() + let ctrl = $controller('JobManagerCtrl', {$scope: $scope}); + expect(ctrl).toBeDefined(); const mockJobs = [ - { noteId: 'TN01', interpreter: 'spark', }, - { noteId: 'TN02', interpreter: 'spark', }, - ] + {noteId: 'TN01', interpreter: 'spark'}, + {noteId: 'TN02', interpreter: 'spark'}, + ]; - $scope.setJobs(mockJobs) + $scope.setJobs(mockJobs); expect($scope.defaultInterpreters).toEqual([ - { name: 'ALL', value: '*', }, - { name: 'spark', value: 'spark', }, - ]) - }) -}) + {name: 'ALL', value: '*'}, + {name: 'spark', value: 'spark'}, + ]); + }); +}); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.filter.js b/zeppelin-web/src/app/jobmanager/jobmanager.filter.js index d6c8d69c744..c4abb1ca440 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.filter.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.filter.js @@ -13,44 +13,44 @@ */ export function JobManagerFilter() { - function filterContext (jobs, filterConfig) { - let interpreter = filterConfig.interpreterFilterValue - let noteName = filterConfig.noteNameFilterValue - let isSortByAsc = filterConfig.isSortByAsc - let filteredJobs = jobs + function filterContext(jobs, filterConfig) { + let interpreter = filterConfig.interpreterFilterValue; + let noteName = filterConfig.noteNameFilterValue; + let isSortByAsc = filterConfig.isSortByAsc; + let filteredJobs = jobs; if (typeof interpreter === 'undefined') { filteredJobs = filteredJobs.filter((jobItem) => { - return typeof jobItem.interpreter === 'undefined' - }) + return typeof jobItem.interpreter === 'undefined'; + }); } else if (interpreter !== '*') { - filteredJobs = filteredJobs.filter(j => j.interpreter === interpreter) + filteredJobs = filteredJobs.filter((j) => j.interpreter === interpreter); } // filter by note name if (noteName !== '') { filteredJobs = filteredJobs.filter((jobItem) => { - let lowerFilterValue = noteName.toLocaleLowerCase() - let lowerNotebookName = jobItem.noteName.toLocaleLowerCase() - return lowerNotebookName.match(new RegExp('.*' + lowerFilterValue + '.*')) - }) + let lowerFilterValue = noteName.toLocaleLowerCase(); + let lowerNotebookName = jobItem.noteName.toLocaleLowerCase(); + return lowerNotebookName.match(new RegExp('.*' + lowerFilterValue + '.*')); + }); } // sort by name filteredJobs = filteredJobs.sort((jobItem) => { - return jobItem.noteName.toLowerCase() - }) + return jobItem.noteName.toLowerCase(); + }); // sort by timestamp filteredJobs = filteredJobs.sort((x, y) => { if (isSortByAsc) { - return x.unixTimeLastRun - y.unixTimeLastRun + return x.unixTimeLastRun - y.unixTimeLastRun; } else { - return y.unixTimeLastRun - x.unixTimeLastRun + return y.unixTimeLastRun - x.unixTimeLastRun; } - }) + }); - return filteredJobs + return filteredJobs; } - return filterContext + return filterContext; } diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.service.js b/zeppelin-web/src/app/jobmanager/jobmanager.service.js index 603950fee0e..472ac6d2f39 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.service.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.service.js @@ -14,51 +14,51 @@ export class JobManagerService { constructor($http, $rootScope, baseUrlSrv, websocketMsgSrv) { - 'ngInject' + 'ngInject'; - this.$http = $http - this.$rootScope = $rootScope - this.BaseUrlService = baseUrlSrv - this.WebsocketMessageService = websocketMsgSrv + this.$http = $http; + this.$rootScope = $rootScope; + this.BaseUrlService = baseUrlSrv; + this.WebsocketMessageService = websocketMsgSrv; } sendStopJobRequest(noteId) { - const apiURL = this.BaseUrlService.getRestApiBase() + `/notebook/job/${noteId}` - return this.$http({ method: 'DELETE', url: apiURL, }) + const apiURL = this.BaseUrlService.getRestApiBase() + `/notebook/job/${noteId}`; + return this.$http({method: 'DELETE', url: apiURL}); } sendRunJobRequest(noteId) { - const apiURL = this.BaseUrlService.getRestApiBase() + `/notebook/job/${noteId}` - return this.$http({ method: 'POST', url: apiURL, }) + const apiURL = this.BaseUrlService.getRestApiBase() + `/notebook/job/${noteId}`; + return this.$http({method: 'POST', url: apiURL}); } getJobs() { - this.WebsocketMessageService.getJobs() + this.WebsocketMessageService.getJobs(); } disconnect() { - this.WebsocketMessageService.disconnectJobEvent() + this.WebsocketMessageService.disconnectJobEvent(); } subscribeSetJobs(controllerScope, receiveCallback) { - const event = 'jobmanager:set-jobs' - console.log(`(Event) Subscribed: ${event}`) - const unsubscribeHandler = this.$rootScope.$on(event, receiveCallback) + const event = 'jobmanager:set-jobs'; + console.log(`(Event) Subscribed: ${event}`); + const unsubscribeHandler = this.$rootScope.$on(event, receiveCallback); controllerScope.$on('$destroy', () => { - console.log(`(Event) Unsubscribed: ${event}`) - unsubscribeHandler() - }) + console.log(`(Event) Unsubscribed: ${event}`); + unsubscribeHandler(); + }); } subscribeUpdateJobs(controllerScope, receiveCallback) { - const event = 'jobmanager:update-jobs' - console.log(`(Event) Subscribed: ${event}`) - const unsubscribeHandler = this.$rootScope.$on(event, receiveCallback) + const event = 'jobmanager:update-jobs'; + console.log(`(Event) Subscribed: ${event}`); + const unsubscribeHandler = this.$rootScope.$on(event, receiveCallback); controllerScope.$on('$destroy', () => { - console.log(`(Event) Unsubscribed: ${event}`) - unsubscribeHandler() - }) + console.log(`(Event) Unsubscribed: ${event}`); + unsubscribeHandler(); + }); } } diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.service.test.js b/zeppelin-web/src/app/jobmanager/jobmanager.service.test.js index fbb082929c7..be7196d8b7b 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.service.test.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.service.test.js @@ -1,56 +1,56 @@ -import { ParagraphStatus } from '../notebook/paragraph/paragraph.status' -import { JobManagerService } from './jobmanager.service' +import {ParagraphStatus} from '../notebook/paragraph/paragraph.status'; +import {JobManagerService} from './jobmanager.service'; describe('JobManagerService', () => { - const baseUrlSrvMock = { getRestApiBase: () => '' } - let service - let $httpBackend + const baseUrlSrvMock = {getRestApiBase: () => ''}; + let service; + let $httpBackend; - beforeEach(angular.mock.module('zeppelinWebApp')) + beforeEach(angular.mock.module('zeppelinWebApp')); beforeEach(angular.mock.inject((_$rootScope_, _$httpBackend_, _$http_, _websocketMsgSrv_) => { - $httpBackend = _$httpBackend_ - service = new JobManagerService(_$http_, _$rootScope_, baseUrlSrvMock, _websocketMsgSrv_) - })) + $httpBackend = _$httpBackend_; + service = new JobManagerService(_$http_, _$rootScope_, baseUrlSrvMock, _websocketMsgSrv_); + })); it('should sent valid request to run a job', () => { - const paragraphs = [ { status: ParagraphStatus.PENDING }, ] - const mockNote = createMockNote(paragraphs) + const paragraphs = [{status: ParagraphStatus.PENDING}]; + const mockNote = createMockNote(paragraphs); - const noteId = mockNote.noteId - service.sendRunJobRequest(noteId) + const noteId = mockNote.noteId; + service.sendRunJobRequest(noteId); - const url = `/notebook/job/${noteId}` + const url = `/notebook/job/${noteId}`; $httpBackend .when('POST', url) - .respond(200, { /** return nothing */ }) - $httpBackend.expectPOST(url) - $httpBackend.flush() + .respond(200, { /** return nothing */ }); + $httpBackend.expectPOST(url); + $httpBackend.flush(); - checkUnknownHttpRequests() - }) + checkUnknownHttpRequests(); + }); it('should sent valid request to stop a job', () => { - const paragraphs = [ { status: ParagraphStatus.PENDING }, ] - const mockNote = createMockNote(paragraphs) + const paragraphs = [{status: ParagraphStatus.PENDING}]; + const mockNote = createMockNote(paragraphs); - const noteId = mockNote.noteId - service.sendStopJobRequest(noteId) + const noteId = mockNote.noteId; + service.sendStopJobRequest(noteId); - const url = `/notebook/job/${noteId}` + const url = `/notebook/job/${noteId}`; $httpBackend .when('DELETE', url) - .respond(200, { /** return nothing */ }) - $httpBackend.expectDELETE(url) - $httpBackend.flush() + .respond(200, { /** return nothing */ }); + $httpBackend.expectDELETE(url); + $httpBackend.flush(); - checkUnknownHttpRequests() - }) + checkUnknownHttpRequests(); + }); function checkUnknownHttpRequests() { - $httpBackend.verifyNoOutstandingExpectation() - $httpBackend.verifyNoOutstandingRequest() + $httpBackend.verifyNoOutstandingExpectation(); + $httpBackend.verifyNoOutstandingRequest(); } function createMockNote(paragraphs) { @@ -60,6 +60,6 @@ describe('JobManagerService', () => { noteId: 'NT01', noteName: 'TestNote01', noteType: 'normal', - } + }; } -}) +}); diff --git a/zeppelin-web/src/app/notebook-repository/notebook-repository.controller.js b/zeppelin-web/src/app/notebook-repository/notebook-repository.controller.js index 0f62bc0c2a3..d6d13b32c79 100644 --- a/zeppelin-web/src/app/notebook-repository/notebook-repository.controller.js +++ b/zeppelin-web/src/app/notebook-repository/notebook-repository.controller.js @@ -12,76 +12,76 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').controller('NotebookRepositoryCtrl', NotebookRepositoryCtrl) +angular.module('zeppelinWebApp').controller('NotebookRepositoryCtrl', NotebookRepositoryCtrl); function NotebookRepositoryCtrl($http, baseUrlSrv, ngToast) { - 'ngInject' + 'ngInject'; - let vm = this - vm.notebookRepos = [] - vm.showDropdownSelected = showDropdownSelected - vm.saveNotebookRepo = saveNotebookRepo + let vm = this; + vm.notebookRepos = []; + vm.showDropdownSelected = showDropdownSelected; + vm.saveNotebookRepo = saveNotebookRepo; - _init() + _init(); // Public functions - function saveNotebookRepo (valueform, repo, data) { - console.log('data %o', data) + function saveNotebookRepo(valueform, repo, data) { + console.log('data %o', data); $http.put(baseUrlSrv.getRestApiBase() + '/notebook-repositories', { 'name': repo.className, - 'settings': data - }).success(function (data) { - let index = _.findIndex(vm.notebookRepos, {'className': repo.className}) + 'settings': data, + }).success(function(data) { + let index = _.findIndex(vm.notebookRepos, {'className': repo.className}); if (index >= 0) { - vm.notebookRepos[index] = data.body - console.log('repos %o, data %o', vm.notebookRepos, data.body) + vm.notebookRepos[index] = data.body; + console.log('repos %o, data %o', vm.notebookRepos, data.body); } - valueform.$show() - }).error(function () { + valueform.$show(); + }).error(function() { ngToast.danger({ content: 'We couldn\'t save that NotebookRepo\'s settings', verticalPosition: 'bottom', - timeout: '3000' - }) - valueform.$show() - }) + timeout: '3000', + }); + valueform.$show(); + }); - return 'manual' + return 'manual'; } - function showDropdownSelected (setting) { - let index = _.findIndex(setting.value, {'value': setting.selected}) + function showDropdownSelected(setting) { + let index = _.findIndex(setting.value, {'value': setting.selected}); if (index < 0) { - return 'No value' + return 'No value'; } else { - return setting.value[index].name + return setting.value[index].name; } } // Private functions - function _getInterpreterSettings () { + function _getInterpreterSettings() { $http.get(baseUrlSrv.getRestApiBase() + '/notebook-repositories') - .success(function (data, status, headers, config) { - vm.notebookRepos = data.body - console.log('ya notebookRepos %o', vm.notebookRepos) - }).error(function (data, status, headers, config) { + .success(function(data, status, headers, config) { + vm.notebookRepos = data.body; + console.log('ya notebookRepos %o', vm.notebookRepos); + }).error(function(data, status, headers, config) { if (status === 401) { ngToast.danger({ content: 'You don\'t have permission on this page', verticalPosition: 'bottom', - timeout: '3000' - }) - setTimeout(function () { - window.location = baseUrlSrv.getBase() - }, 3000) + timeout: '3000', + }); + setTimeout(function() { + window.location = baseUrlSrv.getBase(); + }, 3000); } - console.log('Error %o %o', status, data.message) - }) + console.log('Error %o %o', status, data.message); + }); } - function _init () { - _getInterpreterSettings() + function _init() { + _getInterpreterSettings(); } } diff --git a/zeppelin-web/src/app/notebook/dropdown-input/dropdown-input.directive.js b/zeppelin-web/src/app/notebook/dropdown-input/dropdown-input.directive.js index a64204af063..0fe43f7eaa3 100644 --- a/zeppelin-web/src/app/notebook/dropdown-input/dropdown-input.directive.js +++ b/zeppelin-web/src/app/notebook/dropdown-input/dropdown-input.directive.js @@ -12,15 +12,15 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').directive('dropdownInput', dropdownInputDirective) +angular.module('zeppelinWebApp').directive('dropdownInput', dropdownInputDirective); function dropdownInputDirective() { return { restrict: 'A', - link: function (scope, element) { - element.bind('click', function (event) { - event.stopPropagation() - }) - } - } + link: function(scope, element) { + element.bind('click', function(event) { + event.stopPropagation(); + }); + }, + }; } diff --git a/zeppelin-web/src/app/notebook/dynamic-forms/dynamic-forms.directive.js b/zeppelin-web/src/app/notebook/dynamic-forms/dynamic-forms.directive.js index 40a70eb94e0..1b1043e8b75 100644 --- a/zeppelin-web/src/app/notebook/dynamic-forms/dynamic-forms.directive.js +++ b/zeppelin-web/src/app/notebook/dynamic-forms/dynamic-forms.directive.js @@ -12,9 +12,9 @@ * limitations under the License. */ -import './dynamic-forms.css' +import './dynamic-forms.css'; -angular.module('zeppelinWebApp').directive('dynamicForms', DynamicFormDirective) +angular.module('zeppelinWebApp').directive('dynamicForms', DynamicFormDirective); function DynamicFormDirective($templateRequest, $compile) { return { @@ -27,36 +27,36 @@ function DynamicFormDirective($templateRequest, $compile) { forms: '=forms', params: '=params', action: '=action', - removeaction: '=removeaction' + removeaction: '=removeaction', }, - link: function (scope, element, attrs, controller) { - scope.loadForm = this.loadForm - scope.toggleCheckbox = this.toggleCheckbox - $templateRequest('app/notebook/dynamic-forms/dynamic-forms.directive.html').then(function (formsHtml) { - let forms = angular.element(formsHtml) - element.append(forms) - $compile(forms)(scope) - }) + link: function(scope, element, attrs, controller) { + scope.loadForm = this.loadForm; + scope.toggleCheckbox = this.toggleCheckbox; + $templateRequest('app/notebook/dynamic-forms/dynamic-forms.directive.html').then(function(formsHtml) { + let forms = angular.element(formsHtml); + element.append(forms); + $compile(forms)(scope); + }); }, - loadForm: function (formulaire, params) { - let value = formulaire.defaultValue + loadForm: function(formulaire, params) { + let value = formulaire.defaultValue; if (params[formulaire.name]) { - value = params[formulaire.name] + value = params[formulaire.name]; } - params[formulaire.name] = value + params[formulaire.name] = value; }, - toggleCheckbox: function (formulaire, option, params) { - let idx = params[formulaire.name].indexOf(option.value) + toggleCheckbox: function(formulaire, option, params) { + let idx = params[formulaire.name].indexOf(option.value); if (idx > -1) { - params[formulaire.name].splice(idx, 1) + params[formulaire.name].splice(idx, 1); } else { - params[formulaire.name].push(option.value) + params[formulaire.name].push(option.value); } - } + }, - } + }; } diff --git a/zeppelin-web/src/app/notebook/elastic-input/elastic-input.controller.js b/zeppelin-web/src/app/notebook/elastic-input/elastic-input.controller.js index 507b2f61493..c11f95ba124 100644 --- a/zeppelin-web/src/app/notebook/elastic-input/elastic-input.controller.js +++ b/zeppelin-web/src/app/notebook/elastic-input/elastic-input.controller.js @@ -12,10 +12,10 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').controller('ElasticInputCtrl', ElasticInputCtrl) +angular.module('zeppelinWebApp').controller('ElasticInputCtrl', ElasticInputCtrl); -function ElasticInputCtrl () { - let vm = this - vm.showEditor = false - vm.value = '' +function ElasticInputCtrl() { + let vm = this; + vm.showEditor = false; + vm.value = ''; } diff --git a/zeppelin-web/src/app/notebook/note-var-share.service.js b/zeppelin-web/src/app/notebook/note-var-share.service.js index e79f389cc3e..a5975ce49a5 100644 --- a/zeppelin-web/src/app/notebook/note-var-share.service.js +++ b/zeppelin-web/src/app/notebook/note-var-share.service.js @@ -12,28 +12,28 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').service('noteVarShareService', NoteVarShareService) +angular.module('zeppelinWebApp').service('noteVarShareService', NoteVarShareService); -function NoteVarShareService () { - 'ngInject' +function NoteVarShareService() { + 'ngInject'; - let store = {} + let store = {}; - this.clear = function () { - store = {} - } + this.clear = function() { + store = {}; + }; - this.put = function (key, value) { - store[key] = value - } + this.put = function(key, value) { + store[key] = value; + }; - this.get = function (key) { - return store[key] - } + this.get = function(key) { + return store[key]; + }; - this.del = function (key) { - let v = store[key] - delete store[key] - return v - } + this.del = function(key) { + let v = store[key]; + delete store[key]; + return v; + }; } diff --git a/zeppelin-web/src/app/notebook/notebook.controller.js b/zeppelin-web/src/app/notebook/notebook.controller.js index b02b74ee8d5..05ab9fb7992 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.js @@ -12,30 +12,30 @@ * limitations under the License. */ -import moment from 'moment' +import moment from 'moment'; -import { isParagraphRunning, } from './paragraph/paragraph.status' +import {isParagraphRunning} from './paragraph/paragraph.status'; -angular.module('zeppelinWebApp').controller('NotebookCtrl', NotebookCtrl) +angular.module('zeppelinWebApp').controller('NotebookCtrl', NotebookCtrl); -function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, +function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope, $http, websocketMsgSrv, baseUrlSrv, $timeout, saveAsService, ngToast, noteActionService, noteVarShareService, TRASH_FOLDER_ID, heliumService) { - 'ngInject' - - ngToast.dismiss() - - $scope.note = null - $scope.actionOnFormSelectionChange = true - $scope.hideForms = false - $scope.disableForms = false - $scope.editorToggled = false - $scope.tableToggled = false - $scope.viewOnly = false - $scope.showSetting = false - $scope.showRevisionsComparator = false - $scope.looknfeelOption = ['default', 'simple', 'report'] + 'ngInject'; + + ngToast.dismiss(); + + $scope.note = null; + $scope.actionOnFormSelectionChange = true; + $scope.hideForms = false; + $scope.disableForms = false; + $scope.editorToggled = false; + $scope.tableToggled = false; + $scope.viewOnly = false; + $scope.showSetting = false; + $scope.showRevisionsComparator = false; + $scope.looknfeelOption = ['default', 'simple', 'report']; $scope.cronOption = [ {name: 'None', value: undefined}, {name: '1m', value: '0 0/1 * * * ?'}, @@ -44,28 +44,28 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, {name: '3h', value: '0 0 0/3 * * ?'}, {name: '6h', value: '0 0 0/6 * * ?'}, {name: '12h', value: '0 0 0/12 * * ?'}, - {name: '1d', value: '0 0 0 * * ?'} - ] + {name: '1d', value: '0 0 0 * * ?'}, + ]; $scope.formatRevisionDate = function(date) { - return moment.unix(date).format('MMMM Do YYYY, h:mm a') - } + return moment.unix(date).format('MMMM Do YYYY, h:mm a'); + }; - $scope.interpreterSettings = [] - $scope.interpreterBindings = [] - $scope.isNoteDirty = null - $scope.saveTimer = null - $scope.paragraphWarningDialog = {} + $scope.interpreterSettings = []; + $scope.interpreterBindings = []; + $scope.isNoteDirty = null; + $scope.saveTimer = null; + $scope.paragraphWarningDialog = {}; - let connectedOnce = false - let isRevisionPath = function (path) { - let pattern = new RegExp('^.*\/notebook\/[a-zA-Z0-9_]*\/revision\/[a-zA-Z0-9_]*') - return pattern.test(path) - } + let connectedOnce = false; + let isRevisionPath = function(path) { + let pattern = new RegExp('^.*\/notebook\/[a-zA-Z0-9_]*\/revision\/[a-zA-Z0-9_]*'); + return pattern.test(path); + }; - $scope.noteRevisions = [] - $scope.currentRevision = 'Head' - $scope.revisionView = isRevisionPath($location.path()) + $scope.noteRevisions = []; + $scope.currentRevision = 'Head'; + $scope.revisionView = isRevisionPath($location.path()); $scope.search = { searchText: '', @@ -78,48 +78,48 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, currentOccurrence: 0, searchBoxOpened: false, searchBoxWidth: 350, - left: '0px' - } - let currentSearchParagraph = 0 + left: '0px', + }; + let currentSearchParagraph = 0; - $scope.$watch('note', function (value) { - let title + $scope.$watch('note', function(value) { + let title; if (value) { - title = value.name.substr(value.name.lastIndexOf('/') + 1, value.name.length) - title += ' - Zeppelin' + title = value.name.substr(value.name.lastIndexOf('/') + 1, value.name.length); + title += ' - Zeppelin'; } else { - title = 'Zeppelin' + title = 'Zeppelin'; } - $rootScope.pageTitle = title - }, true) + $rootScope.pageTitle = title; + }, true); - $scope.$on('setConnectedStatus', function (event, param) { + $scope.$on('setConnectedStatus', function(event, param) { if (connectedOnce && param) { - initNotebook() + initNotebook(); } - connectedOnce = true - }) + connectedOnce = true; + }); - $scope.getCronOptionNameFromValue = function (value) { + $scope.getCronOptionNameFromValue = function(value) { if (!value) { - return '' + return ''; } for (let o in $scope.cronOption) { if ($scope.cronOption[o].value === value) { - return $scope.cronOption[o].name + return $scope.cronOption[o].name; } } - return value - } + return value; + }; - $scope.blockAnonUsers = function () { - let zeppelinVersion = $rootScope.zeppelinVersion - let url = 'https://zeppelin.apache.org/docs/' + zeppelinVersion + '/security/notebook_authorization.html' + $scope.blockAnonUsers = function() { + let zeppelinVersion = $rootScope.zeppelinVersion; + let url = 'https://zeppelin.apache.org/docs/' + zeppelinVersion + '/security/notebook_authorization.html'; let content = 'Only authenticated user can set the permission.' + '' + '' + - '' + ''; BootstrapDialog.show({ closable: false, closeByBackdrop: false, @@ -128,866 +128,878 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, message: content, buttons: [{ label: 'Close', - action: function (dialog) { - dialog.close() - } - }] - }) - } + action: function(dialog) { + dialog.close(); + }, + }], + }); + }; /** Init the new controller */ - const initNotebook = function () { - noteVarShareService.clear() + const initNotebook = function() { + noteVarShareService.clear(); if ($routeParams.revisionId) { - websocketMsgSrv.getNoteByRevision($routeParams.noteId, $routeParams.revisionId) + websocketMsgSrv.getNoteByRevision($routeParams.noteId, $routeParams.revisionId); } else { - websocketMsgSrv.getNote($routeParams.noteId) + websocketMsgSrv.getNote($routeParams.noteId); } - websocketMsgSrv.listRevisionHistory($routeParams.noteId) - let currentRoute = $route.current + websocketMsgSrv.listRevisionHistory($routeParams.noteId); + let currentRoute = $route.current; if (currentRoute) { setTimeout( - function () { - let routeParams = currentRoute.params - let $id = angular.element('#' + routeParams.paragraph + '_container') + function() { + let routeParams = currentRoute.params; + let $id = angular.element('#' + routeParams.paragraph + '_container'); if ($id.length > 0) { // adjust for navbar - let top = $id.offset().top - 103 - angular.element('html, body').scrollTo({top: top, left: 0}) + let top = $id.offset().top - 103; + angular.element('html, body').scrollTo({top: top, left: 0}); } }, 1000 - ) + ); } - } + }; - initNotebook() + initNotebook(); - $scope.focusParagraphOnClick = function (clickEvent) { + $scope.focusParagraphOnClick = function(clickEvent) { if (!$scope.note) { - return + return; } for (let i = 0; i < $scope.note.paragraphs.length; i++) { - let paragraphId = $scope.note.paragraphs[i].id + let paragraphId = $scope.note.paragraphs[i].id; if (jQuery.contains(angular.element('#' + paragraphId + '_container')[0], clickEvent.target)) { - $scope.$broadcast('focusParagraph', paragraphId, 0, null, true) - break + $scope.$broadcast('focusParagraph', paragraphId, 0, null, true); + break; } } - } + }; // register mouseevent handler for focus paragraph - document.addEventListener('click', $scope.focusParagraphOnClick) + document.addEventListener('click', $scope.focusParagraphOnClick); - let keyboardShortcut = function (keyEvent) { + let keyboardShortcut = function(keyEvent) { // handle keyevent if (!$scope.viewOnly && !$scope.revisionView) { - $scope.$broadcast('keyEvent', keyEvent) + $scope.$broadcast('keyEvent', keyEvent); } - } + }; - $scope.keydownEvent = function (keyEvent) { + $scope.keydownEvent = function(keyEvent) { if ((keyEvent.ctrlKey || keyEvent.metaKey) && String.fromCharCode(keyEvent.which).toLowerCase() === 's') { - keyEvent.preventDefault() + keyEvent.preventDefault(); } - keyboardShortcut(keyEvent) - } + keyboardShortcut(keyEvent); + }; // register mouseevent handler for focus paragraph - document.addEventListener('keydown', $scope.keydownEvent) + document.addEventListener('keydown', $scope.keydownEvent); - $scope.paragraphOnDoubleClick = function (paragraphId) { - $scope.$broadcast('doubleClickParagraph', paragraphId) - } + $scope.paragraphOnDoubleClick = function(paragraphId) { + $scope.$broadcast('doubleClickParagraph', paragraphId); + }; // Move the note to trash and go back to the main page - $scope.moveNoteToTrash = function (noteId) { - noteActionService.moveNoteToTrash(noteId, true) - } + $scope.moveNoteToTrash = function(noteId) { + noteActionService.moveNoteToTrash(noteId, true); + }; // Remove the note permanently if it's in the trash - $scope.removeNote = function (noteId) { - noteActionService.removeNote(noteId, true) - } + $scope.removeNote = function(noteId) { + noteActionService.removeNote(noteId, true); + }; - $scope.isTrash = function (note) { - return note ? note.name.split('/')[0] === TRASH_FOLDER_ID : false - } + $scope.isTrash = function(note) { + return note ? note.name.split('/')[0] === TRASH_FOLDER_ID : false; + }; // Export notebook - $scope.exportNote = function () { - let jsonContent = JSON.stringify($scope.note) - saveAsService.saveAs(jsonContent, $scope.note.name, 'json') - } + $scope.exportNote = function() { + let jsonContent = JSON.stringify($scope.note); + saveAsService.saveAs(jsonContent, $scope.note.name, 'json'); + }; // Clone note - $scope.cloneNote = function (noteId) { + $scope.cloneNote = function(noteId) { BootstrapDialog.confirm({ closable: true, title: '', message: 'Do you want to clone this note?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.cloneNote(noteId) - $location.path('/') + websocketMsgSrv.cloneNote(noteId); + $location.path('/'); } - } - }) - } + }, + }); + }; // checkpoint/commit notebook - $scope.checkpointNote = function (commitMessage) { + $scope.checkpointNote = function(commitMessage) { BootstrapDialog.confirm({ closable: true, title: '', message: 'Commit note to current repository?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.checkpointNote($routeParams.noteId, commitMessage) + websocketMsgSrv.checkpointNote($routeParams.noteId, commitMessage); } - } - }) - document.getElementById('note.checkpoint.message').value = '' - } + }, + }); + document.getElementById('note.checkpoint.message').value = ''; + }; // set notebook head to given revision - $scope.setNoteRevision = function () { + $scope.setNoteRevision = function() { BootstrapDialog.confirm({ closable: true, title: '', message: 'Set notebook head to current revision?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.setNoteRevision($routeParams.noteId, $routeParams.revisionId) + websocketMsgSrv.setNoteRevision($routeParams.noteId, $routeParams.revisionId); } - } - }) - } + }, + }); + }; $scope.preVisibleRevisionsComparator = function() { - $scope.mergeNoteRevisionsForCompare = null - $scope.firstNoteRevisionForCompare = null - $scope.secondNoteRevisionForCompare = null - $scope.currentFirstRevisionForCompare = 'Choose...' - $scope.currentSecondRevisionForCompare = 'Choose...' - $scope.$apply() - } - - $scope.$on('listRevisionHistory', function (event, data) { - console.debug('received list of revisions %o', data) - $scope.noteRevisions = data.revisionList + $scope.mergeNoteRevisionsForCompare = null; + $scope.firstNoteRevisionForCompare = null; + $scope.secondNoteRevisionForCompare = null; + $scope.currentFirstRevisionForCompare = 'Choose...'; + $scope.currentSecondRevisionForCompare = 'Choose...'; + $scope.$apply(); + }; + + $scope.$on('listRevisionHistory', function(event, data) { + console.debug('received list of revisions %o', data); + $scope.noteRevisions = data.revisionList; if ($scope.noteRevisions.length === 0 || $scope.noteRevisions[0].id !== 'Head') { $scope.noteRevisions.splice(0, 0, { id: 'Head', - message: 'Head' - }) + message: 'Head', + }); } if ($routeParams.revisionId) { - let index = _.findIndex($scope.noteRevisions, {'id': $routeParams.revisionId}) + let index = _.findIndex($scope.noteRevisions, {'id': $routeParams.revisionId}); if (index > -1) { - $scope.currentRevision = $scope.noteRevisions[index].message + $scope.currentRevision = $scope.noteRevisions[index].message; } } - }) + }); - $scope.$on('noteRevision', function (event, data) { - console.log('received note revision %o', data) + $scope.$on('noteRevision', function(event, data) { + console.log('received note revision %o', data); if (data.note) { - $scope.note = data.note - initializeLookAndFeel() + $scope.note = data.note; + initializeLookAndFeel(); } else { - $location.path('/') + $location.path('/'); } - }) + }); - $scope.$on('setNoteRevisionResult', function (event, data) { - console.log('received set note revision result %o', data) + $scope.$on('setNoteRevisionResult', function(event, data) { + console.log('received set note revision result %o', data); if (data.status) { - $location.path('/notebook/' + $routeParams.noteId) + $location.path('/notebook/' + $routeParams.noteId); } - }) + }); - $scope.visitRevision = function (revision) { + $scope.visitRevision = function(revision) { if (revision.id) { if (revision.id === 'Head') { - $location.path('/notebook/' + $routeParams.noteId) + $location.path('/notebook/' + $routeParams.noteId); } else { - $location.path('/notebook/' + $routeParams.noteId + '/revision/' + revision.id) + $location.path('/notebook/' + $routeParams.noteId + '/revision/' + revision.id); } } else { ngToast.danger({content: 'There is a problem with this Revision', verticalPosition: 'top', - dismissOnTimeout: false - }) + dismissOnTimeout: false, + }); } - } + }; - $scope.runAllParagraphs = function (noteId) { + $scope.runAllParagraphs = function(noteId) { BootstrapDialog.confirm({ closable: true, title: '', message: 'Run all paragraphs?', - callback: function (result) { + callback: function(result) { if (result) { - const paragraphs = $scope.note.paragraphs.map(p => { + const paragraphs = $scope.note.paragraphs.map((p) => { return { id: p.id, title: p.title, paragraph: p.text, config: p.config, - params: p.settings.params - } - }) - websocketMsgSrv.runAllParagraphs(noteId, paragraphs) + params: p.settings.params, + }; + }); + websocketMsgSrv.runAllParagraphs(noteId, paragraphs); } - } - }) - } + }, + }); + }; - $scope.saveNote = function () { + $scope.saveNote = function() { if ($scope.note && $scope.note.paragraphs) { - _.forEach($scope.note.paragraphs, function (par) { + _.forEach($scope.note.paragraphs, function(par) { angular .element('#' + par.id + '_paragraphColumn_main') .scope() - .saveParagraph(par) - }) - $scope.isNoteDirty = null + .saveParagraph(par); + }); + $scope.isNoteDirty = null; } - } + }; - $scope.clearAllParagraphOutput = function (noteId) { - noteActionService.clearAllParagraphOutput(noteId) - } + $scope.clearAllParagraphOutput = function(noteId) { + noteActionService.clearAllParagraphOutput(noteId); + }; - $scope.toggleAllEditor = function () { + $scope.toggleAllEditor = function() { if ($scope.editorToggled) { - $scope.$broadcast('openEditor') + $scope.$broadcast('openEditor'); } else { - $scope.$broadcast('closeEditor') + $scope.$broadcast('closeEditor'); } - $scope.editorToggled = !$scope.editorToggled - } + $scope.editorToggled = !$scope.editorToggled; + }; - $scope.showAllEditor = function () { - $scope.$broadcast('openEditor') - } + $scope.showAllEditor = function() { + $scope.$broadcast('openEditor'); + }; - $scope.hideAllEditor = function () { - $scope.$broadcast('closeEditor') - } + $scope.hideAllEditor = function() { + $scope.$broadcast('closeEditor'); + }; - $scope.toggleAllTable = function () { + $scope.toggleAllTable = function() { if ($scope.tableToggled) { - $scope.$broadcast('openTable') + $scope.$broadcast('openTable'); } else { - $scope.$broadcast('closeTable') + $scope.$broadcast('closeTable'); } - $scope.tableToggled = !$scope.tableToggled - } + $scope.tableToggled = !$scope.tableToggled; + }; - $scope.showAllTable = function () { - $scope.$broadcast('openTable') - } + $scope.showAllTable = function() { + $scope.$broadcast('openTable'); + }; - $scope.hideAllTable = function () { - $scope.$broadcast('closeTable') - } + $scope.hideAllTable = function() { + $scope.$broadcast('closeTable'); + }; /** * @returns {boolean} true if one more paragraphs are running. otherwise return false. */ - $scope.isNoteRunning = function () { - if (!$scope.note) { return false } + $scope.isNoteRunning = function() { + if (!$scope.note) { + return false; + } for (let i = 0; i < $scope.note.paragraphs.length; i++) { if (isParagraphRunning($scope.note.paragraphs[i])) { - return true + return true; } } - return false - } + return false; + }; - $scope.killSaveTimer = function () { + $scope.killSaveTimer = function() { if ($scope.saveTimer) { - $timeout.cancel($scope.saveTimer) - $scope.saveTimer = null + $timeout.cancel($scope.saveTimer); + $scope.saveTimer = null; } - } + }; - $scope.startSaveTimer = function () { - $scope.killSaveTimer() - $scope.isNoteDirty = true + $scope.startSaveTimer = function() { + $scope.killSaveTimer(); + $scope.isNoteDirty = true; // console.log('startSaveTimer called ' + $scope.note.id); - $scope.saveTimer = $timeout(function () { - $scope.saveNote() - }, 10000) - } + $scope.saveTimer = $timeout(function() { + $scope.saveNote(); + }, 10000); + }; - $scope.setLookAndFeel = function (looknfeel) { - $scope.note.config.looknfeel = looknfeel + $scope.setLookAndFeel = function(looknfeel) { + $scope.note.config.looknfeel = looknfeel; if ($scope.revisionView === true) { - $rootScope.$broadcast('setLookAndFeel', $scope.note.config.looknfeel) + $rootScope.$broadcast('setLookAndFeel', $scope.note.config.looknfeel); } else { - $scope.setConfig() + $scope.setConfig(); } - } + }; /** Set cron expression for this note **/ - $scope.setCronScheduler = function (cronExpr) { + $scope.setCronScheduler = function(cronExpr) { if (cronExpr) { if (!$scope.note.config.cronExecutingUser) { - $scope.note.config.cronExecutingUser = $rootScope.ticket.principal + $scope.note.config.cronExecutingUser = $rootScope.ticket.principal; } } else { - $scope.note.config.cronExecutingUser = '' + $scope.note.config.cronExecutingUser = ''; } - $scope.note.config.cron = cronExpr - $scope.setConfig() - } + $scope.note.config.cron = cronExpr; + $scope.setConfig(); + }; /** Set the username of the user to be used to execute all notes in notebook **/ - $scope.setCronExecutingUser = function (cronExecutingUser) { - $scope.note.config.cronExecutingUser = cronExecutingUser - $scope.setConfig() - } + $scope.setCronExecutingUser = function(cronExecutingUser) { + $scope.note.config.cronExecutingUser = cronExecutingUser; + $scope.setConfig(); + }; /** Set release resource for this note **/ - $scope.setReleaseResource = function (value) { - $scope.note.config.releaseresource = value - $scope.setConfig() - } + $scope.setReleaseResource = function(value) { + $scope.note.config.releaseresource = value; + $scope.setConfig(); + }; /** Update note config **/ - $scope.setConfig = function (config) { + $scope.setConfig = function(config) { if (config) { - $scope.note.config = config + $scope.note.config = config; } - websocketMsgSrv.updateNote($scope.note.id, $scope.note.name, $scope.note.config) - } + websocketMsgSrv.updateNote($scope.note.id, $scope.note.name, $scope.note.config); + }; /** Update the note name */ - $scope.updateNoteName = function (newName) { - const trimmedNewName = newName.trim() + $scope.updateNoteName = function(newName) { + const trimmedNewName = newName.trim(); if (trimmedNewName.length > 0 && $scope.note.name !== trimmedNewName) { - $scope.note.name = trimmedNewName - websocketMsgSrv.renameNote($scope.note.id, $scope.note.name) + $scope.note.name = trimmedNewName; + websocketMsgSrv.renameNote($scope.note.id, $scope.note.name); } - } + }; - const initializeLookAndFeel = function () { + const initializeLookAndFeel = function() { if (!$scope.note.config.looknfeel) { - $scope.note.config.looknfeel = 'default' + $scope.note.config.looknfeel = 'default'; } else { - $scope.viewOnly = $scope.note.config.looknfeel === 'report' ? true : false + $scope.viewOnly = $scope.note.config.looknfeel === 'report' ? true : false; } if ($scope.note.paragraphs && $scope.note.paragraphs[0]) { - $scope.note.paragraphs[0].focus = true - } - $rootScope.$broadcast('setLookAndFeel', $scope.note.config.looknfeel) - } - - let cleanParagraphExcept = function (paragraphId, note) { - let noteCopy = {} - noteCopy.id = note.id - noteCopy.name = note.name - noteCopy.config = note.config - noteCopy.info = note.info - noteCopy.paragraphs = [] + $scope.note.paragraphs[0].focus = true; + } + $rootScope.$broadcast('setLookAndFeel', $scope.note.config.looknfeel); + }; + + let cleanParagraphExcept = function(paragraphId, note) { + let noteCopy = {}; + noteCopy.id = note.id; + noteCopy.name = note.name; + noteCopy.config = note.config; + noteCopy.info = note.info; + noteCopy.paragraphs = []; for (let i = 0; i < note.paragraphs.length; i++) { if (note.paragraphs[i].id === paragraphId) { - noteCopy.paragraphs[0] = note.paragraphs[i] + noteCopy.paragraphs[0] = note.paragraphs[i]; if (!noteCopy.paragraphs[0].config) { - noteCopy.paragraphs[0].config = {} + noteCopy.paragraphs[0].config = {}; } - noteCopy.paragraphs[0].config.editorHide = true - noteCopy.paragraphs[0].config.tableHide = false - break + noteCopy.paragraphs[0].config.editorHide = true; + noteCopy.paragraphs[0].config.tableHide = false; + break; } } - return noteCopy - } + return noteCopy; + }; - let addPara = function (paragraph, index) { - $scope.note.paragraphs.splice(index, 0, paragraph) - $scope.note.paragraphs.map(para => { + let addPara = function(paragraph, index) { + $scope.note.paragraphs.splice(index, 0, paragraph); + $scope.note.paragraphs.map((para) => { if (para.id === paragraph.id) { - para.focus = true + para.focus = true; // we need `$timeout` since angular DOM might not be initialized - $timeout(() => { $scope.$broadcast('focusParagraph', para.id, 0, null, false) }) + $timeout(() => { + $scope.$broadcast('focusParagraph', para.id, 0, null, false); + }); } - }) - } + }); + }; - let removePara = function (paragraphId) { - let removeIdx - _.each($scope.note.paragraphs, function (para, idx) { + let removePara = function(paragraphId) { + let removeIdx; + _.each($scope.note.paragraphs, function(para, idx) { if (para.id === paragraphId) { - removeIdx = idx + removeIdx = idx; } - }) - return $scope.note.paragraphs.splice(removeIdx, 1) - } + }); + return $scope.note.paragraphs.splice(removeIdx, 1); + }; - $scope.$on('addParagraph', function (event, paragraph, index) { + $scope.$on('addParagraph', function(event, paragraph, index) { if ($scope.paragraphUrl || $scope.revisionView === true) { - return + return; } - addPara(paragraph, index) - }) + addPara(paragraph, index); + }); - $scope.$on('removeParagraph', function (event, paragraphId) { + $scope.$on('removeParagraph', function(event, paragraphId) { if ($scope.paragraphUrl || $scope.revisionView === true) { - return + return; } - removePara(paragraphId) - }) + removePara(paragraphId); + }); - $scope.$on('moveParagraph', function (event, paragraphId, newIdx) { + $scope.$on('moveParagraph', function(event, paragraphId, newIdx) { if ($scope.revisionView === true) { - return + return; } - let removedPara = removePara(paragraphId) + let removedPara = removePara(paragraphId); if (removedPara && removedPara.length === 1) { - addPara(removedPara[0], newIdx) + addPara(removedPara[0], newIdx); } - }) + }); - $scope.$on('updateNote', function (event, name, config, info) { + $scope.$on('updateNote', function(event, name, config, info) { /** update Note name */ if (name !== $scope.note.name) { - console.log('change note name to : %o', $scope.note.name) - $scope.note.name = name + console.log('change note name to : %o', $scope.note.name); + $scope.note.name = name; } - $scope.note.config = config - $scope.note.info = info - initializeLookAndFeel() - }) + $scope.note.config = config; + $scope.note.info = info; + initializeLookAndFeel(); + }); - let getInterpreterBindings = function () { - websocketMsgSrv.getInterpreterBindings($scope.note.id) - } + let getInterpreterBindings = function() { + websocketMsgSrv.getInterpreterBindings($scope.note.id); + }; - $scope.$on('interpreterBindings', function (event, data) { - $scope.interpreterBindings = data.interpreterBindings - $scope.interpreterBindingsOrig = angular.copy($scope.interpreterBindings) // to check dirty + $scope.$on('interpreterBindings', function(event, data) { + $scope.interpreterBindings = data.interpreterBindings; + $scope.interpreterBindingsOrig = angular.copy($scope.interpreterBindings); // to check dirty - let selected = false - let key - let setting + let selected = false; + let key; + let setting; for (key in $scope.interpreterBindings) { - setting = $scope.interpreterBindings[key] - if (setting.selected) { - selected = true - break + if($scope.interpreterBindings.hasOwnProperty(key)) { + setting = $scope.interpreterBindings[key]; + if (setting.selected) { + selected = true; + break; + } } } if (!selected) { // make default selection - let selectedIntp = {} + let selectedIntp = {}; for (key in $scope.interpreterBindings) { - setting = $scope.interpreterBindings[key] - if (!selectedIntp[setting.name]) { - setting.selected = true - selectedIntp[setting.name] = true + if ($scope.interpreterBindings.hasOwnProperty(key)) { + setting = $scope.interpreterBindings[key]; + if (!selectedIntp[setting.name]) { + setting.selected = true; + selectedIntp[setting.name] = true; + } } } - $scope.showSetting = true + $scope.showSetting = true; } - }) + }); $scope.interpreterSelectionListeners = { - accept: function (sourceItemHandleScope, destSortableScope) { return true }, - itemMoved: function (event) {}, - orderChanged: function (event) {} - } + accept: function(sourceItemHandleScope, destSortableScope) { + return true; + }, + itemMoved: function(event) {}, + orderChanged: function(event) {}, + }; $scope.closeAdditionalBoards = function() { - $scope.closeSetting() - $scope.closePermissions() - $scope.closeRevisionsComparator() - } + $scope.closeSetting(); + $scope.closePermissions(); + $scope.closeRevisionsComparator(); + }; - $scope.openSetting = function () { - $scope.showSetting = true - getInterpreterBindings() - } + $scope.openSetting = function() { + $scope.showSetting = true; + getInterpreterBindings(); + }; - $scope.closeSetting = function () { + $scope.closeSetting = function() { if (isSettingDirty()) { BootstrapDialog.confirm({ closable: true, title: '', message: 'Interpreter setting changes will be discarded.', - callback: function (result) { + callback: function(result) { if (result) { - $scope.$apply(function () { - $scope.showSetting = false - }) + $scope.$apply(function() { + $scope.showSetting = false; + }); } - } - }) + }, + }); } else { - $scope.showSetting = false + $scope.showSetting = false; } - } + }; - $scope.saveSetting = function () { - let selectedSettingIds = [] + $scope.saveSetting = function() { + let selectedSettingIds = []; for (let no in $scope.interpreterBindings) { - let setting = $scope.interpreterBindings[no] - if (setting.selected) { - selectedSettingIds.push(setting.id) + if ($scope.interpreterBindings.hasOwnProperty(no)) { + let setting = $scope.interpreterBindings[no]; + if (setting.selected) { + selectedSettingIds.push(setting.id); + } } } - websocketMsgSrv.saveInterpreterBindings($scope.note.id, selectedSettingIds) - console.log('Interpreter bindings %o saved', selectedSettingIds) + websocketMsgSrv.saveInterpreterBindings($scope.note.id, selectedSettingIds); + console.log('Interpreter bindings %o saved', selectedSettingIds); - _.forEach($scope.note.paragraphs, function (n, key) { - let regExp = /^\s*%/g + _.forEach($scope.note.paragraphs, function(n, key) { + let regExp = /^\s*%/g; if (n.text && !regExp.exec(n.text)) { - $scope.$broadcast('saveInterpreterBindings', n.id) + $scope.$broadcast('saveInterpreterBindings', n.id); } - }) + }); - $scope.showSetting = false - } + $scope.showSetting = false; + }; - $scope.toggleSetting = function () { + $scope.toggleSetting = function() { if ($scope.showSetting) { - $scope.closeSetting() + $scope.closeSetting(); } else { - $scope.closeAdditionalBoards() - $scope.openSetting() - angular.element('html, body').animate({ scrollTop: 0 }, 'slow') + $scope.closeAdditionalBoards(); + $scope.openSetting(); + angular.element('html, body').animate({scrollTop: 0}, 'slow'); } - } + }; - $scope.openRevisionsComparator = function () { - $scope.showRevisionsComparator = true - } + $scope.openRevisionsComparator = function() { + $scope.showRevisionsComparator = true; + }; - $scope.closeRevisionsComparator = function () { - $scope.showRevisionsComparator = false - } + $scope.closeRevisionsComparator = function() { + $scope.showRevisionsComparator = false; + }; - $scope.toggleRevisionsComparator = function () { + $scope.toggleRevisionsComparator = function() { if ($scope.showRevisionsComparator) { - $scope.closeRevisionsComparator() + $scope.closeRevisionsComparator(); } else { - $scope.closeAdditionalBoards() - $scope.openRevisionsComparator() - angular.element('html, body').animate({ scrollTop: 0 }, 'slow') + $scope.closeAdditionalBoards(); + $scope.openRevisionsComparator(); + angular.element('html, body').animate({scrollTop: 0}, 'slow'); } - } + }; - let getPermissions = function (callback) { + let getPermissions = function(callback) { $http.get(baseUrlSrv.getRestApiBase() + '/notebook/' + $scope.note.id + '/permissions') - .success(function (data, status, headers, config) { - $scope.permissions = data.body - $scope.permissionsOrig = angular.copy($scope.permissions) // to check dirty + .success(function(data, status, headers, config) { + $scope.permissions = data.body; + $scope.permissionsOrig = angular.copy($scope.permissions); // to check dirty let selectJson = { tokenSeparators: [',', ' '], ajax: { - url: function (params) { + url: function(params) { if (!params.term) { - return false + return false; } - return baseUrlSrv.getRestApiBase() + '/security/userlist/' + params.term + return baseUrlSrv.getRestApiBase() + '/security/userlist/' + params.term; }, delay: 250, - processResults: function (data, params) { - let results = [] + processResults: function(data, params) { + let results = []; if (data.body.users.length !== 0) { - let users = [] + let users = []; for (let len = 0; len < data.body.users.length; len++) { users.push({ 'id': data.body.users[len], - 'text': data.body.users[len] - }) + 'text': data.body.users[len], + }); } results.push({ 'text': 'Users :', - 'children': users - }) + 'children': users, + }); } if (data.body.roles.length !== 0) { - let roles = [] + let roles = []; for (let len = 0; len < data.body.roles.length; len++) { roles.push({ 'id': data.body.roles[len], - 'text': data.body.roles[len] - }) + 'text': data.body.roles[len], + }); } results.push({ 'text': 'Roles :', - 'children': roles - }) + 'children': roles, + }); } return { results: results, pagination: { - more: false - } - } + more: false, + }, + }; }, - cache: false + cache: false, }, width: ' ', tags: true, - minimumInputLength: 3 - } - - $scope.setIamOwner() - angular.element('#selectOwners').select2(selectJson) - angular.element('#selectReaders').select2(selectJson) - angular.element('#selectRunners').select2(selectJson) - angular.element('#selectWriters').select2(selectJson) + minimumInputLength: 3, + }; + + $scope.setIamOwner(); + angular.element('#selectOwners').select2(selectJson); + angular.element('#selectReaders').select2(selectJson); + angular.element('#selectRunners').select2(selectJson); + angular.element('#selectWriters').select2(selectJson); if (callback) { - callback() + callback(); } }) - .error(function (data, status, headers, config) { + .error(function(data, status, headers, config) { if (status !== 0) { - console.log('Error %o %o', status, data.message) + console.log('Error %o %o', status, data.message); } - }) - } + }); + }; - $scope.openPermissions = function () { - $scope.showPermissions = true - getPermissions() - } + $scope.openPermissions = function() { + $scope.showPermissions = true; + getPermissions(); + }; - $scope.closePermissions = function () { + $scope.closePermissions = function() { if (isPermissionsDirty()) { BootstrapDialog.confirm({ closable: true, title: '', message: 'Changes will be discarded.', - callback: function (result) { + callback: function(result) { if (result) { - $scope.$apply(function () { - $scope.showPermissions = false - }) + $scope.$apply(function() { + $scope.showPermissions = false; + }); } - } - }) + }, + }); } else { - $scope.showPermissions = false + $scope.showPermissions = false; } - } + }; - function convertPermissionsToArray () { - $scope.permissions.owners = angular.element('#selectOwners').val() - $scope.permissions.readers = angular.element('#selectReaders').val() - $scope.permissions.runners = angular.element('#selectRunners').val() - $scope.permissions.writers = angular.element('#selectWriters').val() - angular.element('.permissionsForm select').find('option:not([is-select2="false"])').remove() + function convertPermissionsToArray() { + $scope.permissions.owners = angular.element('#selectOwners').val(); + $scope.permissions.readers = angular.element('#selectReaders').val(); + $scope.permissions.runners = angular.element('#selectRunners').val(); + $scope.permissions.writers = angular.element('#selectWriters').val(); + angular.element('.permissionsForm select').find('option:not([is-select2="false"])').remove(); } $scope.hasMatches = function() { - return $scope.search.occurrencesCount > 0 - } + return $scope.search.occurrencesCount > 0; + }; const markAllOccurrences = function() { - $scope.search.occurrencesCount = 0 - $scope.search.occurrencesHidden = false - currentSearchParagraph = 0 - $scope.$broadcast('markAllOccurrences', $scope.search.searchText) - $scope.search.currentOccurrence = $scope.search.occurrencesCount > 0 ? 1 : 0 - } + $scope.search.occurrencesCount = 0; + $scope.search.occurrencesHidden = false; + currentSearchParagraph = 0; + $scope.$broadcast('markAllOccurrences', $scope.search.searchText); + $scope.search.currentOccurrence = $scope.search.occurrencesCount > 0 ? 1 : 0; + }; $scope.markAllOccurrencesAndHighlightFirst = function() { - $scope.search.needHighlightFirst = true - markAllOccurrences() - } + $scope.search.needHighlightFirst = true; + markAllOccurrences(); + }; const increaseCurrentOccurence = function() { - ++$scope.search.currentOccurrence + ++$scope.search.currentOccurrence; if ($scope.search.currentOccurrence > $scope.search.occurrencesCount) { - $scope.search.currentOccurrence = 1 + $scope.search.currentOccurrence = 1; } - } + }; const decreaseCurrentOccurence = function() { - --$scope.search.currentOccurrence + --$scope.search.currentOccurrence; if ($scope.search.currentOccurrence === 0) { - $scope.search.currentOccurrence = $scope.search.occurrencesCount + $scope.search.currentOccurrence = $scope.search.occurrencesCount; } - } + }; const sendNextOccurrenceMessage = function() { if ($scope.search.occurrencesCount === 0) { - markAllOccurrences() + markAllOccurrences(); if ($scope.search.occurrencesCount === 0) { - return + return; } } if ($scope.search.occurrencesHidden) { - markAllOccurrences() + markAllOccurrences(); } - $scope.$broadcast('nextOccurrence', $scope.note.paragraphs[currentSearchParagraph].id) - } + $scope.$broadcast('nextOccurrence', $scope.note.paragraphs[currentSearchParagraph].id); + }; const sendPrevOccurrenceMessage = function() { if ($scope.search.occurrencesCount === 0) { - markAllOccurrences() + markAllOccurrences(); if ($scope.search.occurrencesCount === 0) { - return + return; } } if ($scope.search.occurrencesHidden) { - markAllOccurrences() - currentSearchParagraph = $scope.note.paragraphs.length - 1 + markAllOccurrences(); + currentSearchParagraph = $scope.note.paragraphs.length - 1; } - $scope.$broadcast('prevOccurrence', $scope.note.paragraphs[currentSearchParagraph].id) - } + $scope.$broadcast('prevOccurrence', $scope.note.paragraphs[currentSearchParagraph].id); + }; const increaseCurrentSearchParagraph = function() { - ++currentSearchParagraph + ++currentSearchParagraph; if (currentSearchParagraph >= $scope.note.paragraphs.length) { - currentSearchParagraph = 0 + currentSearchParagraph = 0; } - } + }; - const decreaseCurrentSearchParagraph = function () { - --currentSearchParagraph + const decreaseCurrentSearchParagraph = function() { + --currentSearchParagraph; if (currentSearchParagraph === -1) { - currentSearchParagraph = $scope.note.paragraphs.length - 1 + currentSearchParagraph = $scope.note.paragraphs.length - 1; } - } + }; $scope.$on('occurrencesExists', function(event, count) { - $scope.search.occurrencesCount += count + $scope.search.occurrencesCount += count; if ($scope.search.needHighlightFirst) { - sendNextOccurrenceMessage() - $scope.search.needHighlightFirst = false + sendNextOccurrenceMessage(); + $scope.search.needHighlightFirst = false; } - }) + }); $scope.nextOccurrence = function() { - sendNextOccurrenceMessage() - increaseCurrentOccurence() - } + sendNextOccurrenceMessage(); + increaseCurrentOccurence(); + }; $scope.$on('noNextOccurrence', function(event) { - increaseCurrentSearchParagraph() - sendNextOccurrenceMessage() - }) + increaseCurrentSearchParagraph(); + sendNextOccurrenceMessage(); + }); $scope.prevOccurrence = function() { - sendPrevOccurrenceMessage() - decreaseCurrentOccurence() - } + sendPrevOccurrenceMessage(); + decreaseCurrentOccurence(); + }; $scope.$on('noPrevOccurrence', function(event) { - decreaseCurrentSearchParagraph() - sendPrevOccurrenceMessage() - }) + decreaseCurrentSearchParagraph(); + sendPrevOccurrenceMessage(); + }); $scope.$on('editorClicked', function() { - $scope.search.occurrencesHidden = true - $scope.$broadcast('unmarkAll') - }) + $scope.search.occurrencesHidden = true; + $scope.$broadcast('unmarkAll'); + }); $scope.replace = function() { if ($scope.search.occurrencesCount === 0) { - $scope.markAllOccurrencesAndHighlightFirst() + $scope.markAllOccurrencesAndHighlightFirst(); if ($scope.search.occurrencesCount === 0) { - return + return; } } if ($scope.search.occurrencesHidden) { - $scope.markAllOccurrencesAndHighlightFirst() - return + $scope.markAllOccurrencesAndHighlightFirst(); + return; } - $scope.$broadcast('replaceCurrent', $scope.search.searchText, $scope.search.replaceText) + $scope.$broadcast('replaceCurrent', $scope.search.searchText, $scope.search.replaceText); if ($scope.search.needToSendNextOccurrenceAfterReplace) { - sendNextOccurrenceMessage() - $scope.search.needToSendNextOccurrenceAfterReplace = false + sendNextOccurrenceMessage(); + $scope.search.needToSendNextOccurrenceAfterReplace = false; } - } + }; $scope.$on('occurrencesCountChanged', function(event, cnt) { - $scope.search.occurrencesCount += cnt + $scope.search.occurrencesCount += cnt; if ($scope.search.occurrencesCount === 0) { - $scope.search.currentOccurrence = 0 + $scope.search.currentOccurrence = 0; } else { - $scope.search.currentOccurrence += cnt + 1 + $scope.search.currentOccurrence += cnt + 1; if ($scope.search.currentOccurrence > $scope.search.occurrencesCount) { - $scope.search.currentOccurrence = 1 + $scope.search.currentOccurrence = 1; } } - }) + }); $scope.replaceAll = function() { if ($scope.search.occurrencesCount === 0) { - return + return; } if ($scope.search.occurrencesHidden) { - $scope.markAllOccurrencesAndHighlightFirst() + $scope.markAllOccurrencesAndHighlightFirst(); } - $scope.$broadcast('replaceAll', $scope.search.searchText, $scope.search.replaceText) - $scope.markAllOccurrencesAndHighlightFirst() - } + $scope.$broadcast('replaceAll', $scope.search.searchText, $scope.search.replaceText); + $scope.markAllOccurrencesAndHighlightFirst(); + }; $scope.$on('noNextOccurrenceAfterReplace', function() { - $scope.search.occurrencesCount = 0 - $scope.search.needHighlightFirst = false - $scope.search.needToSendNextOccurrenceAfterReplace = false - $scope.$broadcast('checkOccurrences') - increaseCurrentSearchParagraph() + $scope.search.occurrencesCount = 0; + $scope.search.needHighlightFirst = false; + $scope.search.needToSendNextOccurrenceAfterReplace = false; + $scope.$broadcast('checkOccurrences'); + increaseCurrentSearchParagraph(); if ($scope.search.occurrencesCount > 0) { - $scope.search.needToSendNextOccurrenceAfterReplace = true + $scope.search.needToSendNextOccurrenceAfterReplace = true; } - }) + }); $scope.onPressOnFindInput = function(event) { if (event.keyCode === 13) { - $scope.nextOccurrence() + $scope.nextOccurrence(); } - } + }; let makeSearchBoxVisible = function() { if ($scope.search.searchBoxOpened) { - $scope.search.searchBoxOpened = false - console.log('make 0') - $scope.search.left = '0px' + $scope.search.searchBoxOpened = false; + console.log('make 0'); + $scope.search.left = '0px'; } else { - $scope.search.searchBoxOpened = true - let searchGroupRect = angular.element('#searchGroup')[0].getBoundingClientRect() - console.log('make visible') - let dropdownRight = searchGroupRect.left + $scope.search.searchBoxWidth - console.log(dropdownRight + ' ' + window.innerWidth) + $scope.search.searchBoxOpened = true; + let searchGroupRect = angular.element('#searchGroup')[0].getBoundingClientRect(); + console.log('make visible'); + let dropdownRight = searchGroupRect.left + $scope.search.searchBoxWidth; + console.log(dropdownRight + ' ' + window.innerWidth); if (dropdownRight + 5 > window.innerWidth) { - $scope.search.left = window.innerWidth - dropdownRight - 15 + 'px' + $scope.search.left = window.innerWidth - dropdownRight - 15 + 'px'; } } - } + }; $scope.searchClicked = function() { - makeSearchBoxVisible() - } + makeSearchBoxVisible(); + }; $scope.$on('toggleSearchBox', function() { - let elem = angular.element('#searchGroup') + let elem = angular.element('#searchGroup'); if ($scope.search.searchBoxOpened) { - elem.removeClass('open') + elem.removeClass('open'); } else { - elem.addClass('open') + elem.addClass('open'); } - $timeout(makeSearchBoxVisible()) - }) + $timeout(makeSearchBoxVisible()); + }); $scope.restartInterpreter = function(interpreter) { const thisConfirm = BootstrapDialog.confirm({ @@ -999,37 +1011,37 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, callback: function(result) { if (result) { let payload = { - 'noteId': $scope.note.id - } + 'noteId': $scope.note.id, + }; - thisConfirm.$modalFooter.find('button').addClass('disabled') + thisConfirm.$modalFooter.find('button').addClass('disabled'); thisConfirm.$modalFooter.find('button:contains("OK")') - .html(' Saving Setting') + .html(' Saving Setting'); $http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/restart/' + interpreter.id, payload) .success(function(data, status, headers, config) { - let index = _.findIndex($scope.interpreterSettings, {'id': interpreter.id}) - $scope.interpreterSettings[index] = data.body - thisConfirm.close() - }).error(function (data, status, headers, config) { - thisConfirm.close() - console.log('Error %o %o', status, data.message) + let index = _.findIndex($scope.interpreterSettings, {'id': interpreter.id}); + $scope.interpreterSettings[index] = data.body; + thisConfirm.close(); + }).error(function(data, status, headers, config) { + thisConfirm.close(); + console.log('Error %o %o', status, data.message); BootstrapDialog.show({ title: 'Error restart interpreter.', - message: data.message - }) - }) - return false + message: data.message, + }); + }); + return false; } - } - }) - } + }, + }); + }; - $scope.savePermissions = function () { + $scope.savePermissions = function() { if ($scope.isAnonymous || $rootScope.ticket.principal.trim().length === 0) { - $scope.blockAnonUsers() + $scope.blockAnonUsers(); } - convertPermissionsToArray() + convertPermissionsToArray(); if ($scope.isOwnerEmpty()) { BootstrapDialog.show({ closable: false, @@ -1040,43 +1052,43 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, { label: 'Set', action: function(dialog) { - dialog.close() - $scope.permissions.owners = [$rootScope.ticket.principal] - $scope.setPermissions() - } + dialog.close(); + $scope.permissions.owners = [$rootScope.ticket.principal]; + $scope.setPermissions(); + }, }, { label: 'Cancel', action: function(dialog) { - dialog.close() - $scope.openPermissions() - } - } - ] - }) + dialog.close(); + $scope.openPermissions(); + }, + }, + ], + }); } else { - $scope.setPermissions() + $scope.setPermissions(); } - } + }; $scope.setPermissions = function() { $http.put(baseUrlSrv.getRestApiBase() + '/notebook/' + $scope.note.id + '/permissions', $scope.permissions, {withCredentials: true}) - .success(function (data, status, headers, config) { - getPermissions(function () { - console.log('Note permissions %o saved', $scope.permissions) + .success(function(data, status, headers, config) { + getPermissions(function() { + console.log('Note permissions %o saved', $scope.permissions); BootstrapDialog.alert({ closable: true, title: 'Permissions Saved Successfully', message: 'Owners : ' + $scope.permissions.owners + '\n\n' + 'Readers : ' + $scope.permissions.readers + '\n\n' + 'Runners : ' + $scope.permissions.runners + - '\n\n' + 'Writers : ' + $scope.permissions.writers - }) - $scope.showPermissions = false - }) + '\n\n' + 'Writers : ' + $scope.permissions.writers, + }); + $scope.showPermissions = false; + }); }) - .error(function (data, status, headers, config) { - console.log('Error %o %o', status, data.message) + .error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); BootstrapDialog.show({ closable: false, closeByBackdrop: false, @@ -1086,362 +1098,366 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, buttons: [ { label: 'Login', - action: function (dialog) { - dialog.close() + action: function(dialog) { + dialog.close(); angular.element('#loginModal').modal({ - show: 'true' - }) - } + show: 'true', + }); + }, }, { label: 'Cancel', - action: function (dialog) { - dialog.close() - $location.path('/') - } - } - ] - }) - }) - } - - $scope.togglePermissions = function () { - let principal = $rootScope.ticket.principal - $scope.isAnonymous = principal === 'anonymous' ? true : false + action: function(dialog) { + dialog.close(); + $location.path('/'); + }, + }, + ], + }); + }); + }; + + $scope.togglePermissions = function() { + let principal = $rootScope.ticket.principal; + $scope.isAnonymous = principal === 'anonymous' ? true : false; if (!!principal && $scope.isAnonymous) { - $scope.blockAnonUsers() + $scope.blockAnonUsers(); } else { if ($scope.showPermissions) { - $scope.closePermissions() - angular.element('#selectOwners').select2({}) - angular.element('#selectReaders').select2({}) - angular.element('#selectRunners').select2({}) - angular.element('#selectWriters').select2({}) + $scope.closePermissions(); + angular.element('#selectOwners').select2({}); + angular.element('#selectReaders').select2({}); + angular.element('#selectRunners').select2({}); + angular.element('#selectWriters').select2({}); } else { - $scope.closeAdditionalBoards() - $scope.openPermissions() + $scope.closeAdditionalBoards(); + $scope.openPermissions(); } } - } + }; - $scope.setIamOwner = function () { + $scope.setIamOwner = function() { if ($scope.permissions.owners.length > 0 && _.indexOf($scope.permissions.owners, $rootScope.ticket.principal) < 0) { - $scope.isOwner = false - return false + $scope.isOwner = false; + return false; } - $scope.isOwner = true - return true - } + $scope.isOwner = true; + return true; + }; - $scope.toggleNotePersonalizedMode = function () { - let personalizedMode = $scope.note.config.personalizedMode + $scope.toggleNotePersonalizedMode = function() { + let personalizedMode = $scope.note.config.personalizedMode; if ($scope.isOwner) { BootstrapDialog.confirm({ closable: true, title: 'Setting the result display', - message: function (dialog) { - let modeText = $scope.note.config.personalizedMode === 'true' ? 'collaborate' : 'personalize' - return 'Do you want to ' + modeText + ' your analysis?' + message: function(dialog) { + let modeText = $scope.note.config.personalizedMode === 'true' ? 'collaborate' : 'personalize'; + return 'Do you want to ' + modeText + ' your analysis?'; }, - callback: function (result) { + callback: function(result) { if (result) { if ($scope.note.config.personalizedMode === undefined) { - $scope.note.config.personalizedMode = 'false' + $scope.note.config.personalizedMode = 'false'; } - $scope.note.config.personalizedMode = personalizedMode === 'true' ? 'false' : 'true' - websocketMsgSrv.updatePersonalizedMode($scope.note.id, $scope.note.config.personalizedMode) + $scope.note.config.personalizedMode = personalizedMode === 'true' ? 'false' : 'true'; + websocketMsgSrv.updatePersonalizedMode($scope.note.id, $scope.note.config.personalizedMode); } - } - }) + }, + }); } - } + }; - const isSettingDirty = function () { + const isSettingDirty = function() { if (angular.equals($scope.interpreterBindings, $scope.interpreterBindingsOrig)) { - return false + return false; } else { - return true + return true; } - } + }; - const isPermissionsDirty = function () { + const isPermissionsDirty = function() { if (angular.equals($scope.permissions, $scope.permissionsOrig)) { - return false + return false; } else { - return true + return true; } - } + }; - angular.element(document).click(function () { - angular.element('.ace_autocomplete').hide() - }) + angular.element(document).click(function() { + angular.element('.ace_autocomplete').hide(); + }); $scope.isOwnerEmpty = function() { if ($scope.permissions.owners.length > 0) { for (let i = 0; i < $scope.permissions.owners.length; i++) { if ($scope.permissions.owners[i].trim().length > 0) { - return false + return false; } else if (i === $scope.permissions.owners.length - 1) { - return true + return true; } } } else { - return true + return true; } - } + }; /* ** $scope.$on functions below */ - $scope.$on('runAllAbove', function (event, paragraph, isNeedConfirm) { - let allParagraphs = $scope.note.paragraphs - let toRunParagraphs = [] + $scope.$on('runAllAbove', function(event, paragraph, isNeedConfirm) { + let allParagraphs = $scope.note.paragraphs; + let toRunParagraphs = []; for (let i = 0; allParagraphs[i] !== paragraph; i++) { - if (i === allParagraphs.length - 1) { return } // if paragraph not in array of all paragraphs - toRunParagraphs.push(allParagraphs[i]) + if (i === allParagraphs.length - 1) { + return; + } // if paragraph not in array of all paragraphs + toRunParagraphs.push(allParagraphs[i]); } - const paragraphs = toRunParagraphs.map(p => { + const paragraphs = toRunParagraphs.map((p) => { return { id: p.id, title: p.title, paragraph: p.text, config: p.config, - params: p.settings.params - } - }) + params: p.settings.params, + }; + }); if (!isNeedConfirm) { - websocketMsgSrv.runAllParagraphs($scope.note.id, paragraphs) + websocketMsgSrv.runAllParagraphs($scope.note.id, paragraphs); } else { BootstrapDialog.confirm({ closable: true, title: '', message: 'Run all above?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.runAllParagraphs($scope.note.id, paragraphs) + websocketMsgSrv.runAllParagraphs($scope.note.id, paragraphs); } - } - }) + }, + }); } - $scope.saveCursorPosition(paragraph) - }) + $scope.saveCursorPosition(paragraph); + }); - $scope.$on('runAllBelowAndCurrent', function (event, paragraph, isNeedConfirm) { - let allParagraphs = $scope.note.paragraphs - let toRunParagraphs = [] + $scope.$on('runAllBelowAndCurrent', function(event, paragraph, isNeedConfirm) { + let allParagraphs = $scope.note.paragraphs; + let toRunParagraphs = []; for (let i = allParagraphs.length - 1; allParagraphs[i] !== paragraph; i--) { - if (i < 0) { return } // if paragraph not in array of all paragraphs - toRunParagraphs.push(allParagraphs[i]) + if (i < 0) { + return; + } // if paragraph not in array of all paragraphs + toRunParagraphs.push(allParagraphs[i]); } - toRunParagraphs.push(paragraph) - toRunParagraphs.reverse() + toRunParagraphs.push(paragraph); + toRunParagraphs.reverse(); - const paragraphs = toRunParagraphs.map(p => { + const paragraphs = toRunParagraphs.map((p) => { return { id: p.id, title: p.title, paragraph: p.text, config: p.config, - params: p.settings.params - } - }) + params: p.settings.params, + }; + }); if (!isNeedConfirm) { - websocketMsgSrv.runAllParagraphs($scope.note.id, paragraphs) + websocketMsgSrv.runAllParagraphs($scope.note.id, paragraphs); } else { BootstrapDialog.confirm({ closable: true, title: '', message: 'Run current and all below?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.runAllParagraphs($scope.note.id, paragraphs) + websocketMsgSrv.runAllParagraphs($scope.note.id, paragraphs); } - } - }) + }, + }); } - $scope.saveCursorPosition(paragraph) - }) + $scope.saveCursorPosition(paragraph); + }); - $scope.saveCursorPosition = function (paragraph) { + $scope.saveCursorPosition = function(paragraph) { let angParagEditor = angular .element('#' + paragraph.id + '_paragraphColumn_main') - .scope().editor - let col = angParagEditor.selection.lead.column - let row = angParagEditor.selection.lead.row - $scope.$broadcast('focusParagraph', paragraph.id, row + 1, col) - } + .scope().editor; + let col = angParagEditor.selection.lead.column; + let row = angParagEditor.selection.lead.row; + $scope.$broadcast('focusParagraph', paragraph.id, row + 1, col); + }; - $scope.$on('setConnectedStatus', function (event, param) { + $scope.$on('setConnectedStatus', function(event, param) { if (connectedOnce && param) { - initNotebook() + initNotebook(); } - connectedOnce = true - }) + connectedOnce = true; + }); - $scope.$on('moveParagraphUp', function (event, paragraph) { - let newIndex = -1 + $scope.$on('moveParagraphUp', function(event, paragraph) { + let newIndex = -1; for (let i = 0; i < $scope.note.paragraphs.length; i++) { if ($scope.note.paragraphs[i].id === paragraph.id) { - newIndex = i - 1 - break + newIndex = i - 1; + break; } } if (newIndex < 0 || newIndex >= $scope.note.paragraphs.length) { - return + return; } // save dirtyText of moving paragraphs. - let prevParagraph = $scope.note.paragraphs[newIndex] + let prevParagraph = $scope.note.paragraphs[newIndex]; angular .element('#' + paragraph.id + '_paragraphColumn_main') .scope() - .saveParagraph(paragraph) + .saveParagraph(paragraph); angular .element('#' + prevParagraph.id + '_paragraphColumn_main') .scope() - .saveParagraph(prevParagraph) - websocketMsgSrv.moveParagraph(paragraph.id, newIndex) - }) + .saveParagraph(prevParagraph); + websocketMsgSrv.moveParagraph(paragraph.id, newIndex); + }); - $scope.$on('moveParagraphDown', function (event, paragraph) { - let newIndex = -1 + $scope.$on('moveParagraphDown', function(event, paragraph) { + let newIndex = -1; for (let i = 0; i < $scope.note.paragraphs.length; i++) { if ($scope.note.paragraphs[i].id === paragraph.id) { - newIndex = i + 1 - break + newIndex = i + 1; + break; } } if (newIndex < 0 || newIndex >= $scope.note.paragraphs.length) { - return + return; } // save dirtyText of moving paragraphs. - let nextParagraph = $scope.note.paragraphs[newIndex] + let nextParagraph = $scope.note.paragraphs[newIndex]; angular .element('#' + paragraph.id + '_paragraphColumn_main') .scope() - .saveParagraph(paragraph) + .saveParagraph(paragraph); angular .element('#' + nextParagraph.id + '_paragraphColumn_main') .scope() - .saveParagraph(nextParagraph) - websocketMsgSrv.moveParagraph(paragraph.id, newIndex) - }) + .saveParagraph(nextParagraph); + websocketMsgSrv.moveParagraph(paragraph.id, newIndex); + }); - $scope.$on('moveFocusToPreviousParagraph', function (event, currentParagraphId) { - let focus = false + $scope.$on('moveFocusToPreviousParagraph', function(event, currentParagraphId) { + let focus = false; for (let i = $scope.note.paragraphs.length - 1; i >= 0; i--) { if (focus === false) { if ($scope.note.paragraphs[i].id === currentParagraphId) { - focus = true - continue + focus = true; + continue; } } else { - $scope.$broadcast('focusParagraph', $scope.note.paragraphs[i].id, -1) - break + $scope.$broadcast('focusParagraph', $scope.note.paragraphs[i].id, -1); + break; } } - }) + }); - $scope.$on('moveFocusToNextParagraph', function (event, currentParagraphId) { - let focus = false + $scope.$on('moveFocusToNextParagraph', function(event, currentParagraphId) { + let focus = false; for (let i = 0; i < $scope.note.paragraphs.length; i++) { if (focus === false) { if ($scope.note.paragraphs[i].id === currentParagraphId) { - focus = true - continue + focus = true; + continue; } } else { - $scope.$broadcast('focusParagraph', $scope.note.paragraphs[i].id, 0) - break + $scope.$broadcast('focusParagraph', $scope.note.paragraphs[i].id, 0); + break; } } - }) + }); - $scope.$on('insertParagraph', function (event, paragraphId, position) { + $scope.$on('insertParagraph', function(event, paragraphId, position) { if ($scope.revisionView === true) { - return + return; } - let newIndex = -1 + let newIndex = -1; for (let i = 0; i < $scope.note.paragraphs.length; i++) { if ($scope.note.paragraphs[i].id === paragraphId) { // determine position of where to add new paragraph; default is below if (position === 'above') { - newIndex = i + newIndex = i; } else { - newIndex = i + 1 + newIndex = i + 1; } - break + break; } } if (newIndex < 0 || newIndex > $scope.note.paragraphs.length) { - return + return; } - websocketMsgSrv.insertParagraph(newIndex) - }) + websocketMsgSrv.insertParagraph(newIndex); + }); - $scope.$on('setNoteContent', function (event, note) { + $scope.$on('setNoteContent', function(event, note) { if (note === undefined) { - $location.path('/') + $location.path('/'); } - $scope.note = note + $scope.note = note; - $scope.paragraphUrl = $routeParams.paragraphId - $scope.asIframe = $routeParams.asIframe + $scope.paragraphUrl = $routeParams.paragraphId; + $scope.asIframe = $routeParams.asIframe; if ($scope.paragraphUrl) { - $scope.note = cleanParagraphExcept($scope.paragraphUrl, $scope.note) - $scope.$broadcast('$unBindKeyEvent', $scope.$unBindKeyEvent) - $rootScope.$broadcast('setIframe', $scope.asIframe) - initializeLookAndFeel() - return + $scope.note = cleanParagraphExcept($scope.paragraphUrl, $scope.note); + $scope.$broadcast('$unBindKeyEvent', $scope.$unBindKeyEvent); + $rootScope.$broadcast('setIframe', $scope.asIframe); + initializeLookAndFeel(); + return; } - initializeLookAndFeel() + initializeLookAndFeel(); // open interpreter binding setting when there're none selected - getInterpreterBindings() - getPermissions() - let isPersonalized = $scope.note.config.personalizedMode - isPersonalized = isPersonalized === undefined ? 'false' : isPersonalized - $scope.note.config.personalizedMode = isPersonalized - }) - - $scope.$on('$routeChangeStart', function (event, next, current) { + getInterpreterBindings(); + getPermissions(); + let isPersonalized = $scope.note.config.personalizedMode; + isPersonalized = isPersonalized === undefined ? 'false' : isPersonalized; + $scope.note.config.personalizedMode = isPersonalized; + }); + + $scope.$on('$routeChangeStart', function(event, next, current) { if (!$scope.note || !$scope.note.paragraphs) { - return + return; } if ($scope.note && $scope.note.paragraphs) { - $scope.note.paragraphs.map(par => { + $scope.note.paragraphs.map((par) => { if ($scope.allowLeave === true) { - return + return; } let thisScope = angular.element( - '#' + par.id + '_paragraphColumn_main').scope() + '#' + par.id + '_paragraphColumn_main').scope(); if (thisScope.dirtyText === undefined || thisScope.originalText === undefined || thisScope.dirtyText === thisScope.originalText) { - return true + return true; } else { - event.preventDefault() - $scope.showParagraphWarning(next) + event.preventDefault(); + $scope.showParagraphWarning(next); } - }) + }); } - }) + }); - $scope.showParagraphWarning = function (next) { + $scope.showParagraphWarning = function(next) { if ($scope.paragraphWarningDialog.opened !== true) { $scope.paragraphWarningDialog = BootstrapDialog.show({ closable: false, @@ -1451,62 +1467,62 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, message: 'Changes that you have made will not be saved.', buttons: [{ label: 'Stay', - action: function (dialog) { - dialog.close() - } + action: function(dialog) { + dialog.close(); + }, }, { label: 'Leave', - action: function (dialog) { - dialog.close() - let locationToRedirect = next['$$route']['originalPath'] - Object.keys(next.pathParams).map(key => { + action: function(dialog) { + dialog.close(); + let locationToRedirect = next['$$route']['originalPath']; + Object.keys(next.pathParams).map((key) => { locationToRedirect = locationToRedirect.replace(':' + key, - next.pathParams[key]) - }) - $scope.allowLeave = true - $location.path(locationToRedirect) - } - }] - }) + next.pathParams[key]); + }); + $scope.allowLeave = true; + $location.path(locationToRedirect); + }, + }], + }); } - } + }; - $scope.$on('saveNoteForms', function (event, data) { - $scope.note.noteForms = data.formsData.forms - $scope.note.noteParams = data.formsData.params - }) + $scope.$on('saveNoteForms', function(event, data) { + $scope.note.noteForms = data.formsData.forms; + $scope.note.noteParams = data.formsData.params; + }); $scope.isShowNoteForms = function() { if ($scope.note && !angular.equals({}, $scope.note.noteForms)) { - return true + return true; } - return false - } + return false; + }; - $scope.saveNoteForms = function () { - websocketMsgSrv.saveNoteForms($scope.note) - } + $scope.saveNoteForms = function() { + websocketMsgSrv.saveNoteForms($scope.note); + }; - $scope.removeNoteForms = function (formName) { - websocketMsgSrv.removeNoteForms($scope.note, formName) - } + $scope.removeNoteForms = function(formName) { + websocketMsgSrv.removeNoteForms($scope.note, formName); + }; - $scope.$on('$destroy', function () { - angular.element(window).off('beforeunload') - $scope.killSaveTimer() - $scope.saveNote() + $scope.$on('$destroy', function() { + angular.element(window).off('beforeunload'); + $scope.killSaveTimer(); + $scope.saveNote(); - document.removeEventListener('click', $scope.focusParagraphOnClick) - document.removeEventListener('keydown', $scope.keyboardShortcut) - }) + document.removeEventListener('click', $scope.focusParagraphOnClick); + document.removeEventListener('keydown', $scope.keyboardShortcut); + }); - $scope.$on('$unBindKeyEvent', function () { - document.removeEventListener('click', $scope.focusParagraphOnClick) - document.removeEventListener('keydown', $scope.keyboardShortcut) - }) + $scope.$on('$unBindKeyEvent', function() { + document.removeEventListener('click', $scope.focusParagraphOnClick); + document.removeEventListener('keydown', $scope.keyboardShortcut); + }); - angular.element(window).bind('resize', function () { - const actionbarHeight = document.getElementById('actionbar').lastElementChild.clientHeight - angular.element(document.getElementById('content')).css('padding-top', actionbarHeight - 20) - }) + angular.element(window).bind('resize', function() { + const actionbarHeight = document.getElementById('actionbar').lastElementChild.clientHeight; + angular.element(document.getElementById('content')).css('padding-top', actionbarHeight - 20); + }); } diff --git a/zeppelin-web/src/app/notebook/notebook.controller.test.js b/zeppelin-web/src/app/notebook/notebook.controller.test.js index f393d2c09c2..be9f9568e46 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.test.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.test.js @@ -1,139 +1,139 @@ -describe('Controller: NotebookCtrl', function () { - beforeEach(angular.mock.module('zeppelinWebApp')) +describe('Controller: NotebookCtrl', function() { + beforeEach(angular.mock.module('zeppelinWebApp')); - let scope + let scope; let websocketMsgSrvMock = { - getNote: function () {}, - listRevisionHistory: function () {}, - getInterpreterBindings: function () {}, - updateNote: function () {}, - renameNote: function () {} - } + getNote: function() {}, + listRevisionHistory: function() {}, + getInterpreterBindings: function() {}, + updateNote: function() {}, + renameNote: function() {}, + }; let baseUrlSrvMock = { - getRestApiBase: function () { - return 'http://localhost:8080' - } - } + getRestApiBase: function() { + return 'http://localhost:8080'; + }, + }; let noteMock = { id: 1, name: 'my note', config: {}, - } + }; - beforeEach(inject(function ($controller, $rootScope) { - scope = $rootScope.$new() + beforeEach(inject(function($controller, $rootScope) { + scope = $rootScope.$new(); $controller('NotebookCtrl', { $scope: scope, websocketMsgSrv: websocketMsgSrvMock, - baseUrlSrv: baseUrlSrvMock - }) - })) + baseUrlSrv: baseUrlSrvMock, + }); + })); - beforeEach(function () { - scope.note = noteMock - }) + beforeEach(function() { + scope.note = noteMock; + }); let functions = ['getCronOptionNameFromValue', 'removeNote', 'runAllParagraphs', 'saveNote', 'toggleAllEditor', 'showAllEditor', 'hideAllEditor', 'toggleAllTable', 'hideAllTable', 'showAllTable', 'isNoteRunning', 'killSaveTimer', 'startSaveTimer', 'setLookAndFeel', 'setCronScheduler', 'setConfig', 'updateNoteName', - 'openSetting', 'closeSetting', 'saveSetting', 'toggleSetting'] - - functions.forEach(function (fn) { - it('check for scope functions to be defined : ' + fn, function () { - expect(scope[fn]).toBeDefined() - }) - }) - - it('should set default value of "editorToggled" to false', function () { - expect(scope.editorToggled).toEqual(false) - }) - - it('should set "showSetting" to true when openSetting() is called', function () { - scope.openSetting() - expect(scope.showSetting).toEqual(true) - }) - - it('should set "showSetting" to false when closeSetting() is called', function () { - scope.closeSetting() - expect(scope.showSetting).toEqual(false) - }) - - it('should return the correct value for getCronOptionNameFromValue()', function () { - let none = scope.getCronOptionNameFromValue() - let oneMin = scope.getCronOptionNameFromValue('0 0/1 * * * ?') - let fiveMin = scope.getCronOptionNameFromValue('0 0/5 * * * ?') - let oneHour = scope.getCronOptionNameFromValue('0 0 0/1 * * ?') - let threeHours = scope.getCronOptionNameFromValue('0 0 0/3 * * ?') - let sixHours = scope.getCronOptionNameFromValue('0 0 0/6 * * ?') - let twelveHours = scope.getCronOptionNameFromValue('0 0 0/12 * * ?') - let oneDay = scope.getCronOptionNameFromValue('0 0 0 * * ?') - - expect(none).toEqual('') - expect(oneMin).toEqual('1m') - expect(fiveMin).toEqual('5m') - expect(oneHour).toEqual('1h') - expect(threeHours).toEqual('3h') - expect(sixHours).toEqual('6h') - expect(twelveHours).toEqual('12h') - expect(oneDay).toEqual('1d') - }) - - it('should have "isNoteDirty" as null by default', function () { - expect(scope.isNoteDirty).toEqual(null) - }) - - it('should first call killSaveTimer() when calling startSaveTimer()', function () { - expect(scope.saveTimer).toEqual(null) - spyOn(scope, 'killSaveTimer') - scope.startSaveTimer() - expect(scope.killSaveTimer).toHaveBeenCalled() - }) - - it('should set "saveTimer" when saveTimer() and killSaveTimer() are called', function () { - expect(scope.saveTimer).toEqual(null) - scope.startSaveTimer() - expect(scope.saveTimer).toBeTruthy() - scope.killSaveTimer() - expect(scope.saveTimer).toEqual(null) - }) - - it('should NOT update note name when updateNoteName() is called with an invalid name', function () { - spyOn(websocketMsgSrvMock, 'renameNote') - scope.updateNoteName('') - expect(scope.note.name).toEqual(noteMock.name) - expect(websocketMsgSrvMock.renameNote).not.toHaveBeenCalled() - scope.updateNoteName(' ') - expect(scope.note.name).toEqual(noteMock.name) - expect(websocketMsgSrvMock.renameNote).not.toHaveBeenCalled() - scope.updateNoteName(scope.note.name) - expect(scope.note.name).toEqual(noteMock.name) - expect(websocketMsgSrvMock.renameNote).not.toHaveBeenCalled() - }) - - it('should update note name when updateNoteName() is called with a valid name', function () { - spyOn(websocketMsgSrvMock, 'renameNote') - let newName = 'Your Note' - scope.updateNoteName(newName) - expect(scope.note.name).toEqual(newName) - expect(websocketMsgSrvMock.renameNote).toHaveBeenCalled() - }) - - it('should reload note info once per one "setNoteMenu" event', function () { - spyOn(websocketMsgSrvMock, 'getNote') - spyOn(websocketMsgSrvMock, 'listRevisionHistory') - - scope.$broadcast('setNoteMenu') - expect(websocketMsgSrvMock.getNote.calls.count()).toEqual(0) - expect(websocketMsgSrvMock.listRevisionHistory.calls.count()).toEqual(0) - - websocketMsgSrvMock.getNote.calls.reset() - websocketMsgSrvMock.listRevisionHistory.calls.reset() - - scope.$broadcast('setNoteMenu') - expect(websocketMsgSrvMock.getNote.calls.count()).toEqual(0) - expect(websocketMsgSrvMock.listRevisionHistory.calls.count()).toEqual(0) - }) -}) + 'openSetting', 'closeSetting', 'saveSetting', 'toggleSetting']; + + functions.forEach(function(fn) { + it('check for scope functions to be defined : ' + fn, function() { + expect(scope[fn]).toBeDefined(); + }); + }); + + it('should set default value of "editorToggled" to false', function() { + expect(scope.editorToggled).toEqual(false); + }); + + it('should set "showSetting" to true when openSetting() is called', function() { + scope.openSetting(); + expect(scope.showSetting).toEqual(true); + }); + + it('should set "showSetting" to false when closeSetting() is called', function() { + scope.closeSetting(); + expect(scope.showSetting).toEqual(false); + }); + + it('should return the correct value for getCronOptionNameFromValue()', function() { + let none = scope.getCronOptionNameFromValue(); + let oneMin = scope.getCronOptionNameFromValue('0 0/1 * * * ?'); + let fiveMin = scope.getCronOptionNameFromValue('0 0/5 * * * ?'); + let oneHour = scope.getCronOptionNameFromValue('0 0 0/1 * * ?'); + let threeHours = scope.getCronOptionNameFromValue('0 0 0/3 * * ?'); + let sixHours = scope.getCronOptionNameFromValue('0 0 0/6 * * ?'); + let twelveHours = scope.getCronOptionNameFromValue('0 0 0/12 * * ?'); + let oneDay = scope.getCronOptionNameFromValue('0 0 0 * * ?'); + + expect(none).toEqual(''); + expect(oneMin).toEqual('1m'); + expect(fiveMin).toEqual('5m'); + expect(oneHour).toEqual('1h'); + expect(threeHours).toEqual('3h'); + expect(sixHours).toEqual('6h'); + expect(twelveHours).toEqual('12h'); + expect(oneDay).toEqual('1d'); + }); + + it('should have "isNoteDirty" as null by default', function() { + expect(scope.isNoteDirty).toEqual(null); + }); + + it('should first call killSaveTimer() when calling startSaveTimer()', function() { + expect(scope.saveTimer).toEqual(null); + spyOn(scope, 'killSaveTimer'); + scope.startSaveTimer(); + expect(scope.killSaveTimer).toHaveBeenCalled(); + }); + + it('should set "saveTimer" when saveTimer() and killSaveTimer() are called', function() { + expect(scope.saveTimer).toEqual(null); + scope.startSaveTimer(); + expect(scope.saveTimer).toBeTruthy(); + scope.killSaveTimer(); + expect(scope.saveTimer).toEqual(null); + }); + + it('should NOT update note name when updateNoteName() is called with an invalid name', function() { + spyOn(websocketMsgSrvMock, 'renameNote'); + scope.updateNoteName(''); + expect(scope.note.name).toEqual(noteMock.name); + expect(websocketMsgSrvMock.renameNote).not.toHaveBeenCalled(); + scope.updateNoteName(' '); + expect(scope.note.name).toEqual(noteMock.name); + expect(websocketMsgSrvMock.renameNote).not.toHaveBeenCalled(); + scope.updateNoteName(scope.note.name); + expect(scope.note.name).toEqual(noteMock.name); + expect(websocketMsgSrvMock.renameNote).not.toHaveBeenCalled(); + }); + + it('should update note name when updateNoteName() is called with a valid name', function() { + spyOn(websocketMsgSrvMock, 'renameNote'); + let newName = 'Your Note'; + scope.updateNoteName(newName); + expect(scope.note.name).toEqual(newName); + expect(websocketMsgSrvMock.renameNote).toHaveBeenCalled(); + }); + + it('should reload note info once per one "setNoteMenu" event', function() { + spyOn(websocketMsgSrvMock, 'getNote'); + spyOn(websocketMsgSrvMock, 'listRevisionHistory'); + + scope.$broadcast('setNoteMenu'); + expect(websocketMsgSrvMock.getNote.calls.count()).toEqual(0); + expect(websocketMsgSrvMock.listRevisionHistory.calls.count()).toEqual(0); + + websocketMsgSrvMock.getNote.calls.reset(); + websocketMsgSrvMock.listRevisionHistory.calls.reset(); + + scope.$broadcast('setNoteMenu'); + expect(websocketMsgSrvMock.getNote.calls.count()).toEqual(0); + expect(websocketMsgSrvMock.listRevisionHistory.calls.count()).toEqual(0); + }); +}); diff --git a/zeppelin-web/src/app/notebook/paragraph/clipboard.controller.js b/zeppelin-web/src/app/notebook/paragraph/clipboard.controller.js index 0eb78e390c0..ea75b2751db 100644 --- a/zeppelin-web/src/app/notebook/paragraph/clipboard.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/clipboard.controller.js @@ -11,24 +11,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -angular.module('zeppelinWebApp').controller('clipboardCtrl', ClipboardController) +angular.module('zeppelinWebApp').controller('clipboardCtrl', ClipboardController); -function ClipboardController ($scope) { - 'ngInject' +function ClipboardController($scope) { + 'ngInject'; - $scope.complete = function (e) { - $scope.copied = true - $scope.tooltip = 'Copied!' - setTimeout(function () { - $scope.tooltip = 'Copy to clipboard' - }, 400) - } - $scope.$watch('input', function () { - $scope.copied = false - $scope.tooltip = 'Copy to clipboard' - }) - $scope.clipError = function (e) { - console.log('Error: ' + e.name + ' - ' + e.message) - $scope.tooltip = 'Not supported browser' - } + $scope.complete = function(e) { + $scope.copied = true; + $scope.tooltip = 'Copied!'; + setTimeout(function() { + $scope.tooltip = 'Copy to clipboard'; + }, 400); + }; + $scope.$watch('input', function() { + $scope.copied = false; + $scope.tooltip = 'Copy to clipboard'; + }); + $scope.clipError = function(e) { + console.log('Error: ' + e.name + ' - ' + e.message); + $scope.tooltip = 'Not supported browser'; + }; } diff --git a/zeppelin-web/src/app/notebook/paragraph/code-editor/code-editor.directive.js b/zeppelin-web/src/app/notebook/paragraph/code-editor/code-editor.directive.js index 944f05d2782..db4a98fc498 100644 --- a/zeppelin-web/src/app/notebook/paragraph/code-editor/code-editor.directive.js +++ b/zeppelin-web/src/app/notebook/paragraph/code-editor/code-editor.directive.js @@ -12,7 +12,7 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').directive('codeEditor', CodeEditorDirective) +angular.module('zeppelinWebApp').directive('codeEditor', CodeEditorDirective); function CodeEditorDirective($templateRequest, $compile) { return { @@ -23,16 +23,16 @@ function CodeEditorDirective($templateRequest, $compile) { dirtyText: '=dirtyText', originalText: '=originalText', onLoad: '=onLoad', - revisionView: '=revisionView' + revisionView: '=revisionView', }, - link: function (scope, element, attrs, controller) { - $templateRequest('app/notebook/paragraph/code-editor/code-editor.directive.html').then(function (editorHtml) { - let editor = angular.element(editorHtml) - editor.attr('id', scope.paragraphId + '_editor') - element.append(editor) - $compile(editor)(scope) - console.debug('codeEditor directive revision view is ' + scope.revisionView) - }) - } - } + link: function(scope, element, attrs, controller) { + $templateRequest('app/notebook/paragraph/code-editor/code-editor.directive.html').then(function(editorHtml) { + let editor = angular.element(editorHtml); + editor.attr('id', scope.paragraphId + '_editor'); + element.append(editor); + $compile(editor)(scope); + console.debug('codeEditor directive revision view is ' + scope.revisionView); + }); + }, + }; } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 07ebf896dd0..971257cf502 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -12,35 +12,35 @@ * limitations under the License. */ -import {SpellResult} from '../../spell' -import {isParagraphRunning, ParagraphStatus} from './paragraph.status' +import {SpellResult} from '../../spell'; +import {isParagraphRunning, ParagraphStatus} from './paragraph.status'; -import moment from 'moment' +import moment from 'moment'; -require('moment-duration-format') +require('moment-duration-format'); const ParagraphExecutor = { SPELL: 'SPELL', INTERPRETER: 'INTERPRETER', NONE: '', /** meaning `DONE` */ -} +}; -angular.module('zeppelinWebApp').controller('ParagraphCtrl', ParagraphCtrl) +angular.module('zeppelinWebApp').controller('ParagraphCtrl', ParagraphCtrl); -function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $location, +function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $location, $timeout, $compile, $http, $q, websocketMsgSrv, baseUrlSrv, ngToast, noteVarShareService, heliumService) { - 'ngInject' + 'ngInject'; - let ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_' - $rootScope.keys = Object.keys - $scope.parentNote = null - $scope.paragraph = {} - $scope.paragraph.results = {} - $scope.paragraph.results.msg = [] - $scope.originalText = '' - $scope.editor = null + let ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_'; + $rootScope.keys = Object.keys; + $scope.parentNote = null; + $scope.paragraph = {}; + $scope.paragraph.results = {}; + $scope.paragraph.results.msg = []; + $scope.originalText = ''; + $scope.editor = null; // transactional info for spell execution $scope.spellTransaction = { @@ -49,161 +49,161 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca propagated: false, resultsMsg: [], paragraphText: '', - } + }; - let searchRanges = [] + let searchRanges = []; const getCurrentRangeDefault = function() { - return {id: -1, markerId: -1} - } - let currentRange = getCurrentRangeDefault() + return {id: -1, markerId: -1}; + }; + let currentRange = getCurrentRangeDefault(); - let editorSetting = {} + let editorSetting = {}; // flag that is used to set editor setting on paste percent sign - let pastePercentSign = false + let pastePercentSign = false; // flag that is used to set editor setting on save interpreter bindings - let setInterpreterBindings = false - let paragraphScope = $rootScope.$new(true, $rootScope) + let setInterpreterBindings = false; + let paragraphScope = $rootScope.$new(true, $rootScope); // to keep backward compatibility - $scope.compiledScope = paragraphScope + $scope.compiledScope = paragraphScope; paragraphScope.z = { // z.runParagraph('20150213-231621_168813393') - runParagraph: function (paragraphId) { + runParagraph: function(paragraphId) { if (paragraphId) { - let filtered = $scope.parentNote.paragraphs.filter(function (x) { - return x.id === paragraphId - }) + let filtered = $scope.parentNote.paragraphs.filter(function(x) { + return x.id === paragraphId; + }); if (filtered.length === 1) { - let paragraph = filtered[0] + let paragraph = filtered[0]; websocketMsgSrv.runParagraph(paragraph.id, paragraph.title, paragraph.text, - paragraph.config, paragraph.settings.params) + paragraph.config, paragraph.settings.params); } else { ngToast.danger({ content: 'Cannot find a paragraph with id \'' + paragraphId + '\'', verticalPosition: 'top', - dismissOnTimeout: false - }) + dismissOnTimeout: false, + }); } } else { ngToast.danger({ content: 'Please provide a \'paragraphId\' when calling z.runParagraph(paragraphId)', verticalPosition: 'top', - dismissOnTimeout: false - }) + dismissOnTimeout: false, + }); } }, // Example: z.angularBind('my_var', 'Test Value', '20150213-231621_168813393') - angularBind: function (varName, value, paragraphId) { + angularBind: function(varName, value, paragraphId) { // Only push to server if there paragraphId is defined if (paragraphId) { - websocketMsgSrv.clientBindAngularObject($routeParams.noteId, varName, value, paragraphId) + websocketMsgSrv.clientBindAngularObject($routeParams.noteId, varName, value, paragraphId); } else { ngToast.danger({ content: 'Please provide a \'paragraphId\' when calling ' + 'z.angularBind(varName, value, \'PUT_HERE_PARAGRAPH_ID\')', verticalPosition: 'top', - dismissOnTimeout: false - }) + dismissOnTimeout: false, + }); } }, // Example: z.angularUnBind('my_var', '20150213-231621_168813393') - angularUnbind: function (varName, paragraphId) { + angularUnbind: function(varName, paragraphId) { // Only push to server if paragraphId is defined if (paragraphId) { - websocketMsgSrv.clientUnbindAngularObject($routeParams.noteId, varName, paragraphId) + websocketMsgSrv.clientUnbindAngularObject($routeParams.noteId, varName, paragraphId); } else { ngToast.danger({ content: 'Please provide a \'paragraphId\' when calling ' + 'z.angularUnbind(varName, \'PUT_HERE_PARAGRAPH_ID\')', verticalPosition: 'top', - dismissOnTimeout: false}) + dismissOnTimeout: false}); } - } - } + }, + }; - let angularObjectRegistry = {} + let angularObjectRegistry = {}; // Controller init - $scope.init = function (newParagraph, note) { - $scope.paragraph = newParagraph - $scope.parentNote = note - $scope.originalText = angular.copy(newParagraph.text) - $scope.chart = {} - $scope.baseMapOption = ['Streets', 'Satellite', 'Hybrid', 'Topo', 'Gray', 'Oceans', 'Terrain'] - $scope.colWidthOption = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] - $scope.fontSizeOption = [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] - $scope.paragraphFocused = false + $scope.init = function(newParagraph, note) { + $scope.paragraph = newParagraph; + $scope.parentNote = note; + $scope.originalText = angular.copy(newParagraph.text); + $scope.chart = {}; + $scope.baseMapOption = ['Streets', 'Satellite', 'Hybrid', 'Topo', 'Gray', 'Oceans', 'Terrain']; + $scope.colWidthOption = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + $scope.fontSizeOption = [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]; + $scope.paragraphFocused = false; if (newParagraph.focus) { - $scope.paragraphFocused = true + $scope.paragraphFocused = true; } if (!$scope.paragraph.config) { - $scope.paragraph.config = {} + $scope.paragraph.config = {}; } - noteVarShareService.put($scope.paragraph.id + '_paragraphScope', paragraphScope) + noteVarShareService.put($scope.paragraph.id + '_paragraphScope', paragraphScope); - initializeDefault($scope.paragraph.config) - } + initializeDefault($scope.paragraph.config); + }; - const initializeDefault = function (config) { - let forms = $scope.paragraph.settings.forms + const initializeDefault = function(config) { + let forms = $scope.paragraph.settings.forms; if (!config.colWidth) { - config.colWidth = 12 + config.colWidth = 12; } if (!config.fontSize) { - config.fontSize = 9 + config.fontSize = 9; } if (config.enabled === undefined) { - config.enabled = true + config.enabled = true; } for (let idx in forms) { if (forms[idx]) { if (forms[idx].options) { if (config.runOnSelectionChange === undefined) { - config.runOnSelectionChange = true + config.runOnSelectionChange = true; } } } } if (!config.results) { - config.results = {} + config.results = {}; } if (!config.editorSetting) { - config.editorSetting = {} + config.editorSetting = {}; } else if (config.editorSetting.editOnDblClick) { - editorSetting.isOutputHidden = config.editorSetting.editOnDblClick + editorSetting.isOutputHidden = config.editorSetting.editOnDblClick; } - } + }; const isTabCompletion = function() { - const completionKey = $scope.paragraph.config.editorSetting.completionKey - return completionKey === 'TAB' - } + const completionKey = $scope.paragraph.config.editorSetting.completionKey; + return completionKey === 'TAB'; + }; - $scope.$on('updateParagraphOutput', function (event, data) { + $scope.$on('updateParagraphOutput', function(event, data) { if ($scope.paragraph.id === data.paragraphId) { if (!$scope.paragraph.results) { - $scope.paragraph.results = {} + $scope.paragraph.results = {}; } if (!$scope.paragraph.results.msg) { - $scope.paragraph.results.msg = [] + $scope.paragraph.results.msg = []; } - let update = ($scope.paragraph.results.msg[data.index]) ? true : false + let update = ($scope.paragraph.results.msg[data.index]) ? true : false; $scope.paragraph.results.msg[data.index] = { data: data.data, - type: data.type - } + type: data.type, + }; if (update) { $rootScope.$broadcast( @@ -211,62 +211,62 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca $scope.paragraph.results.msg[data.index], $scope.paragraph.config.results[data.index], $scope.paragraph, - data.index) + data.index); } } - }) + }); - $scope.getIframeDimensions = function () { + $scope.getIframeDimensions = function() { if ($scope.asIframe) { - let paragraphid = '#' + $routeParams.paragraphId + '_container' - let height = angular.element(paragraphid).height() - return height + let paragraphid = '#' + $routeParams.paragraphId + '_container'; + let height = angular.element(paragraphid).height(); + return height; } - return 0 - } + return 0; + }; - $scope.$watch($scope.getIframeDimensions, function (newValue, oldValue) { + $scope.$watch($scope.getIframeDimensions, function(newValue, oldValue) { if ($scope.asIframe && newValue) { - let message = {} - message.height = newValue - message.url = $location.$$absUrl - $window.parent.postMessage(angular.toJson(message), '*') + let message = {}; + message.height = newValue; + message.url = $location.$$absUrl; + $window.parent.postMessage(angular.toJson(message), '*'); } - }) + }); - $scope.getEditor = function () { - return $scope.editor - } + $scope.getEditor = function() { + return $scope.editor; + }; - $scope.$watch($scope.getEditor, function (newValue, oldValue) { + $scope.$watch($scope.getEditor, function(newValue, oldValue) { if (!$scope.editor) { - return + return; } if (newValue === null || newValue === undefined) { - console.log('editor isnt loaded yet, returning') - return + console.log('editor isnt loaded yet, returning'); + return; } if ($scope.revisionView === true) { - $scope.editor.setReadOnly(true) + $scope.editor.setReadOnly(true); } else { - $scope.editor.setReadOnly(false) + $scope.editor.setReadOnly(false); } - }) + }); - let isEmpty = function (object) { - return !object - } + let isEmpty = function(object) { + return !object; + }; - $scope.isRunning = function (paragraph) { - return isParagraphRunning(paragraph) - } + $scope.isRunning = function(paragraph) { + return isParagraphRunning(paragraph); + }; - $scope.cancelParagraph = function (paragraph) { - console.log('Cancel %o', paragraph.id) - websocketMsgSrv.cancelParagraphRun(paragraph.id) - } + $scope.cancelParagraph = function(paragraph) { + console.log('Cancel %o', paragraph.id); + websocketMsgSrv.cancelParagraphRun(paragraph.id); + }; - $scope.propagateSpellResult = function (paragraphId, paragraphTitle, + $scope.propagateSpellResult = function(paragraphId, paragraphTitle, paragraphText, paragraphResults, paragraphStatus, paragraphErrorMessage, paragraphConfig, paragraphSettingsParam, @@ -277,18 +277,18 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca paragraphStatus, paragraphErrorMessage, paragraphConfig, paragraphSettingsParam, paragraphDateStarted, paragraphDateFinished - ) - } + ); + }; - $scope.handleSpellError = function (paragraphText, error, + $scope.handleSpellError = function(paragraphText, error, digestRequired, propagated) { - const errorMessage = error.stack - $scope.paragraph.status = ParagraphStatus.ERROR - $scope.paragraph.errorMessage = errorMessage - console.error('Failed to execute interpret() in spell\n', error) + const errorMessage = error.stack; + $scope.paragraph.status = ParagraphStatus.ERROR; + $scope.paragraph.errorMessage = errorMessage; + console.error('Failed to execute interpret() in spell\n', error); if (!propagated) { - $scope.paragraph.dateFinished = $scope.getFormattedParagraphTime() + $scope.paragraph.dateFinished = $scope.getFormattedParagraphTime(); } if (!propagated) { @@ -296,547 +296,551 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca $scope.paragraph.id, $scope.paragraph.title, paragraphText, [], $scope.paragraph.status, errorMessage, $scope.paragraph.config, $scope.paragraph.settings.params, - $scope.paragraph.dateStarted, $scope.paragraph.dateFinished) + $scope.paragraph.dateStarted, $scope.paragraph.dateFinished); } - } + }; - $scope.prepareSpellTransaction = function (resultsMsg, propagated, paragraphText) { - $scope.spellTransaction.totalResultCount = resultsMsg.length - $scope.spellTransaction.renderedResultCount = 0 - $scope.spellTransaction.propagated = propagated - $scope.spellTransaction.resultsMsg = resultsMsg - $scope.spellTransaction.paragraphText = paragraphText - } + $scope.prepareSpellTransaction = function(resultsMsg, propagated, paragraphText) { + $scope.spellTransaction.totalResultCount = resultsMsg.length; + $scope.spellTransaction.renderedResultCount = 0; + $scope.spellTransaction.propagated = propagated; + $scope.spellTransaction.resultsMsg = resultsMsg; + $scope.spellTransaction.paragraphText = paragraphText; + }; /** * - update spell transaction count and * - check transaction is finished based on the result count * @returns {boolean} */ - $scope.increaseSpellTransactionResultCount = function () { - $scope.spellTransaction.renderedResultCount += 1 + $scope.increaseSpellTransactionResultCount = function() { + $scope.spellTransaction.renderedResultCount += 1; - const total = $scope.spellTransaction.totalResultCount - const current = $scope.spellTransaction.renderedResultCount - return total === current - } + const total = $scope.spellTransaction.totalResultCount; + const current = $scope.spellTransaction.renderedResultCount; + return total === current; + }; - $scope.cleanupSpellTransaction = function () { - const status = ParagraphStatus.FINISHED - $scope.paragraph.executor = ParagraphExecutor.NONE - $scope.paragraph.status = status - $scope.paragraph.results.code = status + $scope.cleanupSpellTransaction = function() { + const status = ParagraphStatus.FINISHED; + $scope.paragraph.executor = ParagraphExecutor.NONE; + $scope.paragraph.status = status; + $scope.paragraph.results.code = status; - const propagated = $scope.spellTransaction.propagated - const resultsMsg = $scope.spellTransaction.resultsMsg - const paragraphText = $scope.spellTransaction.paragraphText + const propagated = $scope.spellTransaction.propagated; + const resultsMsg = $scope.spellTransaction.resultsMsg; + const paragraphText = $scope.spellTransaction.paragraphText; if (!propagated) { - $scope.paragraph.dateFinished = $scope.getFormattedParagraphTime() + $scope.paragraph.dateFinished = $scope.getFormattedParagraphTime(); } if (!propagated) { - const propagable = SpellResult.createPropagable(resultsMsg) + const propagable = SpellResult.createPropagable(resultsMsg); $scope.propagateSpellResult( $scope.paragraph.id, $scope.paragraph.title, paragraphText, propagable, status, '', $scope.paragraph.config, $scope.paragraph.settings.params, - $scope.paragraph.dateStarted, $scope.paragraph.dateFinished) + $scope.paragraph.dateStarted, $scope.paragraph.dateFinished); } - } + }; - $scope.runParagraphUsingSpell = function (paragraphText, + $scope.runParagraphUsingSpell = function(paragraphText, magic, digestRequired, propagated) { - $scope.paragraph.status = 'RUNNING' - $scope.paragraph.executor = ParagraphExecutor.SPELL - $scope.paragraph.results = {} - $scope.paragraph.errorMessage = '' - if (digestRequired) { $scope.$digest() } + $scope.paragraph.status = 'RUNNING'; + $scope.paragraph.executor = ParagraphExecutor.SPELL; + $scope.paragraph.results = {}; + $scope.paragraph.errorMessage = ''; + if (digestRequired) { + $scope.$digest(); + } try { // remove magic from paragraphText - const splited = paragraphText.split(magic) + const splited = paragraphText.split(magic); // remove leading spaces - const textWithoutMagic = splited[1].replace(/^\s+/g, '') + const textWithoutMagic = splited[1].replace(/^\s+/g, ''); if (!propagated) { - $scope.paragraph.dateStarted = $scope.getFormattedParagraphTime() + $scope.paragraph.dateStarted = $scope.getFormattedParagraphTime(); } // handle actual result message in promise heliumService.executeSpell(magic, textWithoutMagic) - .then(resultsMsg => { - $scope.prepareSpellTransaction(resultsMsg, propagated, paragraphText) + .then((resultsMsg) => { + $scope.prepareSpellTransaction(resultsMsg, propagated, paragraphText); - $scope.paragraph.results.msg = resultsMsg - $scope.paragraph.config.tableHide = false + $scope.paragraph.results.msg = resultsMsg; + $scope.paragraph.config.tableHide = false; }) - .catch(error => { + .catch((error) => { $scope.handleSpellError(paragraphText, error, - digestRequired, propagated) - }) + digestRequired, propagated); + }); } catch (error) { $scope.handleSpellError(paragraphText, error, - digestRequired, propagated) + digestRequired, propagated); } - } + }; - $scope.runParagraphUsingBackendInterpreter = function (paragraphText) { + $scope.runParagraphUsingBackendInterpreter = function(paragraphText) { websocketMsgSrv.runParagraph($scope.paragraph.id, $scope.paragraph.title, - paragraphText, $scope.paragraph.config, $scope.paragraph.settings.params) - } + paragraphText, $scope.paragraph.config, $scope.paragraph.settings.params); + }; - $scope.bindBeforeUnload = function () { - angular.element(window).off('beforeunload') + $scope.bindBeforeUnload = function() { + angular.element(window).off('beforeunload'); - let confirmOnPageExit = function (e) { + let confirmOnPageExit = function(e) { // If we haven't been passed the event get the window.event - e = e || window.event - let message = 'Do you want to reload this site?' + e = e || window.event; + let message = 'Do you want to reload this site?'; // For IE6-8 and Firefox prior to version 4 if (e) { - e.returnValue = message + e.returnValue = message; } // For Chrome, Safari, IE8+ and Opera 12+ - return message - } - angular.element(window).on('beforeunload', confirmOnPageExit) - } + return message; + }; + angular.element(window).on('beforeunload', confirmOnPageExit); + }; - $scope.unBindBeforeUnload = function () { - angular.element(window).off('beforeunload') - } + $scope.unBindBeforeUnload = function() { + angular.element(window).off('beforeunload'); + }; - $scope.saveParagraph = function (paragraph) { - const dirtyText = paragraph.text + $scope.saveParagraph = function(paragraph) { + const dirtyText = paragraph.text; if (dirtyText === undefined || dirtyText === $scope.originalText) { - return + return; } - $scope.bindBeforeUnload() + $scope.bindBeforeUnload(); - commitParagraph(paragraph).then(function () { - $scope.originalText = dirtyText - $scope.dirtyText = undefined - $scope.unBindBeforeUnload() - }) - } + commitParagraph(paragraph).then(function() { + $scope.originalText = dirtyText; + $scope.dirtyText = undefined; + $scope.unBindBeforeUnload(); + }); + }; - $scope.toggleEnableDisable = function (paragraph) { - paragraph.config.enabled = !paragraph.config.enabled - commitParagraph(paragraph) - } + $scope.toggleEnableDisable = function(paragraph) { + paragraph.config.enabled = !paragraph.config.enabled; + commitParagraph(paragraph); + }; /** * @param paragraphText to be parsed * @param digestRequired true if calling `$digest` is required * @param propagated true if update request is sent from other client */ - $scope.runParagraph = function (paragraphText, digestRequired, propagated) { + $scope.runParagraph = function(paragraphText, digestRequired, propagated) { if (!paragraphText || $scope.isRunning($scope.paragraph)) { - return + return; } - const magic = SpellResult.extractMagic(paragraphText) + const magic = SpellResult.extractMagic(paragraphText); if (heliumService.getSpellByMagic(magic)) { - $scope.runParagraphUsingSpell(paragraphText, magic, digestRequired, propagated) + $scope.runParagraphUsingSpell(paragraphText, magic, digestRequired, propagated); } else { - $scope.runParagraphUsingBackendInterpreter(paragraphText) + $scope.runParagraphUsingBackendInterpreter(paragraphText); } - $scope.originalText = angular.copy(paragraphText) - $scope.dirtyText = undefined + $scope.originalText = angular.copy(paragraphText); + $scope.dirtyText = undefined; if ($scope.paragraph.config.editorSetting.editOnDblClick) { - closeEditorAndOpenTable($scope.paragraph) + closeEditorAndOpenTable($scope.paragraph); } else if (editorSetting.isOutputHidden && !$scope.paragraph.config.editorSetting.editOnDblClick) { // %md/%angular repl make output to be hidden by default after running // so should open output if repl changed from %md/%angular to another - openEditorAndOpenTable($scope.paragraph) + openEditorAndOpenTable($scope.paragraph); } - editorSetting.isOutputHidden = $scope.paragraph.config.editorSetting.editOnDblClick - } + editorSetting.isOutputHidden = $scope.paragraph.config.editorSetting.editOnDblClick; + }; - $scope.runParagraphFromShortcut = function (paragraphText) { + $scope.runParagraphFromShortcut = function(paragraphText) { // passing `digestRequired` as true to update view immediately // without this, results cannot be rendered in view more than once - $scope.runParagraph(paragraphText, true, false) - } + $scope.runParagraph(paragraphText, true, false); + }; - $scope.runParagraphFromButton = function () { + $scope.runParagraphFromButton = function() { // we come here from the view, so we don't need to call `$digest()` - $scope.runParagraph($scope.getEditorValue(), false, false) - } + $scope.runParagraph($scope.getEditorValue(), false, false); + }; $scope.runAllToThis = function(paragraph) { - $scope.$emit('runAllAbove', paragraph, true) - } + $scope.$emit('runAllAbove', paragraph, true); + }; $scope.runAllFromThis = function(paragraph) { - $scope.$emit('runAllBelowAndCurrent', paragraph, true) - } + $scope.$emit('runAllBelowAndCurrent', paragraph, true); + }; - $scope.runAllToOrFromThis = function (paragraph) { + $scope.runAllToOrFromThis = function(paragraph) { BootstrapDialog.show({ message: 'Run paragraphs:', title: '', buttons: [{ label: 'Close', action: function(dialog) { - dialog.close() - } + dialog.close(); + }, }, { label: 'Run all above', cssClass: 'btn-primary', action: function(dialog) { - $scope.$emit('runAllAbove', paragraph, false) - dialog.close() - } + $scope.$emit('runAllAbove', paragraph, false); + dialog.close(); + }, }, { label: 'Run current and all below', cssClass: 'btn-primary', action: function(dialog) { - $scope.$emit('runAllBelowAndCurrent', paragraph, false) - dialog.close() - } - }] - }) - } + $scope.$emit('runAllBelowAndCurrent', paragraph, false); + dialog.close(); + }, + }], + }); + }; - $scope.turnOnAutoRun = function (paragraph) { - paragraph.config.runOnSelectionChange = !paragraph.config.runOnSelectionChange - commitParagraph(paragraph) - } + $scope.turnOnAutoRun = function(paragraph) { + paragraph.config.runOnSelectionChange = !paragraph.config.runOnSelectionChange; + commitParagraph(paragraph); + }; - $scope.moveUp = function (paragraph) { - $scope.$emit('moveParagraphUp', paragraph) - } + $scope.moveUp = function(paragraph) { + $scope.$emit('moveParagraphUp', paragraph); + }; - $scope.moveDown = function (paragraph) { - $scope.$emit('moveParagraphDown', paragraph) - } + $scope.moveDown = function(paragraph) { + $scope.$emit('moveParagraphDown', paragraph); + }; - $scope.insertNew = function (position) { - $scope.$emit('insertParagraph', $scope.paragraph.id, position) - } + $scope.insertNew = function(position) { + $scope.$emit('insertParagraph', $scope.paragraph.id, position); + }; - $scope.copyPara = function (position) { - let editorValue = $scope.getEditorValue() + $scope.copyPara = function(position) { + let editorValue = $scope.getEditorValue(); if (editorValue) { - $scope.copyParagraph(editorValue, position) + $scope.copyParagraph(editorValue, position); } - } + }; - $scope.copyParagraph = function (data, position) { - let newIndex = -1 + $scope.copyParagraph = function(data, position) { + let newIndex = -1; for (let i = 0; i < $scope.note.paragraphs.length; i++) { if ($scope.note.paragraphs[i].id === $scope.paragraph.id) { // determine position of where to add new paragraph; default is below if (position === 'above') { - newIndex = i + newIndex = i; } else { - newIndex = i + 1 + newIndex = i + 1; } - break + break; } } if (newIndex < 0 || newIndex > $scope.note.paragraphs.length) { - return + return; } - let config = angular.copy($scope.paragraph.config) - config.editorHide = false + let config = angular.copy($scope.paragraph.config); + config.editorHide = false; websocketMsgSrv.copyParagraph(newIndex, $scope.paragraph.title, data, - config, $scope.paragraph.settings.params) - } + config, $scope.paragraph.settings.params); + }; - $scope.removeParagraph = function (paragraph) { + $scope.removeParagraph = function(paragraph) { if ($scope.note.paragraphs.length === 1) { BootstrapDialog.alert({ closable: true, - message: 'All the paragraphs can\'t be deleted.' - }) + message: 'All the paragraphs can\'t be deleted.', + }); } else { BootstrapDialog.confirm({ closable: true, title: '', message: 'Do you want to delete this paragraph?', - callback: function (result) { + callback: function(result) { if (result) { - console.log('Remove paragraph') - websocketMsgSrv.removeParagraph(paragraph.id) - $scope.$emit('moveFocusToNextParagraph', $scope.paragraph.id) + console.log('Remove paragraph'); + websocketMsgSrv.removeParagraph(paragraph.id); + $scope.$emit('moveFocusToNextParagraph', $scope.paragraph.id); } - } - }) + }, + }); } - } + }; - $scope.clearParagraphOutput = function (paragraph) { - websocketMsgSrv.clearParagraphOutput(paragraph.id) - } + $scope.clearParagraphOutput = function(paragraph) { + websocketMsgSrv.clearParagraphOutput(paragraph.id); + }; - $scope.toggleEditor = function (paragraph) { + $scope.toggleEditor = function(paragraph) { if (paragraph.config.editorHide) { - $scope.openEditor(paragraph) + $scope.openEditor(paragraph); } else { - $scope.closeEditor(paragraph) - } - } - - $scope.closeEditor = function (paragraph) { - console.log('close the note') - paragraph.config.editorHide = true - commitParagraph(paragraph) - } - - $scope.openEditor = function (paragraph) { - console.log('open the note') - paragraph.config.editorHide = false - commitParagraph(paragraph) - } - - $scope.closeTable = function (paragraph) { - console.log('close the output') - paragraph.config.tableHide = true - commitParagraph(paragraph) - } - - $scope.openTable = function (paragraph) { - console.log('open the output') - paragraph.config.tableHide = false - commitParagraph(paragraph) - } - - let openEditorAndCloseTable = function (paragraph) { - manageEditorAndTableState(paragraph, false, true) - } - - const closeEditorAndOpenTable = function (paragraph) { - manageEditorAndTableState(paragraph, true, false) - } - - const openEditorAndOpenTable = function (paragraph) { - manageEditorAndTableState(paragraph, false, false) - } - - const manageEditorAndTableState = function (paragraph, hideEditor, hideTable) { - paragraph.config.editorHide = hideEditor - paragraph.config.tableHide = hideTable - commitParagraph(paragraph) - } - - $scope.showTitle = function (paragraph) { - paragraph.config.title = true - commitParagraph(paragraph) - } - - $scope.hideTitle = function (paragraph) { - paragraph.config.title = false - commitParagraph(paragraph) - } - - $scope.setTitle = function (paragraph) { - commitParagraph(paragraph) - } - - $scope.showLineNumbers = function (paragraph) { + $scope.closeEditor(paragraph); + } + }; + + $scope.closeEditor = function(paragraph) { + console.log('close the note'); + paragraph.config.editorHide = true; + commitParagraph(paragraph); + }; + + $scope.openEditor = function(paragraph) { + console.log('open the note'); + paragraph.config.editorHide = false; + commitParagraph(paragraph); + }; + + $scope.closeTable = function(paragraph) { + console.log('close the output'); + paragraph.config.tableHide = true; + commitParagraph(paragraph); + }; + + $scope.openTable = function(paragraph) { + console.log('open the output'); + paragraph.config.tableHide = false; + commitParagraph(paragraph); + }; + + let openEditorAndCloseTable = function(paragraph) { + manageEditorAndTableState(paragraph, false, true); + }; + + const closeEditorAndOpenTable = function(paragraph) { + manageEditorAndTableState(paragraph, true, false); + }; + + const openEditorAndOpenTable = function(paragraph) { + manageEditorAndTableState(paragraph, false, false); + }; + + const manageEditorAndTableState = function(paragraph, hideEditor, hideTable) { + paragraph.config.editorHide = hideEditor; + paragraph.config.tableHide = hideTable; + commitParagraph(paragraph); + }; + + $scope.showTitle = function(paragraph) { + paragraph.config.title = true; + commitParagraph(paragraph); + }; + + $scope.hideTitle = function(paragraph) { + paragraph.config.title = false; + commitParagraph(paragraph); + }; + + $scope.setTitle = function(paragraph) { + commitParagraph(paragraph); + }; + + $scope.showLineNumbers = function(paragraph) { if ($scope.editor) { - paragraph.config.lineNumbers = true - $scope.editor.renderer.setShowGutter(true) - commitParagraph(paragraph) + paragraph.config.lineNumbers = true; + $scope.editor.renderer.setShowGutter(true); + commitParagraph(paragraph); } - } + }; - $scope.hideLineNumbers = function (paragraph) { + $scope.hideLineNumbers = function(paragraph) { if ($scope.editor) { - paragraph.config.lineNumbers = false - $scope.editor.renderer.setShowGutter(false) - commitParagraph(paragraph) + paragraph.config.lineNumbers = false; + $scope.editor.renderer.setShowGutter(false); + commitParagraph(paragraph); } - } + }; - $scope.columnWidthClass = function (n) { + $scope.columnWidthClass = function(n) { if ($scope.asIframe) { - return 'col-md-12' + return 'col-md-12'; } else { - return 'paragraph-col col-md-' + n + return 'paragraph-col col-md-' + n; } - } + }; - $scope.changeColWidth = function (paragraph, width) { - angular.element('.navbar-right.open').removeClass('open') - paragraph.config.colWidth = width - $scope.$broadcast('paragraphResized', $scope.paragraph.id) - commitParagraph(paragraph) - } + $scope.changeColWidth = function(paragraph, width) { + angular.element('.navbar-right.open').removeClass('open'); + paragraph.config.colWidth = width; + $scope.$broadcast('paragraphResized', $scope.paragraph.id); + commitParagraph(paragraph); + }; - $scope.changeFontSize = function (paragraph, fontSize) { - angular.element('.navbar-right.open').removeClass('open') + $scope.changeFontSize = function(paragraph, fontSize) { + angular.element('.navbar-right.open').removeClass('open'); if ($scope.editor) { $scope.editor.setOptions({ - fontSize: fontSize + 'pt' - }) - autoAdjustEditorHeight($scope.editor) - paragraph.config.fontSize = fontSize - commitParagraph(paragraph) - } - } - - $scope.toggleOutput = function (paragraph) { - paragraph.config.tableHide = !paragraph.config.tableHide - commitParagraph(paragraph) - } - - $scope.aceChanged = function (_, editor) { - let session = editor.getSession() - let dirtyText = session.getValue() - $scope.dirtyText = dirtyText + fontSize: fontSize + 'pt', + }); + autoAdjustEditorHeight($scope.editor); + paragraph.config.fontSize = fontSize; + commitParagraph(paragraph); + } + }; + + $scope.toggleOutput = function(paragraph) { + paragraph.config.tableHide = !paragraph.config.tableHide; + commitParagraph(paragraph); + }; + + $scope.aceChanged = function(_, editor) { + let session = editor.getSession(); + let dirtyText = session.getValue(); + $scope.dirtyText = dirtyText; if ($scope.dirtyText !== $scope.originalText) { - $scope.startSaveTimer() + $scope.startSaveTimer(); } - setParagraphMode(session, dirtyText, editor.getCursorPosition()) - } + setParagraphMode(session, dirtyText, editor.getCursorPosition()); + }; - $scope.aceLoaded = function (_editor) { - let langTools = ace.require('ace/ext/language_tools') - let Range = ace.require('ace/range').Range + $scope.aceLoaded = function(_editor) { + let langTools = ace.require('ace/ext/language_tools'); + let Range = ace.require('ace/range').Range; - _editor.$blockScrolling = Infinity - $scope.editor = _editor - $scope.editor.on('input', $scope.aceChanged) + _editor.$blockScrolling = Infinity; + $scope.editor = _editor; + $scope.editor.on('input', $scope.aceChanged); if (_editor.container.id !== '{{paragraph.id}}_editor') { - $scope.editor.renderer.setShowGutter($scope.paragraph.config.lineNumbers) - $scope.editor.setShowFoldWidgets(false) - $scope.editor.setHighlightActiveLine(false) - $scope.editor.getSession().setUseWrapMode(true) - $scope.editor.setTheme('ace/theme/chrome') - $scope.editor.setReadOnly($scope.isRunning($scope.paragraph)) - $scope.editor.setHighlightActiveLine($scope.paragraphFocused) + $scope.editor.renderer.setShowGutter($scope.paragraph.config.lineNumbers); + $scope.editor.setShowFoldWidgets(false); + $scope.editor.setHighlightActiveLine(false); + $scope.editor.getSession().setUseWrapMode(true); + $scope.editor.setTheme('ace/theme/chrome'); + $scope.editor.setReadOnly($scope.isRunning($scope.paragraph)); + $scope.editor.setHighlightActiveLine($scope.paragraphFocused); if ($scope.paragraphFocused) { - let prefix = '%' + getInterpreterName($scope.paragraph.text) - let paragraphText = $scope.paragraph.text ? $scope.paragraph.text.trim() : '' + let prefix = '%' + getInterpreterName($scope.paragraph.text); + let paragraphText = $scope.paragraph.text ? $scope.paragraph.text.trim() : ''; - $scope.editor.focus() - $scope.goToEnd($scope.editor) + $scope.editor.focus(); + $scope.goToEnd($scope.editor); if (prefix === paragraphText) { - $timeout(function () { - $scope.editor.gotoLine(2, 0) - }, 0) + $timeout(function() { + $scope.editor.gotoLine(2, 0); + }, 0); } } - autoAdjustEditorHeight(_editor) - angular.element(window).resize(function () { - autoAdjustEditorHeight(_editor) - }) + autoAdjustEditorHeight(_editor); + angular.element(window).resize(function() { + autoAdjustEditorHeight(_editor); + }); if (navigator.appVersion.indexOf('Mac') !== -1) { - $scope.editor.setKeyboardHandler('ace/keyboard/emacs') - $rootScope.isMac = true + $scope.editor.setKeyboardHandler('ace/keyboard/emacs'); + $rootScope.isMac = true; } else if (navigator.appVersion.indexOf('Win') !== -1 || navigator.appVersion.indexOf('X11') !== -1 || navigator.appVersion.indexOf('Linux') !== -1) { - $rootScope.isMac = false + $rootScope.isMac = false; // not applying emacs key binding while the binding override Ctrl-v. default behavior of paste text on windows. } let remoteCompleter = { getCompletions: function(editor, session, pos, prefix, callback) { - let langTools = ace.require('ace/ext/language_tools') - let defaultKeywords = new Set() + let langTools = ace.require('ace/ext/language_tools'); + let defaultKeywords = new Set(); // eslint-disable-next-line handle-callback-err let getDefaultKeywords = function(err, completions) { if (completions !== undefined) { completions.forEach(function(c) { - defaultKeywords.add(c.value) - }) + defaultKeywords.add(c.value); + }); } - } + }; if (langTools.keyWordCompleter !== undefined) { - langTools.keyWordCompleter.getCompletions(editor, session, pos, prefix, getDefaultKeywords) + langTools.keyWordCompleter.getCompletions(editor, session, pos, prefix, getDefaultKeywords); } if (!editor.isFocused()) { - return + return; } - pos = session.getTextRange(new Range(0, 0, pos.row, pos.column)).length - let buf = session.getValue() + pos = session.getTextRange(new Range(0, 0, pos.row, pos.column)).length; + let buf = session.getValue(); - websocketMsgSrv.completion($scope.paragraph.id, buf, pos) + websocketMsgSrv.completion($scope.paragraph.id, buf, pos); $scope.$on('completionList', function(event, data) { let computeCaption = function(value, meta) { - let metaLength = meta !== undefined ? meta.length : 0 - let length = 42 - let whitespaceLength = 3 - let ellipses = '...' - let maxLengthCaption = length - metaLength - whitespaceLength - ellipses.length + let metaLength = meta !== undefined ? meta.length : 0; + let length = 42; + let whitespaceLength = 3; + let ellipses = '...'; + let maxLengthCaption = length - metaLength - whitespaceLength - ellipses.length; if (value !== undefined && value.length > maxLengthCaption) { - return value.substr(0, maxLengthCaption) + ellipses + return value.substr(0, maxLengthCaption) + ellipses; } - return value - } + return value; + }; if (data.completions) { - let completions = [] + let completions = []; for (let c in data.completions) { - let v = data.completions[c] - if (v.meta !== undefined && v.meta === 'keyword' && defaultKeywords.has(v.value.trim())) { - continue + if (data.completions.hasOwnProperty(c)) { + let v = data.completions[c]; + if (v.meta !== undefined && v.meta === 'keyword' && defaultKeywords.has(v.value.trim())) { + continue; + } + completions.push({ + name: v.name, + value: v.value, + meta: v.meta, + caption: computeCaption(v.value, v.meta), + score: 300, + }); } - completions.push({ - name: v.name, - value: v.value, - meta: v.meta, - caption: computeCaption(v.value, v.meta), - score: 300 - }) } - callback(null, completions) + callback(null, completions); } - }) - } - } + }); + }, + }; langTools.setCompleters([remoteCompleter, langTools.keyWordCompleter, langTools.snippetCompleter, - langTools.textCompleter]) + langTools.textCompleter]); $scope.editor.setOptions({ fontSize: $scope.paragraph.config.fontSize + 'pt', enableBasicAutocompletion: true, enableSnippets: false, - enableLiveAutocompletion: false - }) + enableLiveAutocompletion: false, + }); - $scope.editor.on('focus', function () { - handleFocus(true) - }) + $scope.editor.on('focus', function() { + handleFocus(true); + }); - $scope.editor.on('blur', function () { - handleFocus(false) - $scope.saveParagraph($scope.paragraph) - }) + $scope.editor.on('blur', function() { + handleFocus(false); + $scope.saveParagraph($scope.paragraph); + }); - $scope.editor.on('paste', function (e) { + $scope.editor.on('paste', function(e) { if (e.text.indexOf('%') === 0) { - pastePercentSign = true + pastePercentSign = true; } - }) + }); - $scope.editor.getSession().on('change', function (e, editSession) { - autoAdjustEditorHeight(_editor) - }) + $scope.editor.getSession().on('change', function(e, editSession) { + autoAdjustEditorHeight(_editor); + }); - setParagraphMode($scope.editor.getSession(), $scope.editor.getSession().getValue()) + setParagraphMode($scope.editor.getSession(), $scope.editor.getSession().getValue()); // autocomplete on '.' /* @@ -851,29 +855,29 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca */ // remove binding - $scope.editor.commands.removeCommand('showSettingsMenu') - $scope.editor.commands.removeCommand('find') - $scope.editor.commands.removeCommand('replace') + $scope.editor.commands.removeCommand('showSettingsMenu'); + $scope.editor.commands.removeCommand('find'); + $scope.editor.commands.removeCommand('replace'); - let isOption = $rootScope.isMac ? 'option' : 'alt' + let isOption = $rootScope.isMac ? 'option' : 'alt'; - $scope.editor.commands.bindKey('ctrl-' + isOption + '-n.', null) - $scope.editor.commands.bindKey('ctrl-' + isOption + '-l', null) - $scope.editor.commands.bindKey('ctrl-' + isOption + '-w', null) - $scope.editor.commands.bindKey('ctrl-' + isOption + '-a', null) - $scope.editor.commands.bindKey('ctrl-' + isOption + '-k', null) - $scope.editor.commands.bindKey('ctrl-' + isOption + '-e', null) - $scope.editor.commands.bindKey('ctrl-' + isOption + '-t', null) - $scope.editor.commands.bindKey('ctrl-space', null) + $scope.editor.commands.bindKey('ctrl-' + isOption + '-n.', null); + $scope.editor.commands.bindKey('ctrl-' + isOption + '-l', null); + $scope.editor.commands.bindKey('ctrl-' + isOption + '-w', null); + $scope.editor.commands.bindKey('ctrl-' + isOption + '-a', null); + $scope.editor.commands.bindKey('ctrl-' + isOption + '-k', null); + $scope.editor.commands.bindKey('ctrl-' + isOption + '-e', null); + $scope.editor.commands.bindKey('ctrl-' + isOption + '-t', null); + $scope.editor.commands.bindKey('ctrl-space', null); if ($rootScope.isMac) { - $scope.editor.commands.bindKey('command-l', null) + $scope.editor.commands.bindKey('command-l', null); } else { - $scope.editor.commands.bindKey('ctrl-l', null) + $scope.editor.commands.bindKey('ctrl-l', null); } // autocomplete on 'ctrl+.' - $scope.editor.commands.bindKey('ctrl-.', 'startAutocomplete') + $scope.editor.commands.bindKey('ctrl-.', 'startAutocomplete'); // Show autocomplete on tab $scope.editor.commands.addCommand({ @@ -881,113 +885,123 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca bindKey: { win: 'tab', mac: 'tab', - sender: 'editor|cli' + sender: 'editor|cli', }, exec: function(env, args, request) { - let iCursor = $scope.editor.getCursorPosition() - let currentLine = $scope.editor.session.getLine(iCursor.row) + let iCursor = $scope.editor.getCursorPosition(); + let currentLine = $scope.editor.session.getLine(iCursor.row); let isAllTabs = currentLine.substring(0, iCursor.column - 1).split('').every(function(char) { - return (char === '\t' || char === ' ') - }) + return (char === '\t' || char === ' '); + }); // If user has pressed tab on first line char or if isTabCompletion() is false, keep existing behavior // If user has pressed tab anywhere in between and editor mode is not %md, show autocomplete if (!isAllTabs && iCursor.column && isTabCompletion()) { - $scope.editor.execCommand('startAutocomplete') + $scope.editor.execCommand('startAutocomplete'); } else { - ace.config.loadModule('ace/ext/language_tools', function () { - $scope.editor.insertSnippet('\t') - }) + ace.config.loadModule('ace/ext/language_tools', function() { + $scope.editor.insertSnippet('\t'); + }); } - } - }) + }, + }); - let keyBindingEditorFocusAction = function (scrollValue) { - let numRows = $scope.editor.getSession().getLength() - let currentRow = $scope.editor.getCursorPosition().row + let keyBindingEditorFocusAction = function(scrollValue) { + let numRows = $scope.editor.getSession().getLength(); + let currentRow = $scope.editor.getCursorPosition().row; if (currentRow === 0 && scrollValue <= 0) { // move focus to previous paragraph - $scope.$emit('moveFocusToPreviousParagraph', $scope.paragraph.id) + $scope.$emit('moveFocusToPreviousParagraph', $scope.paragraph.id); } else if (currentRow === numRows - 1 && scrollValue >= 0) { - $scope.$emit('moveFocusToNextParagraph', $scope.paragraph.id) + $scope.$emit('moveFocusToNextParagraph', $scope.paragraph.id); } else { - $scope.scrollToCursor($scope.paragraph.id, scrollValue) + $scope.scrollToCursor($scope.paragraph.id, scrollValue); } - } + }; // handle cursor moves - $scope.editor.keyBinding.origOnCommandKey = $scope.editor.keyBinding.onCommandKey - $scope.editor.keyBinding.onCommandKey = function (e, hashId, keyCode) { + $scope.editor.keyBinding.origOnCommandKey = $scope.editor.keyBinding.onCommandKey; + $scope.editor.keyBinding.onCommandKey = function(e, hashId, keyCode) { if ($scope.editor.completer && $scope.editor.completer.activated) { // if autocompleter is active } else { // fix ace editor focus issue in chrome (textarea element goes to top: -1000px after focused by cursor move) if (parseInt(angular.element('#' + $scope.paragraph.id + '_editor > textarea') .css('top').replace('px', '')) < 0) { - let position = $scope.editor.getCursorPosition() - let cursorPos = $scope.editor.renderer.$cursorLayer.getPixelPosition(position, true) - angular.element('#' + $scope.paragraph.id + '_editor > textarea').css('top', cursorPos.top) + let position = $scope.editor.getCursorPosition(); + let cursorPos = $scope.editor.renderer.$cursorLayer.getPixelPosition(position, true); + angular.element('#' + $scope.paragraph.id + '_editor > textarea').css('top', cursorPos.top); } - let ROW_UP = -1 - let ROW_DOWN = 1 + let ROW_UP = -1; + let ROW_DOWN = 1; switch (keyCode) { case 38: - if (!e.shiftKey) { keyBindingEditorFocusAction(ROW_UP) } - break + if (!e.shiftKey) { + keyBindingEditorFocusAction(ROW_UP); + } + break; case 80: - if (e.ctrlKey && !e.altKey) { keyBindingEditorFocusAction(ROW_UP) } - break + if (e.ctrlKey && !e.altKey) { + keyBindingEditorFocusAction(ROW_UP); + } + break; case 40: - if (!e.shiftKey) { keyBindingEditorFocusAction(ROW_DOWN) } - break + if (!e.shiftKey) { + keyBindingEditorFocusAction(ROW_DOWN); + } + break; case 78: - if (e.ctrlKey && !e.altKey) { keyBindingEditorFocusAction(ROW_DOWN) } - break + if (e.ctrlKey && !e.altKey) { + keyBindingEditorFocusAction(ROW_DOWN); + } + break; } } - this.origOnCommandKey(e, hashId, keyCode) - } + this.origOnCommandKey(e, hashId, keyCode); + }; } - } + }; - const handleFocus = function (focused, isDigestPass) { - $scope.paragraphFocused = focused + const handleFocus = function(focused, isDigestPass) { + $scope.paragraphFocused = focused; - if ($scope.editor) { $scope.editor.setHighlightActiveLine(focused) } + if ($scope.editor) { + $scope.editor.setHighlightActiveLine(focused); + } if (isDigestPass === false || isDigestPass === undefined) { // Protect against error in case digest is already running - $timeout(function () { + $timeout(function() { // Apply changes since they come from 3rd party library - $scope.$digest() - }) + $scope.$digest(); + }); } - } + }; - let getEditorSetting = function (paragraph, interpreterName) { - let deferred = $q.defer() + let getEditorSetting = function(paragraph, interpreterName) { + let deferred = $q.defer(); if (!$scope.revisionView) { - websocketMsgSrv.getEditorSetting(paragraph.id, interpreterName) + websocketMsgSrv.getEditorSetting(paragraph.id, interpreterName); $timeout( - $scope.$on('editorSetting', function (event, data) { + $scope.$on('editorSetting', function(event, data) { if (paragraph.id === data.paragraphId) { - deferred.resolve(data) + deferred.resolve(data); } } - ), 1000) + ), 1000); } - return deferred.promise - } + return deferred.promise; + }; - let setEditorLanguage = function (session, language) { - let mode = 'ace/mode/' - mode += language - $scope.paragraph.config.editorMode = mode - session.setMode(mode) - } + let setEditorLanguage = function(session, language) { + let mode = 'ace/mode/'; + mode += language; + $scope.paragraph.config.editorMode = mode; + session.setMode(mode); + }; - const setParagraphMode = function (session, paragraphText, pos) { + const setParagraphMode = function(session, paragraphText, pos) { // Evaluate the mode only if the the position is undefined // or the first 30 characters of the paragraph have been modified // or cursor position is at beginning of second line.(in case user hit enter after typing %magic) @@ -996,319 +1010,321 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca // If paragraph loading, use config value if exists if ((typeof pos === 'undefined') && $scope.paragraph.config.editorMode && !setInterpreterBindings) { - session.setMode($scope.paragraph.config.editorMode) + session.setMode($scope.paragraph.config.editorMode); } else { - let magic = getInterpreterName(paragraphText) + let magic = getInterpreterName(paragraphText); if (editorSetting.magic !== magic) { - editorSetting.magic = magic + editorSetting.magic = magic; getEditorSetting($scope.paragraph, magic) - .then(function (setting) { - setEditorLanguage(session, setting.editor.language) - _.merge($scope.paragraph.config.editorSetting, setting.editor) - }) + .then(function(setting) { + setEditorLanguage(session, setting.editor.language); + _.merge($scope.paragraph.config.editorSetting, setting.editor); + }); } } } - pastePercentSign = false - setInterpreterBindings = false - } + pastePercentSign = false; + setInterpreterBindings = false; + }; const getInterpreterName = function(paragraphText) { - let intpNameRegexp = /^\s*%(.+?)(\s|\()/g - let match = intpNameRegexp.exec(paragraphText) + let intpNameRegexp = /^\s*%(.+?)(\s|\()/g; + let match = intpNameRegexp.exec(paragraphText); if (match) { - return match[1].trim() + return match[1].trim(); // get default interpreter name if paragraph text doesn't start with '%' // TODO(mina): dig into the cause what makes interpreterBindings to have no element } else if ($scope.$parent.interpreterBindings && $scope.$parent.interpreterBindings.length !== 0) { - return $scope.$parent.interpreterBindings[0].name + return $scope.$parent.interpreterBindings[0].name; } - return '' - } + return ''; + }; - const autoAdjustEditorHeight = function (editor) { + const autoAdjustEditorHeight = function(editor) { let height = editor.getSession().getScreenLength() * - editor.renderer.lineHeight + editor.renderer.lineHeight; - angular.element('#' + editor.container.id).height(height.toString() + 'px') - editor.resize() - } + angular.element('#' + editor.container.id).height(height.toString() + 'px'); + editor.resize(); + }; - $rootScope.$on('scrollToCursor', function (event) { + $rootScope.$on('scrollToCursor', function(event) { // scroll on 'scrollToCursor' event only when cursor is in the last paragraph - let paragraphs = angular.element('div[id$="_paragraphColumn_main"]') + let paragraphs = angular.element('div[id$="_paragraphColumn_main"]'); if (paragraphs[paragraphs.length - 1].id.indexOf($scope.paragraph.id) === 0) { - $scope.scrollToCursor($scope.paragraph.id, 0) + $scope.scrollToCursor($scope.paragraph.id, 0); } - }) + }); /** scrollToCursor if it is necessary * when cursor touches scrollTriggerEdgeMargin from the top (or bottom) of the screen, it autoscroll to place cursor around 1/3 of screen height from the top (or bottom) * paragraphId : paragraph that has active cursor * lastCursorMove : 1(down), 0, -1(up) last cursor move event **/ - $scope.scrollToCursor = function (paragraphId, lastCursorMove) { + $scope.scrollToCursor = function(paragraphId, lastCursorMove) { if (!$scope.editor || !$scope.editor.isFocused()) { // only make sense when editor is focused - return + return; } - let lineHeight = $scope.editor.renderer.lineHeight - let headerHeight = 103 // menubar, notebook titlebar - let scrollTriggerEdgeMargin = 50 + let lineHeight = $scope.editor.renderer.lineHeight; + let headerHeight = 103; // menubar, notebook titlebar + let scrollTriggerEdgeMargin = 50; - let documentHeight = angular.element(document).height() - let windowHeight = angular.element(window).height() // actual viewport height + let documentHeight = angular.element(document).height(); + let windowHeight = angular.element(window).height(); // actual viewport height - let scrollPosition = angular.element(document).scrollTop() - let editorPosition = angular.element('#' + paragraphId + '_editor').offset() - let position = $scope.editor.getCursorPosition() - let lastCursorPosition = $scope.editor.renderer.$cursorLayer.getPixelPosition(position, true) + let scrollPosition = angular.element(document).scrollTop(); + let editorPosition = angular.element('#' + paragraphId + '_editor').offset(); + let position = $scope.editor.getCursorPosition(); + let lastCursorPosition = $scope.editor.renderer.$cursorLayer.getPixelPosition(position, true); - let calculatedCursorPosition = editorPosition.top + lastCursorPosition.top + lineHeight * lastCursorMove + let calculatedCursorPosition = editorPosition.top + lastCursorPosition.top + lineHeight * lastCursorMove; - let scrollTargetPos + let scrollTargetPos; if (calculatedCursorPosition < scrollPosition + headerHeight + scrollTriggerEdgeMargin) { - scrollTargetPos = calculatedCursorPosition - headerHeight - ((windowHeight - headerHeight) / 3) + scrollTargetPos = calculatedCursorPosition - headerHeight - ((windowHeight - headerHeight) / 3); if (scrollTargetPos < 0) { - scrollTargetPos = 0 + scrollTargetPos = 0; } } else if (calculatedCursorPosition > scrollPosition + scrollTriggerEdgeMargin + windowHeight - headerHeight) { - scrollTargetPos = calculatedCursorPosition - headerHeight - ((windowHeight - headerHeight) * 2 / 3) + scrollTargetPos = calculatedCursorPosition - headerHeight - ((windowHeight - headerHeight) * 2 / 3); if (scrollTargetPos > documentHeight) { - scrollTargetPos = documentHeight + scrollTargetPos = documentHeight; } } // cancel previous scroll animation - let bodyEl = angular.element('body') - bodyEl.stop() - bodyEl.finish() + let bodyEl = angular.element('body'); + bodyEl.stop(); + bodyEl.finish(); // scroll to scrollTargetPos - bodyEl.scrollTo(scrollTargetPos, {axis: 'y', interrupt: true, duration: 100}) - } + bodyEl.scrollTo(scrollTargetPos, {axis: 'y', interrupt: true, duration: 100}); + }; - $scope.getEditorValue = function () { - return !$scope.editor ? $scope.paragraph.text : $scope.editor.getValue() - } + $scope.getEditorValue = function() { + return !$scope.editor ? $scope.paragraph.text : $scope.editor.getValue(); + }; - $scope.getProgress = function () { - return $scope.currentProgress || 0 - } + $scope.getProgress = function() { + return $scope.currentProgress || 0; + }; $scope.getFormattedParagraphTime = () => { - return moment().toISOString() - } + return moment().toISOString(); + }; - $scope.getExecutionTime = function (pdata) { - const end = pdata.dateFinished - const start = pdata.dateStarted - let timeMs = Date.parse(end) - Date.parse(start) + $scope.getExecutionTime = function(pdata) { + const end = pdata.dateFinished; + const start = pdata.dateStarted; + let timeMs = Date.parse(end) - Date.parse(start); if (isNaN(timeMs) || timeMs < 0) { if ($scope.isResultOutdated(pdata)) { - return 'outdated' + return 'outdated'; } - return '' + return ''; } - const durationFormat = moment.duration((timeMs / 1000), 'seconds').format('h [hrs] m [min] s [sec]') - const endFormat = moment(pdata.dateFinished).format('MMMM DD YYYY, h:mm:ss A') + const durationFormat = moment.duration((timeMs / 1000), 'seconds').format('h [hrs] m [min] s [sec]'); + const endFormat = moment(pdata.dateFinished).format('MMMM DD YYYY, h:mm:ss A'); - let user = (pdata.user === undefined || pdata.user === null) ? 'anonymous' : pdata.user - let desc = `Took ${durationFormat}. Last updated by ${user} at ${endFormat}.` + let user = (pdata.user === undefined || pdata.user === null) ? 'anonymous' : pdata.user; + let desc = `Took ${durationFormat}. Last updated by ${user} at ${endFormat}.`; - if ($scope.isResultOutdated(pdata)) { desc += ' (outdated)' } + if ($scope.isResultOutdated(pdata)) { + desc += ' (outdated)'; + } - return desc - } + return desc; + }; - $scope.getElapsedTime = function (paragraph) { - return 'Started ' + moment(paragraph.dateStarted).fromNow() + '.' - } + $scope.getElapsedTime = function(paragraph) { + return 'Started ' + moment(paragraph.dateStarted).fromNow() + '.'; + }; - $scope.isResultOutdated = function (pdata) { + $scope.isResultOutdated = function(pdata) { if (pdata.dateUpdated !== undefined && Date.parse(pdata.dateUpdated) > Date.parse(pdata.dateStarted)) { - return true + return true; } - return false - } + return false; + }; - $scope.goToEnd = function (editor) { - editor.navigateFileEnd() - } + $scope.goToEnd = function(editor) { + editor.navigateFileEnd(); + }; - $scope.parseTableCell = function (cell) { + $scope.parseTableCell = function(cell) { if (!isNaN(cell)) { if (cell.length === 0 || Number(cell) > Number.MAX_SAFE_INTEGER || Number(cell) < Number.MIN_SAFE_INTEGER) { - return cell + return cell; } else { - return Number(cell) + return Number(cell); } } - let d = moment(cell) + let d = moment(cell); if (d.isValid()) { - return d + return d; } - return cell - } + return cell; + }; - const commitParagraph = function (paragraph) { + const commitParagraph = function(paragraph) { const { id, title, text, config, settings: {params}, - } = paragraph + } = paragraph; return websocketMsgSrv.commitParagraph(id, title, text, config, params, - $route.current.pathParams.noteId) - } + $route.current.pathParams.noteId); + }; /** Utility function */ - $scope.goToSingleParagraph = function () { - let noteId = $route.current.pathParams.noteId + $scope.goToSingleParagraph = function() { + let noteId = $route.current.pathParams.noteId; let redirectToUrl = location.protocol + '//' + location.host + location.pathname + '#/notebook/' + noteId + - '/paragraph/' + $scope.paragraph.id + '?asIframe' - $window.open(redirectToUrl) - } + '/paragraph/' + $scope.paragraph.id + '?asIframe'; + $window.open(redirectToUrl); + }; - $scope.showScrollDownIcon = function (id) { - let doc = angular.element('#p' + id + '_text') + $scope.showScrollDownIcon = function(id) { + let doc = angular.element('#p' + id + '_text'); if (doc[0]) { - return doc[0].scrollHeight > doc.innerHeight() + return doc[0].scrollHeight > doc.innerHeight(); } - return false - } + return false; + }; - $scope.scrollParagraphDown = function (id) { - let doc = angular.element('#p' + id + '_text') - doc.animate({scrollTop: doc[0].scrollHeight}, 500) - $scope.keepScrollDown = true - } + $scope.scrollParagraphDown = function(id) { + let doc = angular.element('#p' + id + '_text'); + doc.animate({scrollTop: doc[0].scrollHeight}, 500); + $scope.keepScrollDown = true; + }; - $scope.showScrollUpIcon = function (id) { + $scope.showScrollUpIcon = function(id) { if (angular.element('#p' + id + '_text')[0]) { - return angular.element('#p' + id + '_text')[0].scrollTop !== 0 + return angular.element('#p' + id + '_text')[0].scrollTop !== 0; } - return false - } + return false; + }; - $scope.scrollParagraphUp = function (id) { - let doc = angular.element('#p' + id + '_text') - doc.animate({scrollTop: 0}, 500) - $scope.keepScrollDown = false - } + $scope.scrollParagraphUp = function(id) { + let doc = angular.element('#p' + id + '_text'); + doc.animate({scrollTop: 0}, 500); + $scope.keepScrollDown = false; + }; - $scope.$on('angularObjectUpdate', function (event, data) { - let noteId = $route.current.pathParams.noteId + $scope.$on('angularObjectUpdate', function(event, data) { + let noteId = $route.current.pathParams.noteId; if (!data.noteId || data.noteId === noteId) { - let scope - let registry + let scope; + let registry; if (!data.paragraphId || data.paragraphId === $scope.paragraph.id) { - scope = paragraphScope - registry = angularObjectRegistry + scope = paragraphScope; + registry = angularObjectRegistry; } else { - return + return; } - let varName = data.angularObject.name + let varName = data.angularObject.name; if (angular.equals(data.angularObject.object, scope[varName])) { // return when update has no change - return + return; } if (!registry[varName]) { registry[varName] = { interpreterGroupId: data.interpreterGroupId, noteId: data.noteId, - paragraphId: data.paragraphId - } + paragraphId: data.paragraphId, + }; } else { - registry[varName].noteId = registry[varName].noteId || data.noteId - registry[varName].paragraphId = registry[varName].paragraphId || data.paragraphId + registry[varName].noteId = registry[varName].noteId || data.noteId; + registry[varName].paragraphId = registry[varName].paragraphId || data.paragraphId; } - registry[varName].skipEmit = true + registry[varName].skipEmit = true; if (!registry[varName].clearWatcher) { - registry[varName].clearWatcher = scope.$watch(varName, function (newValue, oldValue) { - console.log('angular object (paragraph) updated %o %o', varName, registry[varName]) + registry[varName].clearWatcher = scope.$watch(varName, function(newValue, oldValue) { + console.log('angular object (paragraph) updated %o %o', varName, registry[varName]); if (registry[varName].skipEmit) { - registry[varName].skipEmit = false - return + registry[varName].skipEmit = false; + return; } websocketMsgSrv.updateAngularObject( registry[varName].noteId, registry[varName].paragraphId, varName, newValue, - registry[varName].interpreterGroupId) - }) + registry[varName].interpreterGroupId); + }); } - console.log('angular object (paragraph) created %o', varName) - scope[varName] = data.angularObject.object + console.log('angular object (paragraph) created %o', varName); + scope[varName] = data.angularObject.object; // create proxy for AngularFunction if (varName.indexOf(ANGULAR_FUNCTION_OBJECT_NAME_PREFIX) === 0) { - let funcName = varName.substring((ANGULAR_FUNCTION_OBJECT_NAME_PREFIX).length) - scope[funcName] = function () { + let funcName = varName.substring((ANGULAR_FUNCTION_OBJECT_NAME_PREFIX).length); + scope[funcName] = function() { // eslint-disable-next-line prefer-rest-params - scope[varName] = arguments + scope[varName] = arguments; // eslint-disable-next-line prefer-rest-params - console.log('angular function (paragraph) invoked %o', arguments) - } + console.log('angular function (paragraph) invoked %o', arguments); + }; - console.log('angular function (paragraph) created %o', scope[funcName]) + console.log('angular function (paragraph) created %o', scope[funcName]); } } - }) + }); - $scope.$on('updateParaInfos', function (event, data) { + $scope.$on('updateParaInfos', function(event, data) { if (data.id === $scope.paragraph.id) { - $scope.paragraph.runtimeInfos = data.infos + $scope.paragraph.runtimeInfos = data.infos; } - }) + }); - $scope.$on('angularObjectRemove', function (event, data) { - let noteId = $route.current.pathParams.noteId + $scope.$on('angularObjectRemove', function(event, data) { + let noteId = $route.current.pathParams.noteId; if (!data.noteId || data.noteId === noteId) { - let scope - let registry + let scope; + let registry; if (!data.paragraphId || data.paragraphId === $scope.paragraph.id) { - scope = paragraphScope - registry = angularObjectRegistry + scope = paragraphScope; + registry = angularObjectRegistry; } else { - return + return; } - let varName = data.name + let varName = data.name; // clear watcher if (registry[varName]) { - registry[varName].clearWatcher() - registry[varName] = undefined + registry[varName].clearWatcher(); + registry[varName] = undefined; } // remove scope variable - scope[varName] = undefined + scope[varName] = undefined; // remove proxy for AngularFunction if (varName.indexOf(ANGULAR_FUNCTION_OBJECT_NAME_PREFIX) === 0) { - let funcName = varName.substring((ANGULAR_FUNCTION_OBJECT_NAME_PREFIX).length) - scope[funcName] = undefined + let funcName = varName.substring((ANGULAR_FUNCTION_OBJECT_NAME_PREFIX).length); + scope[funcName] = undefined; } } - }) + }); /** * @returns {boolean} true if updated is needed */ - function isUpdateRequired (oldPara, newPara) { + function isUpdateRequired(oldPara, newPara) { return (newPara.id === oldPara.id && (newPara.dateCreated !== oldPara.dateCreated || newPara.text !== oldPara.text || @@ -1322,453 +1338,457 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca newPara.errorMessage !== oldPara.errorMessage || !angular.equals(newPara.settings, oldPara.settings) || !angular.equals(newPara.config, oldPara.config) || - !angular.equals(newPara.runtimeInfos, oldPara.runtimeInfos))) + !angular.equals(newPara.runtimeInfos, oldPara.runtimeInfos))); } - $scope.updateAllScopeTexts = function (oldPara, newPara) { + $scope.updateAllScopeTexts = function(oldPara, newPara) { if (oldPara.text !== newPara.text) { if ($scope.dirtyText) { // check if editor has local update if ($scope.dirtyText === newPara.text) { // when local update is the same from remote, clear local update - $scope.paragraph.text = newPara.text - $scope.dirtyText = undefined - $scope.originalText = angular.copy(newPara.text) + $scope.paragraph.text = newPara.text; + $scope.dirtyText = undefined; + $scope.originalText = angular.copy(newPara.text); } else { // if there're local update, keep it. - $scope.paragraph.text = newPara.text + $scope.paragraph.text = newPara.text; } } else { - $scope.paragraph.text = newPara.text - $scope.originalText = angular.copy(newPara.text) + $scope.paragraph.text = newPara.text; + $scope.originalText = angular.copy(newPara.text); } } - } + }; - $scope.updateParagraphObjectWhenUpdated = function (newPara) { + $scope.updateParagraphObjectWhenUpdated = function(newPara) { // resize col width if ($scope.paragraph.config.colWidth !== newPara.config.colWidth) { - $scope.$broadcast('paragraphResized', $scope.paragraph.id) + $scope.$broadcast('paragraphResized', $scope.paragraph.id); } if ($scope.paragraph.config.fontSize !== newPara.config.fontSize) { - $rootScope.$broadcast('fontSizeChanged', newPara.config.fontSize) + $rootScope.$broadcast('fontSizeChanged', newPara.config.fontSize); } /** push the rest */ - $scope.paragraph.aborted = newPara.aborted - $scope.paragraph.user = newPara.user - $scope.paragraph.dateUpdated = newPara.dateUpdated - $scope.paragraph.dateCreated = newPara.dateCreated - $scope.paragraph.dateFinished = newPara.dateFinished - $scope.paragraph.dateStarted = newPara.dateStarted - $scope.paragraph.errorMessage = newPara.errorMessage - $scope.paragraph.jobName = newPara.jobName - $scope.paragraph.title = newPara.title - $scope.paragraph.lineNumbers = newPara.lineNumbers - $scope.paragraph.status = newPara.status - $scope.paragraph.fontSize = newPara.fontSize + $scope.paragraph.aborted = newPara.aborted; + $scope.paragraph.user = newPara.user; + $scope.paragraph.dateUpdated = newPara.dateUpdated; + $scope.paragraph.dateCreated = newPara.dateCreated; + $scope.paragraph.dateFinished = newPara.dateFinished; + $scope.paragraph.dateStarted = newPara.dateStarted; + $scope.paragraph.errorMessage = newPara.errorMessage; + $scope.paragraph.jobName = newPara.jobName; + $scope.paragraph.title = newPara.title; + $scope.paragraph.lineNumbers = newPara.lineNumbers; + $scope.paragraph.status = newPara.status; + $scope.paragraph.fontSize = newPara.fontSize; if (newPara.status !== ParagraphStatus.RUNNING) { - $scope.paragraph.results = newPara.results + $scope.paragraph.results = newPara.results; } - $scope.paragraph.settings = newPara.settings - $scope.paragraph.runtimeInfos = newPara.runtimeInfos + $scope.paragraph.settings = newPara.settings; + $scope.paragraph.runtimeInfos = newPara.runtimeInfos; if ($scope.editor) { - $scope.editor.setReadOnly($scope.isRunning(newPara)) + $scope.editor.setReadOnly($scope.isRunning(newPara)); } if (!$scope.asIframe) { - $scope.paragraph.config = newPara.config - initializeDefault(newPara.config) + $scope.paragraph.config = newPara.config; + initializeDefault(newPara.config); } else { - newPara.config.editorHide = true - newPara.config.tableHide = false - $scope.paragraph.config = newPara.config + newPara.config.editorHide = true; + newPara.config.tableHide = false; + $scope.paragraph.config = newPara.config; } - } + }; - $scope.updateParagraph = function (oldPara, newPara, updateCallback) { + $scope.updateParagraph = function(oldPara, newPara, updateCallback) { // 1. can't update on revision view if ($scope.revisionView === true) { - return + return; } // 2. get status, refreshed - const statusChanged = (newPara.status !== oldPara.status) + const statusChanged = (newPara.status !== oldPara.status); const resultRefreshed = (newPara.dateFinished !== oldPara.dateFinished) || isEmpty(newPara.results) !== isEmpty(oldPara.results) || newPara.status === ParagraphStatus.ERROR || - (newPara.status === ParagraphStatus.FINISHED && statusChanged) + (newPara.status === ParagraphStatus.FINISHED && statusChanged); // 3. update texts managed by $scope - $scope.updateAllScopeTexts(oldPara, newPara) + $scope.updateAllScopeTexts(oldPara, newPara); // 4. execute callback to update result - updateCallback() + updateCallback(); // 5. update remaining paragraph objects - $scope.updateParagraphObjectWhenUpdated(newPara) + $scope.updateParagraphObjectWhenUpdated(newPara); // 6. handle scroll down by key properly if new paragraph is added if (statusChanged || resultRefreshed) { // when last paragraph runs, zeppelin automatically appends new paragraph. // this broadcast will focus to the newly inserted paragraph - const paragraphs = angular.element('div[id$="_paragraphColumn_main"]') + const paragraphs = angular.element('div[id$="_paragraphColumn_main"]'); if (paragraphs.length >= 2 && paragraphs[paragraphs.length - 2].id.indexOf($scope.paragraph.id) === 0) { // rendering output can took some time. So delay scrolling event firing for sometime. - setTimeout(() => { $rootScope.$broadcast('scrollToCursor') }, 500) + setTimeout(() => { + $rootScope.$broadcast('scrollToCursor'); + }, 500); } } - } + }; /** $scope.$on */ - $scope.$on('runParagraphUsingSpell', function (event, data) { - const oldPara = $scope.paragraph - let newPara = data.paragraph + $scope.$on('runParagraphUsingSpell', function(event, data) { + const oldPara = $scope.paragraph; + let newPara = data.paragraph; const updateCallback = () => { - $scope.runParagraph(newPara.text, true, true) - } + $scope.runParagraph(newPara.text, true, true); + }; if (!isUpdateRequired(oldPara, newPara)) { - return + return; } - $scope.updateParagraph(oldPara, newPara, updateCallback) - }) + $scope.updateParagraph(oldPara, newPara, updateCallback); + }); - $scope.$on('updateParagraph', function (event, data) { - const oldPara = $scope.paragraph - const newPara = data.paragraph + $scope.$on('updateParagraph', function(event, data) { + const oldPara = $scope.paragraph; + const newPara = data.paragraph; if (!isUpdateRequired(oldPara, newPara)) { - return + return; } const updateCallback = () => { // broadcast `updateResult` message to trigger result update if (newPara.results && newPara.results.msg) { for (let i in newPara.results.msg) { - const newResult = newPara.results.msg ? newPara.results.msg[i] : {} - const oldResult = (oldPara.results && oldPara.results.msg) - ? oldPara.results.msg[i] : {} - const newConfig = newPara.config.results ? newPara.config.results[i] : {} - const oldConfig = oldPara.config.results ? oldPara.config.results[i] : {} - if (!angular.equals(newResult, oldResult) || - !angular.equals(newConfig, oldConfig)) { - $rootScope.$broadcast('updateResult', newResult, newConfig, newPara, parseInt(i)) + if (newPara.results.msg.hasOwnProperty(i)) { + const newResult = newPara.results.msg ? newPara.results.msg[i] : {}; + const oldResult = (oldPara.results && oldPara.results.msg) + ? oldPara.results.msg[i] : {}; + const newConfig = newPara.config.results ? newPara.config.results[i] : {}; + const oldConfig = oldPara.config.results ? oldPara.config.results[i] : {}; + if (!angular.equals(newResult, oldResult) || + !angular.equals(newConfig, oldConfig)) { + $rootScope.$broadcast('updateResult', newResult, newConfig, newPara, parseInt(i)); + } } } } - } + }; - $scope.updateParagraph(oldPara, newPara, updateCallback) - }) + $scope.updateParagraph(oldPara, newPara, updateCallback); + }); - $scope.$on('updateProgress', function (event, data) { + $scope.$on('updateProgress', function(event, data) { if (data.id === $scope.paragraph.id) { - $scope.currentProgress = data.progress + $scope.currentProgress = data.progress; } - }) + }); - $scope.$on('appendParagraphOutput', function (event, data) { + $scope.$on('appendParagraphOutput', function(event, data) { if (data.paragraphId === $scope.paragraph.id) { if (!$scope.paragraph.results) { - $scope.paragraph.results = {} + $scope.paragraph.results = {}; if (!$scope.paragraph.results.msg) { - $scope.paragraph.results.msg = [] + $scope.paragraph.results.msg = []; } $scope.paragraph.results.msg[data.index] = { data: data.data, - type: data.type - } + type: data.type, + }; $rootScope.$broadcast( 'updateResult', $scope.paragraph.results.msg[data.index], $scope.paragraph.config.results[data.index], $scope.paragraph, - data.index) + data.index); } } - }) + }); - $scope.$on('keyEvent', function (event, keyEvent) { + $scope.$on('keyEvent', function(event, keyEvent) { if ($scope.paragraphFocused) { - let paragraphId = $scope.paragraph.id - let keyCode = keyEvent.keyCode - let noShortcutDefined = false - let editorHide = $scope.paragraph.config.editorHide + let paragraphId = $scope.paragraph.id; + let keyCode = keyEvent.keyCode; + let noShortcutDefined = false; + let editorHide = $scope.paragraph.config.editorHide; if (editorHide && (keyCode === 38 || (keyCode === 80 && keyEvent.ctrlKey && !keyEvent.altKey))) { // up // move focus to previous paragraph - $scope.$emit('moveFocusToPreviousParagraph', paragraphId) + $scope.$emit('moveFocusToPreviousParagraph', paragraphId); } else if (editorHide && (keyCode === 40 || (keyCode === 78 && keyEvent.ctrlKey && !keyEvent.altKey))) { // down // move focus to next paragraph // $timeout stops chaining effect of focus propogation - $timeout(() => $scope.$emit('moveFocusToNextParagraph', paragraphId)) + $timeout(() => $scope.$emit('moveFocusToNextParagraph', paragraphId)); } else if (!keyEvent.ctrlKey && keyEvent.shiftKey && keyCode === 13) { // Shift + Enter - $scope.runParagraphFromShortcut($scope.getEditorValue()) + $scope.runParagraphFromShortcut($scope.getEditorValue()); } else if (keyEvent.ctrlKey && keyEvent.shiftKey && keyCode === 13) { // Ctrl + Shift + Enter - $scope.runAllToOrFromThis($scope.paragraph) + $scope.runAllToOrFromThis($scope.paragraph); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 67) { // Ctrl + Alt + c - $scope.cancelParagraph($scope.paragraph) + $scope.cancelParagraph($scope.paragraph); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 68) { // Ctrl + Alt + d - $scope.removeParagraph($scope.paragraph) + $scope.removeParagraph($scope.paragraph); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 75) { // Ctrl + Alt + k - $scope.moveUp($scope.paragraph) + $scope.moveUp($scope.paragraph); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 74) { // Ctrl + Alt + j - $scope.moveDown($scope.paragraph) + $scope.moveDown($scope.paragraph); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 65) { // Ctrl + Alt + a - $scope.insertNew('above') + $scope.insertNew('above'); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 66) { // Ctrl + Alt + b - $scope.insertNew('below') + $scope.insertNew('below'); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 79) { // Ctrl + Alt + o - $scope.toggleOutput($scope.paragraph) + $scope.toggleOutput($scope.paragraph); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 82) { // Ctrl + Alt + r - $scope.toggleEnableDisable($scope.paragraph) + $scope.toggleEnableDisable($scope.paragraph); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 69) { // Ctrl + Alt + e - $scope.toggleEditor($scope.paragraph) + $scope.toggleEditor($scope.paragraph); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 77) { // Ctrl + Alt + m if ($scope.paragraph.config.lineNumbers) { - $scope.hideLineNumbers($scope.paragraph) + $scope.hideLineNumbers($scope.paragraph); } else { - $scope.showLineNumbers($scope.paragraph) + $scope.showLineNumbers($scope.paragraph); } } else if (keyEvent.ctrlKey && keyEvent.shiftKey && keyCode === 189) { // Ctrl + Shift + - - $scope.changeColWidth($scope.paragraph, Math.max(1, $scope.paragraph.config.colWidth - 1)) + $scope.changeColWidth($scope.paragraph, Math.max(1, $scope.paragraph.config.colWidth - 1)); } else if (keyEvent.ctrlKey && keyEvent.shiftKey && keyCode === 187) { // Ctrl + Shift + = - $scope.changeColWidth($scope.paragraph, Math.min(12, $scope.paragraph.config.colWidth + 1)) + $scope.changeColWidth($scope.paragraph, Math.min(12, $scope.paragraph.config.colWidth + 1)); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 84) { // Ctrl + Alt + t if ($scope.paragraph.config.title) { - $scope.hideTitle($scope.paragraph) + $scope.hideTitle($scope.paragraph); } else { - $scope.showTitle($scope.paragraph) + $scope.showTitle($scope.paragraph); } } else if (keyEvent.ctrlKey && keyEvent.shiftKey && keyCode === 67) { // Ctrl + Alt + c - $scope.copyPara('below') + $scope.copyPara('below'); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 76) { // Ctrl + Alt + l - $scope.clearParagraphOutput($scope.paragraph) + $scope.clearParagraphOutput($scope.paragraph); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 87) { // Ctrl + Alt + w - $scope.goToSingleParagraph() + $scope.goToSingleParagraph(); } else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 70) { // Ctrl + f - $scope.$emit('toggleSearchBox') + $scope.$emit('toggleSearchBox'); } else { - noShortcutDefined = true + noShortcutDefined = true; } if (!noShortcutDefined) { - keyEvent.preventDefault() + keyEvent.preventDefault(); } } - }) + }); - $scope.$on('focusParagraph', function (event, paragraphId, cursorPosRow, cursorPosCol, mouseEvent) { + $scope.$on('focusParagraph', function(event, paragraphId, cursorPosRow, cursorPosCol, mouseEvent) { if (cursorPosCol === null || cursorPosCol === undefined) { - cursorPosCol = 0 + cursorPosCol = 0; } if ($scope.paragraph.id === paragraphId) { // focus editor if (!$scope.paragraph.config.editorHide) { if (!mouseEvent) { - $scope.editor.focus() + $scope.editor.focus(); // move cursor to the first row (or the last row) - let row + let row; if (cursorPosRow >= 0) { - row = cursorPosRow - $scope.editor.gotoLine(row, cursorPosCol) + row = cursorPosRow; + $scope.editor.gotoLine(row, cursorPosCol); } else { - row = $scope.editor.session.getLength() - $scope.editor.gotoLine(row, cursorPosCol) + row = $scope.editor.session.getLength(); + $scope.editor.gotoLine(row, cursorPosCol); } - $scope.scrollToCursor($scope.paragraph.id, cursorPosCol) + $scope.scrollToCursor($scope.paragraph.id, cursorPosCol); } } - handleFocus(true) + handleFocus(true); } else { if ($scope.editor !== undefined && $scope.editor !== null) { - $scope.editor.blur() + $scope.editor.blur(); } - let isDigestPass = true - handleFocus(false, isDigestPass) + let isDigestPass = true; + handleFocus(false, isDigestPass); } - }) + }); - $scope.$on('saveInterpreterBindings', function (event, paragraphId) { + $scope.$on('saveInterpreterBindings', function(event, paragraphId) { if ($scope.paragraph.id === paragraphId && $scope.editor) { - setInterpreterBindings = true - setParagraphMode($scope.editor.getSession(), $scope.editor.getSession().getValue()) + setInterpreterBindings = true; + setParagraphMode($scope.editor.getSession(), $scope.editor.getSession().getValue()); } - }) + }); - $scope.$on('doubleClickParagraph', function (event, paragraphId) { + $scope.$on('doubleClickParagraph', function(event, paragraphId) { if ($scope.paragraph.id === paragraphId && $scope.paragraph.config.editorHide && $scope.paragraph.config.editorSetting.editOnDblClick && $scope.revisionView !== true) { - let deferred = $q.defer() - openEditorAndCloseTable($scope.paragraph) + let deferred = $q.defer(); + openEditorAndCloseTable($scope.paragraph); $timeout( - $scope.$on('updateParagraph', function (event, data) { - deferred.resolve(data) + $scope.$on('updateParagraph', function(event, data) { + deferred.resolve(data); } - ), 1000) + ), 1000); - deferred.promise.then(function (data) { + deferred.promise.then(function(data) { if ($scope.editor) { - $scope.editor.focus() - $scope.goToEnd($scope.editor) + $scope.editor.focus(); + $scope.goToEnd($scope.editor); } - }) + }); } - }) + }); - $scope.$on('openEditor', function (event) { - $scope.openEditor($scope.paragraph) - }) + $scope.$on('openEditor', function(event) { + $scope.openEditor($scope.paragraph); + }); - $scope.$on('closeEditor', function (event) { - $scope.closeEditor($scope.paragraph) - }) + $scope.$on('closeEditor', function(event) { + $scope.closeEditor($scope.paragraph); + }); - $scope.$on('openTable', function (event) { - $scope.openTable($scope.paragraph) - }) + $scope.$on('openTable', function(event) { + $scope.openTable($scope.paragraph); + }); - $scope.$on('closeTable', function (event) { - $scope.closeTable($scope.paragraph) - }) + $scope.$on('closeTable', function(event) { + $scope.closeTable($scope.paragraph); + }); - $scope.$on('resultRendered', function (event, paragraphId) { + $scope.$on('resultRendered', function(event, paragraphId) { if ($scope.paragraph.id !== paragraphId) { - return + return; } /** increase spell result count and return if not finished */ if (!$scope.increaseSpellTransactionResultCount()) { - return + return; } - $scope.cleanupSpellTransaction() - }) + $scope.cleanupSpellTransaction(); + }); - $scope.$on('fontSizeChanged', function (event, fontSize) { + $scope.$on('fontSizeChanged', function(event, fontSize) { if ($scope.editor) { $scope.editor.setOptions({ - fontSize: fontSize + 'pt' - }) + fontSize: fontSize + 'pt', + }); } - }) + }); const clearSearchSelection = function() { for (let i = 0; i < searchRanges.length; ++i) { - $scope.editor.session.removeMarker(searchRanges[i].markerId) + $scope.editor.session.removeMarker(searchRanges[i].markerId); } - searchRanges = [] + searchRanges = []; if (currentRange.id !== -1) { - $scope.editor.session.removeMarker(currentRange.markerId) + $scope.editor.session.removeMarker(currentRange.markerId); } - currentRange = getCurrentRangeDefault() - } + currentRange = getCurrentRangeDefault(); + }; $scope.onEditorClick = function() { - $scope.$emit('editorClicked') - } + $scope.$emit('editorClicked'); + }; $scope.$on('unmarkAll', function() { - clearSearchSelection() - }) + clearSearchSelection(); + }); const markAllOccurrences = function(text) { - clearSearchSelection() + clearSearchSelection(); if (text === '') { - return + return; } if ($scope.editor.findAll(text) === 0) { - return + return; } - let ranges = $scope.editor.selection.getAllRanges() - $scope.editor.selection.toSingleRange() - $scope.editor.selection.clearSelection() + let ranges = $scope.editor.selection.getAllRanges(); + $scope.editor.selection.toSingleRange(); + $scope.editor.selection.clearSelection(); for (let i = 0; i < ranges.length; ++i) { - let id = $scope.editor.session.addMarker(ranges[i], 'ace_selected-word', 'text') - searchRanges.push({markerId: id, range: ranges[i]}) + let id = $scope.editor.session.addMarker(ranges[i], 'ace_selected-word', 'text'); + searchRanges.push({markerId: id, range: ranges[i]}); } - } + }; $scope.$on('markAllOccurrences', function(event, text) { - markAllOccurrences(text) + markAllOccurrences(text); if (searchRanges.length > 0) { - $scope.$emit('occurrencesExists', searchRanges.length) + $scope.$emit('occurrencesExists', searchRanges.length); } - }) + }); $scope.$on('nextOccurrence', function(event, paragraphId) { if ($scope.paragraph.id !== paragraphId) { - return + return; } - let highlightedRangeExists = currentRange.id !== -1 + let highlightedRangeExists = currentRange.id !== -1; if (highlightedRangeExists) { - $scope.editor.session.removeMarker(currentRange.markerId) - currentRange.markerId = -1 + $scope.editor.session.removeMarker(currentRange.markerId); + currentRange.markerId = -1; } - ++currentRange.id + ++currentRange.id; if (currentRange.id >= searchRanges.length) { - currentRange.id = -1 - $scope.$emit('noNextOccurrence') - return + currentRange.id = -1; + $scope.$emit('noNextOccurrence'); + return; } currentRange.markerId = $scope.editor.session.addMarker( - searchRanges[currentRange.id].range, 'ace_selection', 'text') - }) + searchRanges[currentRange.id].range, 'ace_selection', 'text'); + }); $scope.$on('prevOccurrence', function(event, paragraphId) { if ($scope.paragraph.id !== paragraphId) { - return + return; } - let highlightedRangeExists = currentRange.id !== -1 + let highlightedRangeExists = currentRange.id !== -1; if (highlightedRangeExists) { - $scope.editor.session.removeMarker(currentRange.markerId) - currentRange.markerId = -1 + $scope.editor.session.removeMarker(currentRange.markerId); + currentRange.markerId = -1; } if (currentRange.id === -1) { - currentRange.id = searchRanges.length + currentRange.id = searchRanges.length; } - --currentRange.id + --currentRange.id; if (currentRange.id === -1) { - $scope.$emit('noPrevOccurrence') - return + $scope.$emit('noPrevOccurrence'); + return; } currentRange.markerId = $scope.editor.session.addMarker( - searchRanges[currentRange.id].range, 'ace_selection', 'text') - }) + searchRanges[currentRange.id].range, 'ace_selection', 'text'); + }); $scope.$on('replaceCurrent', function(event, from, to) { if (currentRange.id === -1) { - return - } - let indexFromEnd = searchRanges.length - currentRange.id - 1 - let prevId = currentRange.id - $scope.editor.session.removeMarker(currentRange.markerId) - $scope.editor.session.replace(searchRanges[currentRange.id].range, to) - markAllOccurrences(from) - let currentIndex = searchRanges.length - indexFromEnd - $scope.$emit('occurrencesCountChanged', currentIndex - prevId - 1) - currentRange.id = currentIndex + return; + } + let indexFromEnd = searchRanges.length - currentRange.id - 1; + let prevId = currentRange.id; + $scope.editor.session.removeMarker(currentRange.markerId); + $scope.editor.session.replace(searchRanges[currentRange.id].range, to); + markAllOccurrences(from); + let currentIndex = searchRanges.length - indexFromEnd; + $scope.$emit('occurrencesCountChanged', currentIndex - prevId - 1); + currentRange.id = currentIndex; if (currentRange.id === searchRanges.length) { - currentRange.id = -1 - $scope.$emit('noNextOccurrenceAfterReplace') + currentRange.id = -1; + $scope.$emit('noNextOccurrenceAfterReplace'); } else { currentRange.markerId = $scope.editor.session.addMarker( - searchRanges[currentRange.id].range, 'ace_selection', 'text') + searchRanges[currentRange.id].range, 'ace_selection', 'text'); } - }) + }); $scope.$on('replaceAll', function(event, from, to) { - clearSearchSelection() - $scope.editor.replaceAll(to, {needle: from}) - }) + clearSearchSelection(); + $scope.editor.replaceAll(to, {needle: from}); + }); $scope.$on('checkOccurrences', function() { if (searchRanges.length > 0) { - $scope.$emit('occurrencesExists', searchRanges.length) + $scope.$emit('occurrencesExists', searchRanges.length); } - }) + }); } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.test.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.test.js index 94230de7f97..38d5480e021 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.test.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.test.js @@ -1,53 +1,53 @@ -describe('Controller: ParagraphCtrl', function () { - beforeEach(angular.mock.module('zeppelinWebApp')) +describe('Controller: ParagraphCtrl', function() { + beforeEach(angular.mock.module('zeppelinWebApp')); - let scope - let websocketMsgSrvMock = {} + let scope; + let websocketMsgSrvMock = {}; let paragraphMock = { config: {}, settings: { - forms: {} - } - } + forms: {}, + }, + }; let route = { current: { pathParams: { - noteId: 'noteId' - } - } - } + noteId: 'noteId', + }, + }, + }; - beforeEach(inject(function ($controller, $rootScope) { - scope = $rootScope.$new() - $rootScope.notebookScope = $rootScope.$new(true, $rootScope) + beforeEach(inject(function($controller, $rootScope) { + scope = $rootScope.$new(); + $rootScope.notebookScope = $rootScope.$new(true, $rootScope); $controller('ParagraphCtrl', { $scope: scope, websocketMsgSrv: websocketMsgSrvMock, $element: {}, - $route: route - }) + $route: route, + }); - scope.init(paragraphMock) - })) + scope.init(paragraphMock); + })); let functions = ['isRunning', 'getIframeDimensions', 'cancelParagraph', 'runParagraph', 'saveParagraph', 'moveUp', 'moveDown', 'insertNew', 'removeParagraph', 'toggleEditor', 'closeEditor', 'openEditor', 'closeTable', 'openTable', 'showTitle', 'hideTitle', 'setTitle', 'showLineNumbers', 'hideLineNumbers', 'changeColWidth', 'columnWidthClass', 'toggleOutput', - 'aceChanged', 'aceLoaded', 'getEditorValue', 'getProgress', 'getExecutionTime', 'isResultOutdated'] - - functions.forEach(function (fn) { - it('check for scope functions to be defined : ' + fn, function () { - expect(scope[fn]).toBeDefined() - }) - }) - - it('should have this array of values for "colWidthOption"', function () { - expect(scope.colWidthOption).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) - }) - - it('should set default value of "paragraphFocused" as false', function () { - expect(scope.paragraphFocused).toEqual(false) - }) -}) + 'aceChanged', 'aceLoaded', 'getEditorValue', 'getProgress', 'getExecutionTime', 'isResultOutdated']; + + functions.forEach(function(fn) { + it('check for scope functions to be defined : ' + fn, function() { + expect(scope[fn]).toBeDefined(); + }); + }); + + it('should have this array of values for "colWidthOption"', function() { + expect(scope.colWidthOption).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + }); + + it('should set default value of "paragraphFocused" as false', function() { + expect(scope.paragraphFocused).toEqual(false); + }); +}); diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.status.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.status.js index f839eeeb219..bd70de3da59 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.status.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.status.js @@ -19,12 +19,16 @@ export const ParagraphStatus = { FINISHED: 'FINISHED', ABORT: 'ABORT', ERROR: 'ERROR', -} +}; -export function isParagraphRunning (paragraph) { - if (!paragraph) { return false } - const status = paragraph.status - if (!status) { return false } +export function isParagraphRunning(paragraph) { + if (!paragraph) { + return false; + } + const status = paragraph.status; + if (!status) { + return false; + } - return status === ParagraphStatus.PENDING || status === ParagraphStatus.RUNNING + return status === ParagraphStatus.PENDING || status === ParagraphStatus.RUNNING; } diff --git a/zeppelin-web/src/app/notebook/paragraph/resizable.directive.js b/zeppelin-web/src/app/notebook/paragraph/resizable.directive.js index 2893cd5d088..874f9d82154 100644 --- a/zeppelin-web/src/app/notebook/paragraph/resizable.directive.js +++ b/zeppelin-web/src/app/notebook/paragraph/resizable.directive.js @@ -12,58 +12,58 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').directive('resizable', ResizableDirective) +angular.module('zeppelinWebApp').directive('resizable', ResizableDirective); -function ResizableDirective () { +function ResizableDirective() { let resizableConfig = { autoHide: true, handles: 'se', helper: 'resizable-helper', - stop: function () { - angular.element(this).css({'width': '100%', 'height': '100%'}) - } - } + stop: function() { + angular.element(this).css({'width': '100%', 'height': '100%'}); + }, + }; return { restrict: 'A', scope: { - callback: '&onResize' + callback: '&onResize', }, - link: function postLink (scope, elem, attrs) { - attrs.$observe('resize', function (resize) { - let resetResize = function (elem, resize) { - let colStep = window.innerWidth / 12 - elem.off('resizestop') - let conf = angular.copy(resizableConfig) + link: function postLink(scope, elem, attrs) { + attrs.$observe('resize', function(resize) { + let resetResize = function(elem, resize) { + let colStep = window.innerWidth / 12; + elem.off('resizestop'); + let conf = angular.copy(resizableConfig); if (resize.graphType === 'TABLE' || resize.graphType === 'NETWORK' || resize.graphType === 'TEXT') { - conf.grid = [colStep, 10] - conf.minHeight = 100 + conf.grid = [colStep, 10]; + conf.minHeight = 100; } else { - conf.grid = [colStep, 10000] - conf.minHeight = 0 + conf.grid = [colStep, 10000]; + conf.minHeight = 0; } - conf.maxWidth = window.innerWidth + conf.maxWidth = window.innerWidth; - elem.resizable(conf) - elem.on('resizestop', function () { + elem.resizable(conf); + elem.on('resizestop', function() { if (scope.callback) { - let height = elem.height() + let height = elem.height(); if (height < 50) { - height = 300 + height = 300; } - scope.callback({width: Math.ceil(elem.width() / colStep), height: height}) + scope.callback({width: Math.ceil(elem.width() / colStep), height: height}); } - }) - } + }); + }; - resize = JSON.parse(resize) + resize = JSON.parse(resize); if (resize.allowresize === 'true') { - resetResize(elem, resize) - angular.element(window).resize(function () { - resetResize(elem, resize) - }) + resetResize(elem, resize); + angular.element(window).resize(function() { + resetResize(elem, resize); + }); } - }) - } - } + }); + }, + }; } diff --git a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js index 5dfe3143971..5bf77dcd71e 100644 --- a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js @@ -12,30 +12,30 @@ * limitations under the License. */ -import moment from 'moment' - -import DatasetFactory from '../../../tabledata/datasetfactory' -import TableVisualization from '../../../visualization/builtins/visualization-table' -import BarchartVisualization from '../../../visualization/builtins/visualization-barchart' -import PiechartVisualization from '../../../visualization/builtins/visualization-piechart' -import AreachartVisualization from '../../../visualization/builtins/visualization-areachart' -import LinechartVisualization from '../../../visualization/builtins/visualization-linechart' -import ScatterchartVisualization from '../../../visualization/builtins/visualization-scatterchart' -import NetworkVisualization from '../../../visualization/builtins/visualization-d3network' -import {DefaultDisplayType, SpellResult} from '../../../spell' -import {ParagraphStatus} from '../paragraph.status' - -const AnsiUp = require('ansi_up') -const AnsiUpConverter = new AnsiUp.default // eslint-disable-line new-parens,new-cap -const TableGridFilterTemplate = require('../../../visualization/builtins/visualization-table-grid-filter.html') - -angular.module('zeppelinWebApp').controller('ResultCtrl', ResultCtrl) - -function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $location, +import moment from 'moment'; + +import DatasetFactory from '../../../tabledata/datasetfactory'; +import TableVisualization from '../../../visualization/builtins/visualization-table'; +import BarchartVisualization from '../../../visualization/builtins/visualization-barchart'; +import PiechartVisualization from '../../../visualization/builtins/visualization-piechart'; +import AreachartVisualization from '../../../visualization/builtins/visualization-areachart'; +import LinechartVisualization from '../../../visualization/builtins/visualization-linechart'; +import ScatterchartVisualization from '../../../visualization/builtins/visualization-scatterchart'; +import NetworkVisualization from '../../../visualization/builtins/visualization-d3network'; +import {DefaultDisplayType, SpellResult} from '../../../spell'; +import {ParagraphStatus} from '../paragraph.status'; + +const AnsiUp = require('ansi_up'); +const AnsiUpConverter = new AnsiUp.default; // eslint-disable-line new-parens,new-cap +const TableGridFilterTemplate = require('../../../visualization/builtins/visualization-table-grid-filter.html'); + +angular.module('zeppelinWebApp').controller('ResultCtrl', ResultCtrl); + +function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location, $timeout, $compile, $http, $q, $templateCache, $templateRequest, $sce, websocketMsgSrv, baseUrlSrv, ngToast, saveAsService, noteVarShareService, heliumService, uiGridConstants) { - 'ngInject' + 'ngInject'; /** * Built-in visualizations @@ -45,49 +45,49 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio id: 'table', // paragraph.config.graph.mode name: 'Table', // human readable name. tooltip icon: '', - supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK] + supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK], }, { id: 'multiBarChart', name: 'Bar Chart', icon: '', transformation: 'pivot', - supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK] + supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK], }, { id: 'pieChart', name: 'Pie Chart', icon: '', transformation: 'pivot', - supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK] + supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK], }, { id: 'stackedAreaChart', name: 'Area Chart', icon: '', transformation: 'pivot', - supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK] + supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK], }, { id: 'lineChart', name: 'Line Chart', icon: '', transformation: 'pivot', - supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK] + supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK], }, { id: 'scatterChart', name: 'Scatter Chart', icon: '', - supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK] + supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK], }, { id: 'network', name: 'Network', icon: '', - supports: [DefaultDisplayType.NETWORK] - } - ] + supports: [DefaultDisplayType.NETWORK], + }, + ]; /** * Holds class and actual runtime instance and related infos of built-in visualizations @@ -95,137 +95,143 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio let builtInVisualizations = { 'table': { class: TableVisualization, - instance: undefined // created from setGraphMode() + instance: undefined, // created from setGraphMode() }, 'multiBarChart': { class: BarchartVisualization, - instance: undefined + instance: undefined, }, 'pieChart': { class: PiechartVisualization, - instance: undefined + instance: undefined, }, 'stackedAreaChart': { class: AreachartVisualization, - instance: undefined + instance: undefined, }, 'lineChart': { class: LinechartVisualization, - instance: undefined + instance: undefined, }, 'scatterChart': { class: ScatterchartVisualization, - instance: undefined + instance: undefined, }, 'network': { class: NetworkVisualization, - instance: undefined - } - } + instance: undefined, + }, + }; // type - $scope.type = null + $scope.type = null; // Data of the result - let data + let data; // config - $scope.config = null + $scope.config = null; // resultId = paragraph.id + index - $scope.id = null + $scope.id = null; // referece to paragraph - let paragraph + let paragraph; // index of the result - let resultIndex + let resultIndex; // TableData instance - let tableData + let tableData; // available columns in tabledata - $scope.tableDataColumns = [] + $scope.tableDataColumns = []; // enable helium - let enableHelium = false + let enableHelium = false; // graphMode - $scope.graphMode = null + $scope.graphMode = null; // image data - $scope.imageData = null + $scope.imageData = null; // queue for append output - const textResultQueueForAppend = [] + const textResultQueueForAppend = []; // prevent body area scrollbar from blocking due to scroll in paragraph results - $scope.mouseOver = false - $scope.onMouseOver = function() { $scope.mouseOver = true } - $scope.onMouseOut = function() { $scope.mouseOver = false } + $scope.mouseOver = false; + $scope.onMouseOver = function() { + $scope.mouseOver = true; + }; + $scope.onMouseOut = function() { + $scope.mouseOver = false; + }; $scope.getPointerEvent = function() { - return ($scope.mouseOver) ? {'pointer-events': 'auto' } - : {'pointer-events': 'none' } - } + return ($scope.mouseOver) ? {'pointer-events': 'auto'} + : {'pointer-events': 'none'}; + }; - $scope.init = function (result, config, paragraph, index) { + $scope.init = function(result, config, paragraph, index) { // register helium plugin vis packages - let visPackages = heliumService.getVisualizationCachedPackages() - const visPackageOrder = heliumService.getVisualizationCachedPackageOrder() + let visPackages = heliumService.getVisualizationCachedPackages(); + const visPackageOrder = heliumService.getVisualizationCachedPackageOrder(); // push the helium vis packages following the order - visPackageOrder.map(visName => { - visPackages.map(vis => { - if (vis.name !== visName) { return } + visPackageOrder.map((visName) => { + visPackages.map((vis) => { + if (vis.name !== visName) { + return; + } $scope.builtInTableDataVisualizationList.push({ id: vis.id, name: vis.name, icon: $sce.trustAsHtml(vis.icon), - supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK] - }) + supports: [DefaultDisplayType.TABLE, DefaultDisplayType.NETWORK], + }); builtInVisualizations[vis.id] = { - class: vis.class - } - }) - }) - - updateData(result, config, paragraph, index) - renderResult($scope.type) - } - - function isDOMLoaded (targetElemId) { - const elem = angular.element(`#${targetElemId}`) - return elem.length + class: vis.class, + }; + }); + }); + + updateData(result, config, paragraph, index); + renderResult($scope.type); + }; + + function isDOMLoaded(targetElemId) { + const elem = angular.element(`#${targetElemId}`); + return elem.length; } - function retryUntilElemIsLoaded (targetElemId, callback) { - function retry () { + function retryUntilElemIsLoaded(targetElemId, callback) { + function retry() { if (!isDOMLoaded(targetElemId)) { - $timeout(retry, 10) - return + $timeout(retry, 10); + return; } - const elem = angular.element(`#${targetElemId}`) - callback(elem) + const elem = angular.element(`#${targetElemId}`); + callback(elem); } - $timeout(retry) + $timeout(retry); } - $scope.$on('updateResult', function (event, result, newConfig, paragraphRef, index) { + $scope.$on('updateResult', function(event, result, newConfig, paragraphRef, index) { if (paragraph.id !== paragraphRef.id || index !== resultIndex) { - return + return; } let refresh = !angular.equals(newConfig, $scope.config) || !angular.equals(result.type, $scope.type) || - !angular.equals(result.data, data) + !angular.equals(result.data, data); - updateData(result, newConfig, paragraph, resultIndex) - renderResult($scope.type, refresh) - }) + updateData(result, newConfig, paragraph, resultIndex); + renderResult($scope.type, refresh); + }); - $scope.$on('appendParagraphOutput', function (event, data) { + $scope.$on('appendParagraphOutput', function(event, data) { /* It has been observed that append events * can be errorneously called even if paragraph * execution has ended, and in that case, no append @@ -238,162 +244,162 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio resultIndex === data.index && (paragraph.status === ParagraphStatus.PENDING || paragraph.status === ParagraphStatus.RUNNING)) { if (DefaultDisplayType.TEXT !== $scope.type) { - $scope.type = DefaultDisplayType.TEXT + $scope.type = DefaultDisplayType.TEXT; } - appendTextOutput(data.data) + appendTextOutput(data.data); } - }) + }); - const updateData = function (result, config, paragraphRef, index) { - data = result.data - paragraph = paragraphRef - resultIndex = parseInt(index) + const updateData = function(result, config, paragraphRef, index) { + data = result.data; + paragraph = paragraphRef; + resultIndex = parseInt(index); - $scope.id = paragraph.id + '_' + index - $scope.type = result.type - config = config ? config : {} + $scope.id = paragraph.id + '_' + index; + $scope.type = result.type; + config = config ? config : {}; // initialize default config values if (!config.graph) { - config.graph = {} + config.graph = {}; } if (!config.graph.mode) { - config.graph.mode = 'table' + config.graph.mode = 'table'; } if (!config.graph.height) { - config.graph.height = 300 + config.graph.height = 300; } if (!config.graph.optionOpen) { - config.graph.optionOpen = false + config.graph.optionOpen = false; } - $scope.graphMode = config.graph.mode - $scope.config = angular.copy(config) + $scope.graphMode = config.graph.mode; + $scope.config = angular.copy(config); // enable only when it is last result - enableHelium = (index === paragraphRef.results.msg.length - 1) + enableHelium = (index === paragraphRef.results.msg.length - 1); if ($scope.type === 'TABLE' || $scope.type === 'NETWORK') { - tableData = new DatasetFactory().createDataset($scope.type) - tableData.loadParagraphResult({type: $scope.type, msg: data}) - $scope.tableDataColumns = tableData.columns - $scope.tableDataComment = tableData.comment + tableData = new DatasetFactory().createDataset($scope.type); + tableData.loadParagraphResult({type: $scope.type, msg: data}); + $scope.tableDataColumns = tableData.columns; + $scope.tableDataComment = tableData.comment; if ($scope.type === 'NETWORK') { - $scope.networkNodes = tableData.networkNodes - $scope.networkRelationships = tableData.networkRelationships - $scope.networkProperties = tableData.networkProperties + $scope.networkNodes = tableData.networkNodes; + $scope.networkRelationships = tableData.networkRelationships; + $scope.networkProperties = tableData.networkProperties; } } else if ($scope.type === 'IMG') { - $scope.imageData = data + $scope.imageData = data; } - } + }; - $scope.createDisplayDOMId = function (baseDOMId, type) { + $scope.createDisplayDOMId = function(baseDOMId, type) { if (type === DefaultDisplayType.TABLE || type === DefaultDisplayType.NETWORK) { - return `${baseDOMId}_graph` + return `${baseDOMId}_graph`; } else if (type === DefaultDisplayType.HTML) { - return `${baseDOMId}_html` + return `${baseDOMId}_html`; } else if (type === DefaultDisplayType.ANGULAR) { - return `${baseDOMId}_angular` + return `${baseDOMId}_angular`; } else if (type === DefaultDisplayType.TEXT) { - return `${baseDOMId}_text` + return `${baseDOMId}_text`; } else if (type === DefaultDisplayType.ELEMENT) { - return `${baseDOMId}_elem` + return `${baseDOMId}_elem`; } else { - console.error(`Cannot create display DOM Id due to unknown display type: ${type}`) + console.error(`Cannot create display DOM Id due to unknown display type: ${type}`); } - } + }; - $scope.renderDefaultDisplay = function (targetElemId, type, data, refresh) { + $scope.renderDefaultDisplay = function(targetElemId, type, data, refresh) { const afterLoaded = () => { if (type === DefaultDisplayType.TABLE || type === DefaultDisplayType.NETWORK) { - renderGraph(targetElemId, $scope.graphMode, refresh) + renderGraph(targetElemId, $scope.graphMode, refresh); } else if (type === DefaultDisplayType.HTML) { - renderHtml(targetElemId, data) + renderHtml(targetElemId, data); } else if (type === DefaultDisplayType.ANGULAR) { - renderAngular(targetElemId, data) + renderAngular(targetElemId, data); } else if (type === DefaultDisplayType.TEXT) { - renderText(targetElemId, data, refresh) + renderText(targetElemId, data, refresh); } else if (type === DefaultDisplayType.ELEMENT) { - renderElem(targetElemId, data) + renderElem(targetElemId, data); } else { - console.error(`Unknown Display Type: ${type}`) + console.error(`Unknown Display Type: ${type}`); } - } + }; - retryUntilElemIsLoaded(targetElemId, afterLoaded) + retryUntilElemIsLoaded(targetElemId, afterLoaded); // send message to parent that this result is rendered - const paragraphId = $scope.$parent.paragraph.id - $scope.$emit('resultRendered', paragraphId) - } + const paragraphId = $scope.$parent.paragraph.id; + $scope.$emit('resultRendered', paragraphId); + }; - const renderResult = function (type, refresh) { - let activeApp + const renderResult = function(type, refresh) { + let activeApp; if (enableHelium) { - getSuggestions() - getApplicationStates() - activeApp = _.get($scope.config, 'helium.activeApp') + getSuggestions(); + getApplicationStates(); + activeApp = _.get($scope.config, 'helium.activeApp'); } if (activeApp) { - const appState = _.find($scope.apps, {id: activeApp}) - renderApp(`p${appState.id}`, appState) + const appState = _.find($scope.apps, {id: activeApp}); + renderApp(`p${appState.id}`, appState); } else { if (!DefaultDisplayType[type]) { - $scope.renderCustomDisplay(type, data) + $scope.renderCustomDisplay(type, data); } else { - const targetElemId = $scope.createDisplayDOMId(`p${$scope.id}`, type) - $scope.renderDefaultDisplay(targetElemId, type, data, refresh) + const targetElemId = $scope.createDisplayDOMId(`p${$scope.id}`, type); + $scope.renderDefaultDisplay(targetElemId, type, data, refresh); } } - } + }; - $scope.isDefaultDisplay = function () { - return DefaultDisplayType[$scope.type] - } + $scope.isDefaultDisplay = function() { + return DefaultDisplayType[$scope.type]; + }; /** * Render multiple sub results for custom display */ - $scope.renderCustomDisplay = function (type, data) { + $scope.renderCustomDisplay = function(type, data) { // get result from intp if (!heliumService.getSpellByMagic(type)) { - console.error(`Can't execute spell due to unknown display type: ${type}`) - return + console.error(`Can't execute spell due to unknown display type: ${type}`); + return; } // custom display result can include multiple subset results heliumService.executeSpellAsDisplaySystem(type, data) - .then(dataWithTypes => { - const containerDOMId = `p${$scope.id}_custom` + .then((dataWithTypes) => { + const containerDOMId = `p${$scope.id}_custom`; const afterLoaded = () => { - const containerDOM = angular.element(`#${containerDOMId}`) + const containerDOM = angular.element(`#${containerDOMId}`); // Spell.interpret() can create multiple outputs for (let i = 0; i < dataWithTypes.length; i++) { - const dt = dataWithTypes[i] - const data = dt.data - const type = dt.type + const dt = dataWithTypes[i]; + const data = dt.data; + const type = dt.type; // prepare each DOM to be filled - const subResultDOMId = $scope.createDisplayDOMId(`p${$scope.id}_custom_${i}`, type) - const subResultDOM = document.createElement('div') - containerDOM.append(subResultDOM) - subResultDOM.setAttribute('id', subResultDOMId) + const subResultDOMId = $scope.createDisplayDOMId(`p${$scope.id}_custom_${i}`, type); + const subResultDOM = document.createElement('div'); + containerDOM.append(subResultDOM); + subResultDOM.setAttribute('id', subResultDOMId); - $scope.renderDefaultDisplay(subResultDOMId, type, data, true) + $scope.renderDefaultDisplay(subResultDOMId, type, data, true); } - } + }; - retryUntilElemIsLoaded(containerDOMId, afterLoaded) - }) - .catch(error => { - console.error(`Failed to render custom display: ${$scope.type}\n` + error) + retryUntilElemIsLoaded(containerDOMId, afterLoaded); }) - } + .catch((error) => { + console.error(`Failed to render custom display: ${$scope.type}\n` + error); + }); + }; /** * generates actually object which will be consumed from `data` property @@ -405,688 +411,710 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio * @param successCallback * @param failureCallback */ - const handleData = function (data, type, successCallback, failureCallback) { + const handleData = function(data, type, successCallback, failureCallback) { if (SpellResult.isFunction(data)) { try { - successCallback(data()) + successCallback(data()); } catch (error) { - failureCallback(error) - console.error(`Failed to handle ${type} type, function data\n`, error) + failureCallback(error); + console.error(`Failed to handle ${type} type, function data\n`, error); } } else if (SpellResult.isObject(data)) { try { - successCallback(data) + successCallback(data); } catch (error) { - console.error(`Failed to handle ${type} type, object data\n`, error) + console.error(`Failed to handle ${type} type, object data\n`, error); } } - } + }; - const renderElem = function (targetElemId, data) { - const elem = angular.element(`#${targetElemId}`) - handleData(() => { data(targetElemId) }, DefaultDisplayType.ELEMENT, + const renderElem = function(targetElemId, data) { + const elem = angular.element(`#${targetElemId}`); + handleData(() => { + data(targetElemId); + }, DefaultDisplayType.ELEMENT, () => {}, /** HTML element will be filled with data. thus pass empty success callback */ - (error) => { elem.html(`${error.stack}`) } - ) - } + (error) => { + elem.html(`${error.stack}`); + } + ); + }; - const renderHtml = function (targetElemId, data) { - const elem = angular.element(`#${targetElemId}`) + const renderHtml = function(targetElemId, data) { + const elem = angular.element(`#${targetElemId}`); handleData(data, DefaultDisplayType.HTML, (generated) => { - elem.html(generated) - elem.find('pre code').each(function (i, e) { - hljs.highlightBlock(e) - }) + elem.html(generated); + elem.find('pre code').each(function(i, e) { + hljs.highlightBlock(e); + }); /* eslint new-cap: [2, {"capIsNewExceptions": ["MathJax.Hub.Queue"]}] */ - MathJax.Hub.Queue(['Typeset', MathJax.Hub, elem[0]]) + MathJax.Hub.Queue(['Typeset', MathJax.Hub, elem[0]]); }, - (error) => { elem.html(`${error.stack}`) } - ) - } + (error) => { + elem.html(`${error.stack}`); + } + ); + }; - const renderAngular = function (targetElemId, data) { - const elem = angular.element(`#${targetElemId}`) - const paragraphScope = noteVarShareService.get(`${paragraph.id}_paragraphScope`) + const renderAngular = function(targetElemId, data) { + const elem = angular.element(`#${targetElemId}`); + const paragraphScope = noteVarShareService.get(`${paragraph.id}_paragraphScope`); handleData(data, DefaultDisplayType.ANGULAR, (generated) => { - elem.html(generated) - $compile(elem.contents())(paragraphScope) + elem.html(generated); + $compile(elem.contents())(paragraphScope); }, - (error) => { elem.html(`${error.stack}`) } - ) - } + (error) => { + elem.html(`${error.stack}`); + } + ); + }; - const getTextResultElemId = function (resultId) { - return `p${resultId}_text` - } + const getTextResultElemId = function(resultId) { + return `p${resultId}_text`; + }; - const checkAndReplaceCarriageReturn = function (str) { + const checkAndReplaceCarriageReturn = function(str) { if (/\r/.test(str)) { - let newGenerated = '' - let strArr = str.split('\n') + let newGenerated = ''; + let strArr = str.split('\n'); for (let str of strArr) { if (/\r/.test(str)) { - let splitCR = str.split('\r') - newGenerated += splitCR[splitCR.length - 1] + '\n' + let splitCR = str.split('\r'); + newGenerated += splitCR[splitCR.length - 1] + '\n'; } else { - newGenerated += str + '\n' + newGenerated += str + '\n'; } } // remove last "\n" character - return newGenerated.slice(0, -1) + return newGenerated.slice(0, -1); } else { - return str + return str; } - } + }; - const renderText = function (targetElemId, data, refresh) { - const elem = angular.element(`#${targetElemId}`) + const renderText = function(targetElemId, data, refresh) { + const elem = angular.element(`#${targetElemId}`); handleData(data, DefaultDisplayType.TEXT, (generated) => { // clear all lines before render - removeChildrenDOM(targetElemId) + removeChildrenDOM(targetElemId); if (generated) { - generated = checkAndReplaceCarriageReturn(generated) - const escaped = AnsiUpConverter.ansi_to_html(generated) - const divDOM = angular.element('
').innerHTML = escaped + generated = checkAndReplaceCarriageReturn(generated); + const escaped = AnsiUpConverter.ansi_to_html(generated); + const divDOM = angular.element('
').innerHTML = escaped; if (refresh) { - elem.html(divDOM) + elem.html(divDOM); } else { - elem.append(divDOM) + elem.append(divDOM); } } else if (refresh) { - elem.html('') + elem.html(''); } - elem.bind('mousewheel', (e) => { $scope.keepScrollDown = false }) + elem.bind('mousewheel', (e) => { + $scope.keepScrollDown = false; + }); }, - (error) => { elem.html(`${error.stack}`) } - ) - } + (error) => { + elem.html(`${error.stack}`); + } + ); + }; - const removeChildrenDOM = function (targetElemId) { - const elem = angular.element(`#${targetElemId}`) + const removeChildrenDOM = function(targetElemId) { + const elem = angular.element(`#${targetElemId}`); if (elem.length) { - elem.children().remove() + elem.children().remove(); } - } + }; - function appendTextOutput (data) { - const elemId = getTextResultElemId($scope.id) - textResultQueueForAppend.push(data) + function appendTextOutput(data) { + const elemId = getTextResultElemId($scope.id); + textResultQueueForAppend.push(data); // if DOM is not loaded, just push data and return if (!isDOMLoaded(elemId)) { - return + return; } - const elem = angular.element(`#${elemId}`) + const elem = angular.element(`#${elemId}`); // pop all stacked data and append to the DOM while (textResultQueueForAppend.length > 0) { - const line = elem.html() + AnsiUpConverter.ansi_to_html(textResultQueueForAppend.pop()) - elem.html(checkAndReplaceCarriageReturn(line)) + const line = elem.html() + AnsiUpConverter.ansi_to_html(textResultQueueForAppend.pop()); + elem.html(checkAndReplaceCarriageReturn(line)); if ($scope.keepScrollDown) { - const doc = angular.element(`#${elemId}`) - doc[0].scrollTop = doc[0].scrollHeight + const doc = angular.element(`#${elemId}`); + doc[0].scrollTop = doc[0].scrollHeight; } } } - const getTrSettingElem = function (scopeId, graphMode) { - return angular.element('#trsetting' + scopeId + '_' + graphMode) - } + const getTrSettingElem = function(scopeId, graphMode) { + return angular.element('#trsetting' + scopeId + '_' + graphMode); + }; - const getVizSettingElem = function (scopeId, graphMode) { - return angular.element('#vizsetting' + scopeId + '_' + graphMode) - } + const getVizSettingElem = function(scopeId, graphMode) { + return angular.element('#vizsetting' + scopeId + '_' + graphMode); + }; - const renderGraph = function (graphElemId, graphMode, refresh) { + const renderGraph = function(graphElemId, graphMode, refresh) { // set graph height - const height = $scope.config.graph.height - const graphElem = angular.element(`#${graphElemId}`) - graphElem.height(height) + const height = $scope.config.graph.height; + const graphElem = angular.element(`#${graphElemId}`); + graphElem.height(height); - if (!graphMode) { graphMode = 'table' } + if (!graphMode) { + graphMode = 'table'; + } - let builtInViz = builtInVisualizations[graphMode] + let builtInViz = builtInVisualizations[graphMode]; if (!builtInViz) { /** helium package is not available, fallback to table vis */ - graphMode = 'table' - $scope.graphMode = graphMode /** html depends on this scope value */ - builtInViz = builtInVisualizations[graphMode] + graphMode = 'table'; + $scope.graphMode = graphMode; /** html depends on this scope value */ + builtInViz = builtInVisualizations[graphMode]; } // deactive previsouly active visualization for (let t in builtInVisualizations) { - const v = builtInVisualizations[t].instance + if (builtInVisualizations.hasOwnProperty(t)) { + const v = builtInVisualizations[t].instance; - if (t !== graphMode && v && v.isActive()) { - v.deactivate() - break + if (t !== graphMode && v && v.isActive()) { + v.deactivate(); + break; + } } } - let afterLoaded = function () { /** will be overwritten */ } + let afterLoaded = function() { /** will be overwritten */ }; if (!builtInViz.instance) { // not instantiated yet // render when targetEl is available - afterLoaded = function (loadedElem) { + afterLoaded = function(loadedElem) { try { - const transformationSettingTargetEl = getTrSettingElem($scope.id, graphMode) - const visualizationSettingTargetEl = getVizSettingElem($scope.id, graphMode) + const transformationSettingTargetEl = getTrSettingElem($scope.id, graphMode); + const visualizationSettingTargetEl = getVizSettingElem($scope.id, graphMode); // set height - loadedElem.height(height) + loadedElem.height(height); // instantiate visualization - const config = getVizConfig(graphMode) - const Visualization = builtInViz.class - builtInViz.instance = new Visualization(loadedElem, config) + const config = getVizConfig(graphMode); + const Visualization = builtInViz.class; + builtInViz.instance = new Visualization(loadedElem, config); // inject emitter, $templateRequest - const emitter = function (graphSetting) { - commitVizConfigChange(graphSetting, graphMode) - } - builtInViz.instance._emitter = emitter - builtInViz.instance._compile = $compile + const emitter = function(graphSetting) { + commitVizConfigChange(graphSetting, graphMode); + }; + builtInViz.instance._emitter = emitter; + builtInViz.instance._compile = $compile; // ui-grid related - $templateCache.put('ui-grid/ui-grid-filter', TableGridFilterTemplate) - builtInViz.instance._uiGridConstants = uiGridConstants - builtInViz.instance._timeout = $timeout - - builtInViz.instance._createNewScope = createNewScope - builtInViz.instance._templateRequest = $templateRequest - const transformation = builtInViz.instance.getTransformation() - transformation._emitter = emitter - transformation._templateRequest = $templateRequest - transformation._compile = $compile - transformation._createNewScope = createNewScope + $templateCache.put('ui-grid/ui-grid-filter', TableGridFilterTemplate); + builtInViz.instance._uiGridConstants = uiGridConstants; + builtInViz.instance._timeout = $timeout; + + builtInViz.instance._createNewScope = createNewScope; + builtInViz.instance._templateRequest = $templateRequest; + const transformation = builtInViz.instance.getTransformation(); + transformation._emitter = emitter; + transformation._templateRequest = $templateRequest; + transformation._compile = $compile; + transformation._createNewScope = createNewScope; // render - const transformed = transformation.transform(tableData) - transformation.renderSetting(transformationSettingTargetEl) - builtInViz.instance.render(transformed) - builtInViz.instance.renderSetting(visualizationSettingTargetEl) - builtInViz.instance.activate() + const transformed = transformation.transform(tableData); + transformation.renderSetting(transformationSettingTargetEl); + builtInViz.instance.render(transformed); + builtInViz.instance.renderSetting(visualizationSettingTargetEl); + builtInViz.instance.activate(); angular.element(window).resize(() => { - builtInViz.instance.resize() - }) + builtInViz.instance.resize(); + }); } catch (err) { - console.error('Graph drawing error %o', err) + console.error('Graph drawing error %o', err); } - } + }; } else if (refresh) { // when graph options or data are changed - console.log('Refresh data %o', tableData) - - afterLoaded = function (loadedElem) { - const transformationSettingTargetEl = getTrSettingElem($scope.id, graphMode) - const visualizationSettingTargetEl = getVizSettingElem($scope.id, graphMode) - const config = getVizConfig(graphMode) - loadedElem.height(height) - const transformation = builtInViz.instance.getTransformation() - transformation.setConfig(config) - const transformed = transformation.transform(tableData) - transformation.renderSetting(transformationSettingTargetEl) - builtInViz.instance.setConfig(config) - builtInViz.instance.render(transformed) - builtInViz.instance.renderSetting(visualizationSettingTargetEl) - builtInViz.instance.activate() - } + console.log('Refresh data %o', tableData); + + afterLoaded = function(loadedElem) { + const transformationSettingTargetEl = getTrSettingElem($scope.id, graphMode); + const visualizationSettingTargetEl = getVizSettingElem($scope.id, graphMode); + const config = getVizConfig(graphMode); + loadedElem.height(height); + const transformation = builtInViz.instance.getTransformation(); + transformation.setConfig(config); + const transformed = transformation.transform(tableData); + transformation.renderSetting(transformationSettingTargetEl); + builtInViz.instance.setConfig(config); + builtInViz.instance.render(transformed); + builtInViz.instance.renderSetting(visualizationSettingTargetEl); + builtInViz.instance.activate(); + }; } else { - afterLoaded = function (loadedElem) { - loadedElem.height(height) - builtInViz.instance.activate() - } + afterLoaded = function(loadedElem) { + loadedElem.height(height); + builtInViz.instance.activate(); + }; } - const tableElemId = `p${$scope.id}_${graphMode}` - retryUntilElemIsLoaded(tableElemId, afterLoaded) - } + const tableElemId = `p${$scope.id}_${graphMode}`; + retryUntilElemIsLoaded(tableElemId, afterLoaded); + }; - $scope.switchViz = function (newMode) { - let newConfig = angular.copy($scope.config) - let newParams = angular.copy(paragraph.settings.params) + $scope.switchViz = function(newMode) { + let newConfig = angular.copy($scope.config); + let newParams = angular.copy(paragraph.settings.params); // graph options - newConfig.graph.mode = newMode + newConfig.graph.mode = newMode; // see switchApp() - _.set(newConfig, 'helium.activeApp', undefined) + _.set(newConfig, 'helium.activeApp', undefined); - commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams) - } + commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams); + }; - const createNewScope = function () { - return $rootScope.$new(true) - } + const createNewScope = function() { + return $rootScope.$new(true); + }; - const commitParagraphResult = function (title, text, config, params) { - let newParagraphConfig = angular.copy(paragraph.config) - newParagraphConfig.results = newParagraphConfig.results || [] - newParagraphConfig.results[resultIndex] = config + const commitParagraphResult = function(title, text, config, params) { + let newParagraphConfig = angular.copy(paragraph.config); + newParagraphConfig.results = newParagraphConfig.results || []; + newParagraphConfig.results[resultIndex] = config; if ($scope.revisionView === true) { // local update without commit updateData({ type: $scope.type, - data: data - }, newParagraphConfig.results[resultIndex], paragraph, resultIndex) - renderResult($scope.type, true) + data: data, + }, newParagraphConfig.results[resultIndex], paragraph, resultIndex); + renderResult($scope.type, true); } else { - return websocketMsgSrv.commitParagraph(paragraph.id, title, text, newParagraphConfig, params) + return websocketMsgSrv.commitParagraph(paragraph.id, title, text, newParagraphConfig, params); } - } + }; - $scope.toggleGraphSetting = function () { - let newConfig = angular.copy($scope.config) + $scope.toggleGraphSetting = function() { + let newConfig = angular.copy($scope.config); if (newConfig.graph.optionOpen) { - newConfig.graph.optionOpen = false + newConfig.graph.optionOpen = false; } else { - newConfig.graph.optionOpen = true + newConfig.graph.optionOpen = true; } - let newParams = angular.copy(paragraph.settings.params) - commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams) - } + let newParams = angular.copy(paragraph.settings.params); + commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams); + }; - const getVizConfig = function (vizId) { - let config - let graph = $scope.config.graph + const getVizConfig = function(vizId) { + let config; + let graph = $scope.config.graph; if (graph) { // copy setting for vizId if (graph.setting) { - config = angular.copy(graph.setting[vizId]) + config = angular.copy(graph.setting[vizId]); } if (!config) { - config = {} + config = {}; } // copy common setting - config.common = angular.copy(graph.commonSetting) || {} + config.common = angular.copy(graph.commonSetting) || {}; // copy pivot setting if (graph.keys) { config.common.pivot = { keys: angular.copy(graph.keys), groups: angular.copy(graph.groups), - values: angular.copy(graph.values) - } + values: angular.copy(graph.values), + }; } } - console.debug('getVizConfig', config) - return config - } + console.debug('getVizConfig', config); + return config; + }; - const commitVizConfigChange = function (config, vizId) { - let newConfig = angular.copy($scope.config) + const commitVizConfigChange = function(config, vizId) { + let newConfig = angular.copy($scope.config); if (!newConfig.graph) { - newConfig.graph = {} + newConfig.graph = {}; } // copy setting for vizId if (!newConfig.graph.setting) { - newConfig.graph.setting = {} + newConfig.graph.setting = {}; } - newConfig.graph.setting[vizId] = angular.copy(config) + newConfig.graph.setting[vizId] = angular.copy(config); // copy common setting if (newConfig.graph.setting[vizId]) { - newConfig.graph.commonSetting = newConfig.graph.setting[vizId].common - delete newConfig.graph.setting[vizId].common + newConfig.graph.commonSetting = newConfig.graph.setting[vizId].common; + delete newConfig.graph.setting[vizId].common; } // copy pivot setting if (newConfig.graph.commonSetting && newConfig.graph.commonSetting.pivot) { - newConfig.graph.keys = newConfig.graph.commonSetting.pivot.keys - newConfig.graph.groups = newConfig.graph.commonSetting.pivot.groups - newConfig.graph.values = newConfig.graph.commonSetting.pivot.values - delete newConfig.graph.commonSetting.pivot + newConfig.graph.keys = newConfig.graph.commonSetting.pivot.keys; + newConfig.graph.groups = newConfig.graph.commonSetting.pivot.groups; + newConfig.graph.values = newConfig.graph.commonSetting.pivot.values; + delete newConfig.graph.commonSetting.pivot; } - console.debug('committVizConfig', newConfig) - let newParams = angular.copy(paragraph.settings.params) - commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams) - } + console.debug('committVizConfig', newConfig); + let newParams = angular.copy(paragraph.settings.params); + commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams); + }; - $scope.$on('paragraphResized', function (event, paragraphId) { + $scope.$on('paragraphResized', function(event, paragraphId) { // paragraph col width changed if (paragraphId === paragraph.id) { - let builtInViz = builtInVisualizations[$scope.graphMode] + let builtInViz = builtInVisualizations[$scope.graphMode]; if (builtInViz && builtInViz.instance) { - $timeout(_ => builtInViz.instance.resize(), 200) + $timeout(() => builtInViz.instance.resize(), 200); } } - }) + }); - $scope.resize = function (width, height) { - $timeout(function () { - changeHeight(width, height) - }, 200) - } + $scope.resize = function(width, height) { + $timeout(function() { + changeHeight(width, height); + }, 200); + }; - const changeHeight = function (width, height) { - let newParams = angular.copy(paragraph.settings.params) - let newConfig = angular.copy($scope.config) + const changeHeight = function(width, height) { + let newParams = angular.copy(paragraph.settings.params); + let newConfig = angular.copy($scope.config); - newConfig.graph.height = height - paragraph.config.colWidth = width + newConfig.graph.height = height; + paragraph.config.colWidth = width; - commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams) - } + commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams); + }; - $scope.exportToDSV = function (delimiter) { - let dsv = '' - let dateFinished = moment(paragraph.dateFinished).format('YYYY-MM-DD hh:mm:ss A') - let exportedFileName = paragraph.title ? paragraph.title + '_' + dateFinished : 'data_' + dateFinished + $scope.exportToDSV = function(delimiter) { + let dsv = ''; + let dateFinished = moment(paragraph.dateFinished).format('YYYY-MM-DD hh:mm:ss A'); + let exportedFileName = paragraph.title ? paragraph.title + '_' + dateFinished : 'data_' + dateFinished; for (let titleIndex in tableData.columns) { - dsv += tableData.columns[titleIndex].name + delimiter + if (tableData.columns.hasOwnProperty(titleIndex)) { + dsv += tableData.columns[titleIndex].name + delimiter; + } } - dsv = dsv.substring(0, dsv.length - 1) + '\n' + dsv = dsv.substring(0, dsv.length - 1) + '\n'; for (let r in tableData.rows) { - let row = tableData.rows[r] - let dsvRow = '' - for (let index in row) { - let stringValue = (row[index]).toString() - if (stringValue.indexOf(delimiter) > -1) { - dsvRow += '"' + stringValue + '"' + delimiter - } else { - dsvRow += row[index] + delimiter + if (tableData.rows.hasOwnProperty(r)) { + let row = tableData.rows[r]; + let dsvRow = ''; + for (let index in row) { + if (row.hasOwnProperty(index)) { + let stringValue = (row[index]).toString(); + if (stringValue.indexOf(delimiter) > -1) { + dsvRow += '"' + stringValue + '"' + delimiter; + } else { + dsvRow += row[index] + delimiter; + } + } } + dsv += dsvRow.substring(0, dsvRow.length - 1) + '\n'; } - dsv += dsvRow.substring(0, dsvRow.length - 1) + '\n' } - let extension = '' + let extension = ''; if (delimiter === '\t') { - extension = 'tsv' + extension = 'tsv'; } else if (delimiter === ',') { - extension = 'csv' + extension = 'csv'; } - saveAsService.saveAs(dsv, exportedFileName, extension) - } + saveAsService.saveAs(dsv, exportedFileName, extension); + }; - $scope.getBase64ImageSrc = function (base64Data) { - return 'data:image/png;base64,' + base64Data - } + $scope.getBase64ImageSrc = function(base64Data) { + return 'data:image/png;base64,' + base64Data; + }; // Helium ---------------- - let ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_' + let ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_'; // app states - $scope.apps = [] + $scope.apps = []; // suggested apps - $scope.suggestion = {} + $scope.suggestion = {}; - $scope.switchApp = function (appId) { - let newConfig = angular.copy($scope.config) - let newParams = angular.copy(paragraph.settings.params) + $scope.switchApp = function(appId) { + let newConfig = angular.copy($scope.config); + let newParams = angular.copy(paragraph.settings.params); // 'helium.activeApp' can be cleared by switchViz() - _.set(newConfig, 'helium.activeApp', appId) + _.set(newConfig, 'helium.activeApp', appId); - commitConfig(newConfig, newParams) - } + commitConfig(newConfig, newParams); + }; - $scope.loadApp = function (heliumPackage) { - let noteId = $route.current.pathParams.noteId + $scope.loadApp = function(heliumPackage) { + let noteId = $route.current.pathParams.noteId; $http.post(baseUrlSrv.getRestApiBase() + '/helium/load/' + noteId + '/' + paragraph.id, heliumPackage) - .success(function (data, status, headers, config) { - console.log('Load app %o', data) + .success(function(data, status, headers, config) { + console.log('Load app %o', data); }) - .error(function (err, status, headers, config) { - console.log('Error %o', err) - }) - } + .error(function(err, status, headers, config) { + console.log('Error %o', err); + }); + }; - const commitConfig = function (config, params) { - commitParagraphResult(paragraph.title, paragraph.text, config, params) - } + const commitConfig = function(config, params) { + commitParagraphResult(paragraph.title, paragraph.text, config, params); + }; - const getApplicationStates = function () { - let appStates = [] + const getApplicationStates = function() { + let appStates = []; // Display ApplicationState if (paragraph.apps) { - _.forEach(paragraph.apps, function (app) { + _.forEach(paragraph.apps, function(app) { appStates.push({ id: app.id, pkg: app.pkg, status: app.status, - output: app.output - }) - }) + output: app.output, + }); + }); } // update or remove app states no longer exists - _.forEach($scope.apps, function (currentAppState, idx) { - let newAppState = _.find(appStates, {id: currentAppState.id}) + _.forEach($scope.apps, function(currentAppState, idx) { + let newAppState = _.find(appStates, {id: currentAppState.id}); if (newAppState) { - angular.extend($scope.apps[idx], newAppState) + angular.extend($scope.apps[idx], newAppState); } else { - $scope.apps.splice(idx, 1) + $scope.apps.splice(idx, 1); } - }) + }); // add new app states - _.forEach(appStates, function (app, idx) { + _.forEach(appStates, function(app, idx) { if ($scope.apps.length <= idx || $scope.apps[idx].id !== app.id) { - $scope.apps.splice(idx, 0, app) + $scope.apps.splice(idx, 0, app); } - }) - } + }); + }; - const getSuggestions = function () { + const getSuggestions = function() { // Get suggested apps - let noteId = $route.current.pathParams.noteId + let noteId = $route.current.pathParams.noteId; if (!noteId) { - return + return; } $http.get(baseUrlSrv.getRestApiBase() + '/helium/suggest/' + noteId + '/' + paragraph.id) - .success(function (data, status, headers, config) { - $scope.suggestion = data.body - }) - .error(function (err, status, headers, config) { - console.log('Error %o', err) + .success(function(data, status, headers, config) { + $scope.suggestion = data.body; }) - } + .error(function(err, status, headers, config) { + console.log('Error %o', err); + }); + }; - const renderApp = function (targetElemId, appState) { + const renderApp = function(targetElemId, appState) { const afterLoaded = (loadedElem) => { try { - console.log('renderApp %o', appState) - loadedElem.html(appState.output) - $compile(loadedElem.contents())(getAppScope(appState)) + console.log('renderApp %o', appState); + loadedElem.html(appState.output); + $compile(loadedElem.contents())(getAppScope(appState)); } catch (err) { - console.log('App rendering error %o', err) + console.log('App rendering error %o', err); } - } - retryUntilElemIsLoaded(targetElemId, afterLoaded) - } + }; + retryUntilElemIsLoaded(targetElemId, afterLoaded); + }; /* ** $scope.$on functions below */ - $scope.$on('appendAppOutput', function (event, data) { + $scope.$on('appendAppOutput', function(event, data) { if (paragraph.id === data.paragraphId) { - let app = _.find($scope.apps, {id: data.appId}) + let app = _.find($scope.apps, {id: data.appId}); if (app) { - app.output += data.data + app.output += data.data; - let paragraphAppState = _.find(paragraph.apps, {id: data.appId}) - paragraphAppState.output = app.output + let paragraphAppState = _.find(paragraph.apps, {id: data.appId}); + paragraphAppState.output = app.output; - let targetEl = angular.element(document.getElementById('p' + app.id)) - targetEl.html(app.output) - $compile(targetEl.contents())(getAppScope(app)) - console.log('append app output %o', $scope.apps) + let targetEl = angular.element(document.getElementById('p' + app.id)); + targetEl.html(app.output); + $compile(targetEl.contents())(getAppScope(app)); + console.log('append app output %o', $scope.apps); } } - }) + }); - $scope.$on('updateAppOutput', function (event, data) { + $scope.$on('updateAppOutput', function(event, data) { if (paragraph.id === data.paragraphId) { - let app = _.find($scope.apps, {id: data.appId}) + let app = _.find($scope.apps, {id: data.appId}); if (app) { - app.output = data.data + app.output = data.data; - let paragraphAppState = _.find(paragraph.apps, {id: data.appId}) - paragraphAppState.output = app.output + let paragraphAppState = _.find(paragraph.apps, {id: data.appId}); + paragraphAppState.output = app.output; - let targetEl = angular.element(document.getElementById('p' + app.id)) - targetEl.html(app.output) - $compile(targetEl.contents())(getAppScope(app)) - console.log('append app output') + let targetEl = angular.element(document.getElementById('p' + app.id)); + targetEl.html(app.output); + $compile(targetEl.contents())(getAppScope(app)); + console.log('append app output'); } } - }) + }); - $scope.$on('appLoad', function (event, data) { + $scope.$on('appLoad', function(event, data) { if (paragraph.id === data.paragraphId) { - let app = _.find($scope.apps, {id: data.appId}) + let app = _.find($scope.apps, {id: data.appId}); if (!app) { app = { id: data.appId, pkg: data.pkg, status: 'UNLOADED', - output: '' - } + output: '', + }; - $scope.apps.push(app) - paragraph.apps.push(app) - $scope.switchApp(app.id) + $scope.apps.push(app); + paragraph.apps.push(app); + $scope.switchApp(app.id); } } - }) + }); - $scope.$on('appStatusChange', function (event, data) { + $scope.$on('appStatusChange', function(event, data) { if (paragraph.id === data.paragraphId) { - let app = _.find($scope.apps, {id: data.appId}) + let app = _.find($scope.apps, {id: data.appId}); if (app) { - app.status = data.status - let paragraphAppState = _.find(paragraph.apps, {id: data.appId}) - paragraphAppState.status = app.status + app.status = data.status; + let paragraphAppState = _.find(paragraph.apps, {id: data.appId}); + paragraphAppState.status = app.status; } } - }) + }); - let getAppRegistry = function (appState) { + let getAppRegistry = function(appState) { if (!appState.registry) { - appState.registry = {} + appState.registry = {}; } - return appState.registry - } + return appState.registry; + }; - const getAppScope = function (appState) { + const getAppScope = function(appState) { if (!appState.scope) { - appState.scope = $rootScope.$new(true, $rootScope) + appState.scope = $rootScope.$new(true, $rootScope); } - return appState.scope - } + return appState.scope; + }; - $scope.$on('angularObjectUpdate', function (event, data) { - let noteId = $route.current.pathParams.noteId + $scope.$on('angularObjectUpdate', function(event, data) { + let noteId = $route.current.pathParams.noteId; if (!data.noteId || data.noteId === noteId) { - let scope - let registry + let scope; + let registry; - let app = _.find($scope.apps, {id: data.paragraphId}) + let app = _.find($scope.apps, {id: data.paragraphId}); if (app) { - scope = getAppScope(app) - registry = getAppRegistry(app) + scope = getAppScope(app); + registry = getAppRegistry(app); } else { // no matching app in this paragraph - return + return; } - let varName = data.angularObject.name + let varName = data.angularObject.name; if (angular.equals(data.angularObject.object, scope[varName])) { // return when update has no change - return + return; } if (!registry[varName]) { registry[varName] = { interpreterGroupId: data.interpreterGroupId, noteId: data.noteId, - paragraphId: data.paragraphId - } + paragraphId: data.paragraphId, + }; } else { - registry[varName].noteId = registry[varName].noteId || data.noteId - registry[varName].paragraphId = registry[varName].paragraphId || data.paragraphId + registry[varName].noteId = registry[varName].noteId || data.noteId; + registry[varName].paragraphId = registry[varName].paragraphId || data.paragraphId; } - registry[varName].skipEmit = true + registry[varName].skipEmit = true; if (!registry[varName].clearWatcher) { - registry[varName].clearWatcher = scope.$watch(varName, function (newValue, oldValue) { - console.log('angular object (paragraph) updated %o %o', varName, registry[varName]) + registry[varName].clearWatcher = scope.$watch(varName, function(newValue, oldValue) { + console.log('angular object (paragraph) updated %o %o', varName, registry[varName]); if (registry[varName].skipEmit) { - registry[varName].skipEmit = false - return + registry[varName].skipEmit = false; + return; } websocketMsgSrv.updateAngularObject( registry[varName].noteId, registry[varName].paragraphId, varName, newValue, - registry[varName].interpreterGroupId) - }) + registry[varName].interpreterGroupId); + }); } - console.log('angular object (paragraph) created %o', varName) - scope[varName] = data.angularObject.object + console.log('angular object (paragraph) created %o', varName); + scope[varName] = data.angularObject.object; // create proxy for AngularFunction if (varName.indexOf(ANGULAR_FUNCTION_OBJECT_NAME_PREFIX) === 0) { - let funcName = varName.substring((ANGULAR_FUNCTION_OBJECT_NAME_PREFIX).length) - scope[funcName] = function () { + let funcName = varName.substring((ANGULAR_FUNCTION_OBJECT_NAME_PREFIX).length); + scope[funcName] = function() { // eslint-disable-next-line prefer-rest-params - scope[varName] = arguments + scope[varName] = arguments; // eslint-disable-next-line prefer-rest-params - console.log('angular function (paragraph) invoked %o', arguments) - } + console.log('angular function (paragraph) invoked %o', arguments); + }; - console.log('angular function (paragraph) created %o', scope[funcName]) + console.log('angular function (paragraph) created %o', scope[funcName]); } } - }) + }); - $scope.$on('angularObjectRemove', function (event, data) { - let noteId = $route.current.pathParams.noteId + $scope.$on('angularObjectRemove', function(event, data) { + let noteId = $route.current.pathParams.noteId; if (!data.noteId || data.noteId === noteId) { - let scope - let registry + let scope; + let registry; - let app = _.find($scope.apps, {id: data.paragraphId}) + let app = _.find($scope.apps, {id: data.paragraphId}); if (app) { - scope = getAppScope(app) - registry = getAppRegistry(app) + scope = getAppScope(app); + registry = getAppRegistry(app); } else { // no matching app in this paragraph - return + return; } - let varName = data.name + let varName = data.name; // clear watcher if (registry[varName]) { - registry[varName].clearWatcher() - registry[varName] = undefined + registry[varName].clearWatcher(); + registry[varName] = undefined; } // remove scope variable - scope[varName] = undefined + scope[varName] = undefined; // remove proxy for AngularFunction if (varName.indexOf(ANGULAR_FUNCTION_OBJECT_NAME_PREFIX) === 0) { - let funcName = varName.substring((ANGULAR_FUNCTION_OBJECT_NAME_PREFIX).length) - scope[funcName] = undefined + let funcName = varName.substring((ANGULAR_FUNCTION_OBJECT_NAME_PREFIX).length); + scope[funcName] = undefined; } } - }) + }); } diff --git a/zeppelin-web/src/app/notebook/revisions-comparator/revisions-comparator.component.js b/zeppelin-web/src/app/notebook/revisions-comparator/revisions-comparator.component.js index 45db38a44f8..debdb6e0fd7 100644 --- a/zeppelin-web/src/app/notebook/revisions-comparator/revisions-comparator.component.js +++ b/zeppelin-web/src/app/notebook/revisions-comparator/revisions-comparator.component.js @@ -12,81 +12,81 @@ * limitations under the License. */ -import revisionsComparatorTemplate from './revisions-comparator.html' -import './revisions-comparator.css' -import moment from 'moment' +import revisionsComparatorTemplate from './revisions-comparator.html'; +import './revisions-comparator.css'; +import moment from 'moment'; function RevisionsComparatorController($scope, websocketMsgSrv, $routeParams) { - 'ngInject' + 'ngInject'; - $scope.firstNoteRevisionForCompare = null - $scope.secondNoteRevisionForCompare = null - $scope.mergeNoteRevisionsForCompare = null - $scope.currentParagraphDiffDisplay = null - $scope.currentFirstRevisionForCompare = 'Choose...' - $scope.currentSecondRevisionForCompare = 'Choose...' + $scope.firstNoteRevisionForCompare = null; + $scope.secondNoteRevisionForCompare = null; + $scope.mergeNoteRevisionsForCompare = null; + $scope.currentParagraphDiffDisplay = null; + $scope.currentFirstRevisionForCompare = 'Choose...'; + $scope.currentSecondRevisionForCompare = 'Choose...'; - $scope.getNoteRevisionForReview = function (revision, position) { + $scope.getNoteRevisionForReview = function(revision, position) { if (position) { if (position === 'first') { - $scope.currentFirstRevisionForCompare = revision.message + $scope.currentFirstRevisionForCompare = revision.message; } else { - $scope.currentSecondRevisionForCompare = revision.message + $scope.currentSecondRevisionForCompare = revision.message; } - websocketMsgSrv.getNoteByRevisionForCompare($routeParams.noteId, revision.id, position) + websocketMsgSrv.getNoteByRevisionForCompare($routeParams.noteId, revision.id, position); } - } + }; // compare revisions - $scope.compareRevisions = function () { + $scope.compareRevisions = function() { if ($scope.firstNoteRevisionForCompare && $scope.secondNoteRevisionForCompare) { - let paragraphs1 = $scope.firstNoteRevisionForCompare.note.paragraphs - let paragraphs2 = $scope.secondNoteRevisionForCompare.note.paragraphs - let added = 'added' - let deleted = 'deleted' - let compared = 'compared' - let merge = [] + let paragraphs1 = $scope.firstNoteRevisionForCompare.note.paragraphs; + let paragraphs2 = $scope.secondNoteRevisionForCompare.note.paragraphs; + let added = 'added'; + let deleted = 'deleted'; + let compared = 'compared'; + let merge = []; for (let p1 of paragraphs1) { - let p2 = null + let p2 = null; for (let p of paragraphs2) { if (p1.id === p.id) { - p2 = p - break + p2 = p; + break; } } if (p2 === null) { - merge.push({paragraph: p1, firstString: (p1.text || '').split('\n')[0], type: deleted}) + merge.push({paragraph: p1, firstString: (p1.text || '').split('\n')[0], type: deleted}); } else { - let colorClass = '' - let span = null - let text1 = p1.text || '' - let text2 = p2.text || '' - - let diff = window.JsDiff.diffLines(text1, text2) - let diffHtml = document.createDocumentFragment() - let identical = true - let identicalClass = 'color-black' - - diff.forEach(function (part) { - colorClass = part.added ? 'color-green-row' : part.removed ? 'color-red-row' : identicalClass - span = document.createElement('span') - span.className = colorClass + let colorClass = ''; + let span = null; + let text1 = p1.text || ''; + let text2 = p2.text || ''; + + let diff = window.JsDiff.diffLines(text1, text2); + let diffHtml = document.createDocumentFragment(); + let identical = true; + let identicalClass = 'color-black'; + + diff.forEach(function(part) { + colorClass = part.added ? 'color-green-row' : part.removed ? 'color-red-row' : identicalClass; + span = document.createElement('span'); + span.className = colorClass; if (identical && colorClass !== identicalClass) { - identical = false + identical = false; } - let str = part.value + let str = part.value; if (str[str.length - 1] !== '\n') { - str = str + '\n' + str = str + '\n'; } - span.appendChild(document.createTextNode(str)) - diffHtml.appendChild(span) - }) + span.appendChild(document.createTextNode(str)); + diffHtml.appendChild(span); + }); - let pre = document.createElement('pre') - pre.appendChild(diffHtml) + let pre = document.createElement('pre'); + pre.appendChild(diffHtml); merge.push( { @@ -94,88 +94,88 @@ function RevisionsComparatorController($scope, websocketMsgSrv, $routeParams) { diff: pre.innerHTML, identical: identical, firstString: (p1.text || '').split('\n')[0], - type: compared - }) + type: compared, + }); } } for (let p2 of paragraphs2) { - let p1 = null + let p1 = null; for (let p of paragraphs1) { if (p2.id === p.id) { - p1 = p - break + p1 = p; + break; } } if (p1 === null) { - merge.push({paragraph: p2, firstString: (p2.text || '').split('\n')[0], type: added}) + merge.push({paragraph: p2, firstString: (p2.text || '').split('\n')[0], type: added}); } } - merge.sort(function (a, b) { + merge.sort(function(a, b) { if (a.type === added) { - return -1 + return -1; } if (a.type === compared) { - return 1 + return 1; } if (a.type === deleted) { if (b.type === compared) { - return -1 + return -1; } else { - return 1 + return 1; } } - }) + }); - $scope.mergeNoteRevisionsForCompare = merge + $scope.mergeNoteRevisionsForCompare = merge; if ($scope.currentParagraphDiffDisplay !== null) { - $scope.changeCurrentParagraphDiffDisplay($scope.currentParagraphDiffDisplay.paragraph.id) + $scope.changeCurrentParagraphDiffDisplay($scope.currentParagraphDiffDisplay.paragraph.id); } } - } + }; - $scope.$on('noteRevisionForCompare', function (event, data) { - console.debug('received note revision for compare %o', data) + $scope.$on('noteRevisionForCompare', function(event, data) { + console.debug('received note revision for compare %o', data); if (data.note && data.position) { if (data.position === 'first') { - $scope.firstNoteRevisionForCompare = data + $scope.firstNoteRevisionForCompare = data; } else { - $scope.secondNoteRevisionForCompare = data + $scope.secondNoteRevisionForCompare = data; } if ($scope.firstNoteRevisionForCompare !== null && $scope.secondNoteRevisionForCompare !== null && $scope.firstNoteRevisionForCompare.revisionId !== $scope.secondNoteRevisionForCompare.revisionId) { - $scope.compareRevisions() + $scope.compareRevisions(); } } - }) + }); - $scope.formatRevisionDate = function (date) { - return moment.unix(date).format('MMMM Do YYYY, h:mm:ss a') - } + $scope.formatRevisionDate = function(date) { + return moment.unix(date).format('MMMM Do YYYY, h:mm:ss a'); + }; - $scope.changeCurrentParagraphDiffDisplay = function (paragraphId) { + $scope.changeCurrentParagraphDiffDisplay = function(paragraphId) { for (let p of $scope.mergeNoteRevisionsForCompare) { if (p.paragraph.id === paragraphId) { - $scope.currentParagraphDiffDisplay = p - return + $scope.currentParagraphDiffDisplay = p; + return; } } - $scope.currentParagraphDiffDisplay = null - } + $scope.currentParagraphDiffDisplay = null; + }; } export const RevisionsComparatorComponent = { template: revisionsComparatorTemplate, controller: RevisionsComparatorController, bindings: { - noteRevisions: '<' - } -} + noteRevisions: '<', + }, +}; export const RevisionsComparatorModule = angular .module('zeppelinWebApp') .component('revisionsComparator', RevisionsComparatorComponent) - .name + .name; diff --git a/zeppelin-web/src/app/notebook/save-as/browser-detect.service.js b/zeppelin-web/src/app/notebook/save-as/browser-detect.service.js index a31e9afa2da..0c74b452204 100644 --- a/zeppelin-web/src/app/notebook/save-as/browser-detect.service.js +++ b/zeppelin-web/src/app/notebook/save-as/browser-detect.service.js @@ -12,28 +12,28 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').service('browserDetectService', BrowserDetectService) +angular.module('zeppelinWebApp').service('browserDetectService', BrowserDetectService); -function BrowserDetectService () { - this.detectIE = function () { - let ua = window.navigator.userAgent - let msie = ua.indexOf('MSIE ') +function BrowserDetectService() { + this.detectIE = function() { + let ua = window.navigator.userAgent; + let msie = ua.indexOf('MSIE '); if (msie > 0) { // IE 10 or older => return version number - return parseInt(ua.substring(msie + 5, ua.indexOf('.', msie)), 10) + return parseInt(ua.substring(msie + 5, ua.indexOf('.', msie)), 10); } - let trident = ua.indexOf('Trident/') + let trident = ua.indexOf('Trident/'); if (trident > 0) { // IE 11 => return version number - let rv = ua.indexOf('rv:') - return parseInt(ua.substring(rv + 3, ua.indexOf('.', rv)), 10) + let rv = ua.indexOf('rv:'); + return parseInt(ua.substring(rv + 3, ua.indexOf('.', rv)), 10); } - let edge = ua.indexOf('Edge/') + let edge = ua.indexOf('Edge/'); if (edge > 0) { // IE 12 (aka Edge) => return version number - return parseInt(ua.substring(edge + 5, ua.indexOf('.', edge)), 10) + return parseInt(ua.substring(edge + 5, ua.indexOf('.', edge)), 10); } // other browser - return false - } + return false; + }; } diff --git a/zeppelin-web/src/app/notebook/save-as/save-as.service.js b/zeppelin-web/src/app/notebook/save-as/save-as.service.js index ff463c85289..72a54d150a2 100644 --- a/zeppelin-web/src/app/notebook/save-as/save-as.service.js +++ b/zeppelin-web/src/app/notebook/save-as/save-as.service.js @@ -12,45 +12,45 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').service('saveAsService', SaveAsService) +angular.module('zeppelinWebApp').service('saveAsService', SaveAsService); -function SaveAsService (browserDetectService) { - 'ngInject' +function SaveAsService(browserDetectService) { + 'ngInject'; - this.saveAs = function (content, filename, extension) { - let BOM = '\uFEFF' + this.saveAs = function(content, filename, extension) { + let BOM = '\uFEFF'; if (browserDetectService.detectIE()) { - angular.element('body').append('') - let frameSaveAs = angular.element('body > iframe#SaveAsId')[0].contentWindow - content = BOM + content - frameSaveAs.document.open('text/json', 'replace') - frameSaveAs.document.write(content) - frameSaveAs.document.close() - frameSaveAs.focus() - let t1 = Date.now() - frameSaveAs.document.execCommand('SaveAs', false, filename + '.' + extension) - let t2 = Date.now() + angular.element('body').append(''); + let frameSaveAs = angular.element('body > iframe#SaveAsId')[0].contentWindow; + content = BOM + content; + frameSaveAs.document.open('text/json', 'replace'); + frameSaveAs.document.write(content); + frameSaveAs.document.close(); + frameSaveAs.focus(); + let t1 = Date.now(); + frameSaveAs.document.execCommand('SaveAs', false, filename + '.' + extension); + let t2 = Date.now(); // This means, this version of IE dosen't support auto download of a file with extension provided in param // falling back to ".txt" if (t1 === t2) { - frameSaveAs.document.execCommand('SaveAs', true, filename + '.txt') + frameSaveAs.document.execCommand('SaveAs', true, filename + '.txt'); } - angular.element('body > iframe#SaveAsId').remove() + angular.element('body > iframe#SaveAsId').remove(); } else { - let binaryData = [] - binaryData.push(BOM) - binaryData.push(content) - content = window.URL.createObjectURL(new Blob(binaryData)) + let binaryData = []; + binaryData.push(BOM); + binaryData.push(content); + content = window.URL.createObjectURL(new Blob(binaryData)); - angular.element('body').append('') - let saveAsElement = angular.element('body > a#SaveAsId') - saveAsElement.attr('href', content) - saveAsElement.attr('download', filename + '.' + extension) - saveAsElement.attr('target', '_blank') - saveAsElement[0].click() - saveAsElement.remove() - window.URL.revokeObjectURL(content) + angular.element('body').append(''); + let saveAsElement = angular.element('body > a#SaveAsId'); + saveAsElement.attr('href', content); + saveAsElement.attr('download', filename + '.' + extension); + saveAsElement.attr('target', '_blank'); + saveAsElement[0].click(); + saveAsElement.remove(); + window.URL.revokeObjectURL(content); } - } + }; } diff --git a/zeppelin-web/src/app/search/result-list.controller.js b/zeppelin-web/src/app/search/result-list.controller.js index 05be721dee0..65c10b1f7bf 100644 --- a/zeppelin-web/src/app/search/result-list.controller.js +++ b/zeppelin-web/src/app/search/result-list.controller.js @@ -12,107 +12,107 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').controller('SearchResultCtrl', SearchResultCtrl) +angular.module('zeppelinWebApp').controller('SearchResultCtrl', SearchResultCtrl); -function SearchResultCtrl ($scope, $routeParams, searchService) { - 'ngInject' +function SearchResultCtrl($scope, $routeParams, searchService) { + 'ngInject'; - $scope.isResult = true - $scope.searchTerm = $routeParams.searchTerm - let results = searchService.search({'q': $routeParams.searchTerm}).query() + $scope.isResult = true; + $scope.searchTerm = $routeParams.searchTerm; + let results = searchService.search({'q': $routeParams.searchTerm}).query(); - results.$promise.then(function (result) { - $scope.notes = result.body.map(function (note) { + results.$promise.then(function(result) { + $scope.notes = result.body.map(function(note) { // redirect to notebook when search result is a notebook itself, // not a paragraph if (!/\/paragraph\//.test(note.id)) { - return note + return note; } note.id = note.id.replace('paragraph/', '?paragraph=') + - '&term=' + $routeParams.searchTerm + '&term=' + $routeParams.searchTerm; - return note - }) + return note; + }); if ($scope.notes.length === 0) { - $scope.isResult = false + $scope.isResult = false; } else { - $scope.isResult = true + $scope.isResult = true; } - $scope.$on('$routeChangeStart', function (event, next, current) { + $scope.$on('$routeChangeStart', function(event, next, current) { if (next.originalPath !== '/search/:searchTerm') { - searchService.searchTerm = '' + searchService.searchTerm = ''; } - }) - }) + }); + }); - $scope.page = 0 - $scope.allResults = false + $scope.page = 0; + $scope.allResults = false; - $scope.highlightSearchResults = function (note) { - return function (_editor) { - function getEditorMode (text) { + $scope.highlightSearchResults = function(note) { + return function(_editor) { + function getEditorMode(text) { let editorModes = { 'ace/mode/scala': /^%(\w*\.)?spark/, 'ace/mode/python': /^%(\w*\.)?(pyspark|python)/, 'ace/mode/r': /^%(\w*\.)?(r|sparkr|knitr)/, 'ace/mode/sql': /^%(\w*\.)?\wql/, 'ace/mode/markdown': /^%md/, - 'ace/mode/sh': /^%sh/ - } + 'ace/mode/sh': /^%sh/, + }; - return Object.keys(editorModes).reduce(function (res, mode) { - return editorModes[mode].test(text) ? mode : res - }, 'ace/mode/scala') + return Object.keys(editorModes).reduce(function(res, mode) { + return editorModes[mode].test(text) ? mode : res; + }, 'ace/mode/scala'); } - let Range = ace.require('ace/range').Range + let Range = ace.require('ace/range').Range; - _editor.setOption('highlightActiveLine', false) - _editor.$blockScrolling = Infinity - _editor.setReadOnly(true) - _editor.renderer.setShowGutter(false) - _editor.setTheme('ace/theme/chrome') - _editor.getSession().setMode(getEditorMode(note.text)) + _editor.setOption('highlightActiveLine', false); + _editor.$blockScrolling = Infinity; + _editor.setReadOnly(true); + _editor.renderer.setShowGutter(false); + _editor.setTheme('ace/theme/chrome'); + _editor.getSession().setMode(getEditorMode(note.text)); - function getIndeces (term) { - return function (str) { - let indeces = [] - let i = -1 + function getIndeces(term) { + return function(str) { + let indeces = []; + let i = -1; while ((i = str.indexOf(term, i + 1)) >= 0) { - indeces.push(i) + indeces.push(i); } - return indeces - } + return indeces; + }; } - let result = '' + let result = ''; if (note.header !== '') { - result = note.header + '\n\n' + note.snippet + result = note.header + '\n\n' + note.snippet; } else { - result = note.snippet + result = note.snippet; } let lines = result .split('\n') - .map(function (line, row) { - let match = line.match(/(.+?)<\/B>/) + .map(function(line, row) { + let match = line.match(/(.+?)<\/B>/); // return early if nothing to highlight if (!match) { - return line + return line; } - let term = match[1] + let term = match[1]; let __line = line .replace(//g, '') - .replace(/<\/B>/g, '') + .replace(/<\/B>/g, ''); - let indeces = getIndeces(term)(__line) + let indeces = getIndeces(term)(__line); - indeces.forEach(function (start) { - let end = start + term.length + indeces.forEach(function(start) { + let end = start + term.length; if (note.header !== '' && row === 0) { _editor .getSession() @@ -120,14 +120,14 @@ function SearchResultCtrl ($scope, $routeParams, searchService) { new Range(row, 0, row, line.length), 'search-results-highlight-header', 'background' - ) + ); _editor .getSession() .addMarker( new Range(row, start, row, end), 'search-results-highlight', 'line' - ) + ); } else { _editor .getSession() @@ -135,20 +135,22 @@ function SearchResultCtrl ($scope, $routeParams, searchService) { new Range(row, start, row, end), 'search-results-highlight', 'line' - ) + ); } - }) - return __line - }) + }); + return __line; + }); // resize editor based on content length _editor.setOption( 'maxLines', - lines.reduce(function (len, line) { return len + line.length }, 0) - ) - - _editor.getSession().setValue(lines.join('\n')) - note.searchResult = lines - } - } + lines.reduce(function(len, line) { + return len + line.length; + }, 0) + ); + + _editor.getSession().setValue(lines.join('\n')); + note.searchResult = lines; + }; + }; } diff --git a/zeppelin-web/src/app/search/search.service.js b/zeppelin-web/src/app/search/search.service.js index fe4b6664363..248bacd2b98 100644 --- a/zeppelin-web/src/app/search/search.service.js +++ b/zeppelin-web/src/app/search/search.service.js @@ -12,22 +12,22 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').service('searchService', SearchService) +angular.module('zeppelinWebApp').service('searchService', SearchService); -function SearchService ($resource, baseUrlSrv) { - 'ngInject' +function SearchService($resource, baseUrlSrv) { + 'ngInject'; - this.search = function (term) { - this.searchTerm = term.q - console.log('Searching for: %o', term.q) + this.search = function(term) { + this.searchTerm = term.q; + console.log('Searching for: %o', term.q); if (!term.q) { // TODO(bzz): empty string check - return + return; } - let encQuery = window.encodeURIComponent(term.q) + let encQuery = window.encodeURIComponent(term.q); return $resource(baseUrlSrv.getRestApiBase() + '/notebook/search?q=' + encQuery, {}, { - query: {method: 'GET'} - }) - } + query: {method: 'GET'}, + }); + }; - this.searchTerm = '' + this.searchTerm = ''; } diff --git a/zeppelin-web/src/app/spell/index.js b/zeppelin-web/src/app/spell/index.js index ac4343c443e..8ec47532316 100644 --- a/zeppelin-web/src/app/spell/index.js +++ b/zeppelin-web/src/app/spell/index.js @@ -18,8 +18,8 @@ export { DefaultDisplayType, SpellResult, -} from './spell-result' +} from './spell-result'; export { SpellBase, -} from './spell-base' +} from './spell-base'; diff --git a/zeppelin-web/src/app/spell/spell-base.js b/zeppelin-web/src/app/spell/spell-base.js index 0b4216f6214..16e0553edc7 100644 --- a/zeppelin-web/src/app/spell/spell-base.js +++ b/zeppelin-web/src/app/spell/spell-base.js @@ -19,12 +19,12 @@ import { DefaultDisplayType, SpellResult, -} from './spell-result' +} from './spell-result'; /* eslint-enable no-unused-vars */ export class SpellBase { - constructor (magic) { - this.magic = magic + constructor(magic) { + this.magic = magic; } /** @@ -34,8 +34,8 @@ export class SpellBase { * @param config {Object} * @return {SpellResult} */ - interpret (paragraphText, config) { - throw new Error('SpellBase.interpret() should be overrided') + interpret(paragraphText, config) { + throw new Error('SpellBase.interpret() should be overrided'); } /** @@ -43,7 +43,7 @@ export class SpellBase { * (e.g `%flowchart`) * @return {string} */ - getMagic () { - return this.magic + getMagic() { + return this.magic; } } diff --git a/zeppelin-web/src/app/spell/spell-result.js b/zeppelin-web/src/app/spell/spell-result.js index 5ba65c28936..52bcdc1cdab 100644 --- a/zeppelin-web/src/app/spell/spell-result.js +++ b/zeppelin-web/src/app/spell/spell-result.js @@ -21,8 +21,8 @@ export const DefaultDisplayType = { HTML: 'HTML', ANGULAR: 'ANGULAR', TEXT: 'TEXT', - NETWORK: 'NETWORK' -} + NETWORK: 'NETWORK', +}; export const DefaultDisplayMagic = { '%element': DefaultDisplayType.ELEMENT, @@ -31,12 +31,12 @@ export const DefaultDisplayMagic = { '%angular': DefaultDisplayType.ANGULAR, '%text': DefaultDisplayType.TEXT, '%network': DefaultDisplayType.NETWORK, -} +}; export class DataWithType { - constructor (data, type, magic, text) { - this.data = data - this.type = type + constructor(data, type, magic, text) { + this.data = data; + this.type = type; /** * keep for `DefaultDisplayType.ELEMENT` (function data type) @@ -46,29 +46,29 @@ export class DataWithType { * since they don't have context where they are created. */ - this.magic = magic - this.text = text + this.magic = magic; + this.text = text; } - static handleDefaultMagic (m) { + static handleDefaultMagic(m) { // let's use default display type instead of magic in case of default // to keep consistency with backend interpreter if (DefaultDisplayMagic[m]) { - return DefaultDisplayMagic[m] + return DefaultDisplayMagic[m]; } else { - return m + return m; } } - static createPropagable (dataWithType) { + static createPropagable(dataWithType) { if (!SpellResult.isFunction(dataWithType.data)) { - return dataWithType + return dataWithType; } - const data = dataWithType.getText() - const type = dataWithType.getMagic() + const data = dataWithType.getText(); + const type = dataWithType.getMagic(); - return new DataWithType(data, type) + return new DataWithType(data, type); } /** @@ -77,45 +77,45 @@ export class DataWithType { * @param customDisplayType * @return {Array} */ - static parseStringData (data, customDisplayMagic) { - function availableMagic (magic) { - return magic && (DefaultDisplayMagic[magic] || customDisplayMagic[magic]) + static parseStringData(data, customDisplayMagic) { + function availableMagic(magic) { + return magic && (DefaultDisplayMagic[magic] || customDisplayMagic[magic]); } - const splited = data.split('\n') + const splited = data.split('\n'); - const gensWithTypes = [] - let mergedGens = [] - let previousMagic = DefaultDisplayType.TEXT + const gensWithTypes = []; + let mergedGens = []; + let previousMagic = DefaultDisplayType.TEXT; // create `DataWithType` whenever see available display type. for (let i = 0; i < splited.length; i++) { - const g = splited[i] - const magic = SpellResult.extractMagic(g) + const g = splited[i]; + const magic = SpellResult.extractMagic(g); // create `DataWithType` only if see new magic if (availableMagic(magic) && mergedGens.length > 0) { - gensWithTypes.push(new DataWithType(mergedGens.join(''), previousMagic)) - mergedGens = [] + gensWithTypes.push(new DataWithType(mergedGens.join(''), previousMagic)); + mergedGens = []; } // accumulate `data` to mergedGens if (availableMagic(magic)) { - const withoutMagic = g.split(magic)[1] - mergedGens.push(`${withoutMagic}\n`) - previousMagic = DataWithType.handleDefaultMagic(magic) + const withoutMagic = g.split(magic)[1]; + mergedGens.push(`${withoutMagic}\n`); + previousMagic = DataWithType.handleDefaultMagic(magic); } else { - mergedGens.push(`${g}\n`) + mergedGens.push(`${g}\n`); } } // cleanup the last `DataWithType` if (mergedGens.length > 0) { - previousMagic = DataWithType.handleDefaultMagic(previousMagic) - gensWithTypes.push(new DataWithType(mergedGens.join(''), previousMagic)) + previousMagic = DataWithType.handleDefaultMagic(previousMagic); + gensWithTypes.push(new DataWithType(mergedGens.join(''), previousMagic)); } - return gensWithTypes + return gensWithTypes; } /** @@ -128,44 +128,46 @@ export class DataWithType { * @param textWithoutMagic * @return {Promise>} */ - static produceMultipleData (dataWithType, customDisplayType, + static produceMultipleData(dataWithType, customDisplayType, magic, textWithoutMagic) { - const data = dataWithType.getData() - const type = dataWithType.getType() + const data = dataWithType.getData(); + const type = dataWithType.getType(); // if the type is specified, just return it // handle non-specified dataWithTypes only if (type) { - return new Promise((resolve) => { resolve([dataWithType]) }) + return new Promise((resolve) => { + resolve([dataWithType]); + }); } - let wrapped + let wrapped; if (SpellResult.isFunction(data)) { // if data is a function, we consider it as ELEMENT type. wrapped = new Promise((resolve) => { const dt = new DataWithType( - data, DefaultDisplayType.ELEMENT, magic, textWithoutMagic) - const result = [dt] - return resolve(result) - }) + data, DefaultDisplayType.ELEMENT, magic, textWithoutMagic); + const result = [dt]; + return resolve(result); + }); } else if (SpellResult.isPromise(data)) { // if data is a promise, - wrapped = data.then(generated => { + wrapped = data.then((generated) => { const result = - DataWithType.parseStringData(generated, customDisplayType) - return result - }) + DataWithType.parseStringData(generated, customDisplayType); + return result; + }); } else { // if data is a object, parse it to multiples wrapped = new Promise((resolve) => { const result = - DataWithType.parseStringData(data, customDisplayType) - return resolve(result) - }) + DataWithType.parseStringData(data, customDisplayType); + return resolve(result); + }); } - return wrapped + return wrapped; } /** @@ -177,8 +179,8 @@ export class DataWithType { * will be called in `then()` of this promise. * @returns {*} `data` which can be object, function or promise. */ - getData () { - return this.data + getData() { + return this.data; } /** @@ -187,66 +189,66 @@ export class DataWithType { * by `SpellResult.parseStringData()` * @returns {string} */ - getType () { - return this.type + getType() { + return this.type; } - getMagic () { - return this.magic + getMagic() { + return this.magic; } - getText () { - return this.text + getText() { + return this.text; } } export class SpellResult { - constructor (resultData, resultType) { - this.dataWithTypes = [] - this.add(resultData, resultType) + constructor(resultData, resultType) { + this.dataWithTypes = []; + this.add(resultData, resultType); } - static isFunction (data) { - return (data && typeof data === 'function') + static isFunction(data) { + return (data && typeof data === 'function'); } - static isPromise (data) { - return (data && typeof data.then === 'function') + static isPromise(data) { + return (data && typeof data.then === 'function'); } - static isObject (data) { + static isObject(data) { return (data && !SpellResult.isFunction(data) && - !SpellResult.isPromise(data)) + !SpellResult.isPromise(data)); } - static extractMagic (allParagraphText) { - const pattern = /^\s*%(\S+)\s*/g + static extractMagic(allParagraphText) { + const pattern = /^\s*%(\S+)\s*/g; try { - let match = pattern.exec(allParagraphText) + let match = pattern.exec(allParagraphText); if (match) { - return `%${match[1].trim()}` + return `%${match[1].trim()}`; } } catch (error) { // failed to parse, ignore } - return undefined + return undefined; } - static createPropagable (resultMsg) { - return resultMsg.map(dt => { - return DataWithType.createPropagable(dt) - }) + static createPropagable(resultMsg) { + return resultMsg.map((dt) => { + return DataWithType.createPropagable(dt); + }); } - add (resultData, resultType) { + add(resultData, resultType) { if (resultData) { this.dataWithTypes.push( - new DataWithType(resultData, resultType)) + new DataWithType(resultData, resultType)); } - return this + return this; } /** @@ -254,23 +256,23 @@ export class SpellResult { * @param textWithoutMagic * @return {Promise>} */ - getAllParsedDataWithTypes (customDisplayType, magic, textWithoutMagic) { - const promises = this.dataWithTypes.map(dt => { + getAllParsedDataWithTypes(customDisplayType, magic, textWithoutMagic) { + const promises = this.dataWithTypes.map((dt) => { return DataWithType.produceMultipleData( - dt, customDisplayType, magic, textWithoutMagic) - }) + dt, customDisplayType, magic, textWithoutMagic); + }); // some promises can include an array so we need to flatten them - const flatten = Promise.all(promises).then(values => { + const flatten = Promise.all(promises).then((values) => { return values.reduce((acc, cur) => { if (Array.isArray(cur)) { - return acc.concat(cur) + return acc.concat(cur); } else { - return acc.concat([cur]) + return acc.concat([cur]); } - }) - }) + }); + }); - return flatten + return flatten; } } diff --git a/zeppelin-web/src/app/tabledata/advanced-transformation-util.js b/zeppelin-web/src/app/tabledata/advanced-transformation-util.js index 0d1c2f657e0..97c1b2c1561 100644 --- a/zeppelin-web/src/app/tabledata/advanced-transformation-util.js +++ b/zeppelin-web/src/app/tabledata/advanced-transformation-util.js @@ -13,40 +13,40 @@ */ export function getCurrentChart(config) { - return config.chart.current + return config.chart.current; } export function getCurrentChartTransform(config) { - return config.spec.charts[getCurrentChart(config)].transform + return config.spec.charts[getCurrentChart(config)].transform; } export function getCurrentChartAxis(config) { - return config.axis[getCurrentChart(config)] + return config.axis[getCurrentChart(config)]; } export function getCurrentChartParam(config) { - return config.parameter[getCurrentChart(config)] + return config.parameter[getCurrentChart(config)]; } export function getCurrentChartAxisSpecs(config) { - return config.axisSpecs[getCurrentChart(config)] + return config.axisSpecs[getCurrentChart(config)]; } export function getCurrentChartParamSpecs(config) { - return config.paramSpecs[getCurrentChart(config)] + return config.paramSpecs[getCurrentChart(config)]; } export function useSharedAxis(config, chart) { - return config.spec.charts[chart].sharedAxis + return config.spec.charts[chart].sharedAxis; } export function serializeSharedAxes(config) { - const availableCharts = getAvailableChartNames(config.spec.charts) + const availableCharts = getAvailableChartNames(config.spec.charts); for (let i = 0; i < availableCharts.length; i++) { - const chartName = availableCharts[i] + const chartName = availableCharts[i]; if (useSharedAxis(config, chartName)) { /** use reference :) in case of sharedAxis */ - config.axis[chartName] = config.sharedAxis + config.axis[chartName] = config.sharedAxis; } } } @@ -56,22 +56,22 @@ export const Widget = { OPTION: 'option', CHECKBOX: 'checkbox', TEXTAREA: 'textarea', -} +}; export function isInputWidget(paramSpec) { - return (paramSpec && !paramSpec.widget) || (paramSpec && paramSpec.widget === Widget.INPUT) + return (paramSpec && !paramSpec.widget) || (paramSpec && paramSpec.widget === Widget.INPUT); } export function isOptionWidget(paramSpec) { - return paramSpec && paramSpec.widget === Widget.OPTION + return paramSpec && paramSpec.widget === Widget.OPTION; } export function isCheckboxWidget(paramSpec) { - return paramSpec && paramSpec.widget === Widget.CHECKBOX + return paramSpec && paramSpec.widget === Widget.CHECKBOX; } export function isTextareaWidget(paramSpec) { - return paramSpec && paramSpec.widget === Widget.TEXTAREA + return paramSpec && paramSpec.widget === Widget.TEXTAREA; } export const ParameterValueType = { @@ -80,59 +80,71 @@ export const ParameterValueType = { BOOLEAN: 'boolean', STRING: 'string', JSON: 'JSON', -} +}; export function parseParameter(paramSpecs, param) { /** copy original params */ - const parsed = JSON.parse(JSON.stringify(param)) + const parsed = JSON.parse(JSON.stringify(param)); for (let i = 0; i < paramSpecs.length; i++) { - const paramSpec = paramSpecs[i] - const name = paramSpec.name + const paramSpec = paramSpecs[i]; + const name = paramSpec.name; if (paramSpec.valueType === ParameterValueType.INT && typeof parsed[name] !== 'number') { - try { parsed[name] = parseInt(parsed[name]) } catch (error) { parsed[name] = paramSpec.defaultValue } + try { + parsed[name] = parseInt(parsed[name]); + } catch (error) { + parsed[name] = paramSpec.defaultValue; + } } else if (paramSpec.valueType === ParameterValueType.FLOAT && typeof parsed[name] !== 'number') { - try { parsed[name] = parseFloat(parsed[name]) } catch (error) { parsed[name] = paramSpec.defaultValue } + try { + parsed[name] = parseFloat(parsed[name]); + } catch (error) { + parsed[name] = paramSpec.defaultValue; + } } else if (paramSpec.valueType === ParameterValueType.BOOLEAN) { if (parsed[name] === 'false') { - parsed[name] = false + parsed[name] = false; } else if (parsed[name] === 'true') { - parsed[name] = true + parsed[name] = true; } else if (typeof parsed[name] !== 'boolean') { - parsed[name] = paramSpec.defaultValue + parsed[name] = paramSpec.defaultValue; } } else if (paramSpec.valueType === ParameterValueType.JSON) { if (parsed[name] !== null && typeof parsed[name] !== 'object') { - try { parsed[name] = JSON.parse(parsed[name]) } catch (error) { parsed[name] = paramSpec.defaultValue } + try { + parsed[name] = JSON.parse(parsed[name]); + } catch (error) { + parsed[name] = paramSpec.defaultValue; + } } else if (parsed[name] === null) { - parsed[name] = paramSpec.defaultValue + parsed[name] = paramSpec.defaultValue; } } } - return parsed + return parsed; } export const AxisType = { AGGREGATOR: 'aggregator', KEY: 'key', GROUP: 'group', -} +}; export function isAggregatorAxis(axisSpec) { - return axisSpec && axisSpec.axisType === AxisType.AGGREGATOR + return axisSpec && axisSpec.axisType === AxisType.AGGREGATOR; } export function isGroupAxis(axisSpec) { - return axisSpec && axisSpec.axisType === AxisType.GROUP + return axisSpec && axisSpec.axisType === AxisType.GROUP; } export function isKeyAxis(axisSpec) { - return axisSpec && axisSpec.axisType === AxisType.KEY + return axisSpec && axisSpec.axisType === AxisType.KEY; } export function isSingleDimensionAxis(axisSpec) { - return axisSpec && axisSpec.dimension === 'single' + return axisSpec && axisSpec.dimension === 'single'; } /** @@ -142,92 +154,112 @@ export function isSingleDimensionAxis(axisSpec) { * add the `name` field while converting to array to easily manipulate */ export function getSpecs(specObject) { - const specs = [] + const specs = []; for (let name in specObject) { - const singleSpec = specObject[name] - if (!singleSpec) { continue } - singleSpec.name = name - specs.push(singleSpec) + if (specObject.hasOwnProperty(name)) { + const singleSpec = specObject[name]; + if (!singleSpec) { + continue; + } + singleSpec.name = name; + specs.push(singleSpec); + } } - return specs + return specs; } export function getAvailableChartNames(charts) { - const available = [] + const available = []; for (let name in charts) { - available.push(name) + if (charts.hasOwnProperty(name)) { + available.push(name); + } } - return available + return available; } export function applyMaxAxisCount(config, axisSpec) { if (isSingleDimensionAxis(axisSpec) || typeof axisSpec.maxAxisCount === 'undefined') { - return + return; } - const columns = getCurrentChartAxis(config)[axisSpec.name] - if (columns.length <= axisSpec.maxAxisCount) { return } + const columns = getCurrentChartAxis(config)[axisSpec.name]; + if (columns.length <= axisSpec.maxAxisCount) { + return; + } - const sliced = columns.slice(1) - getCurrentChartAxis(config)[axisSpec.name] = sliced + const sliced = columns.slice(1); + getCurrentChartAxis(config)[axisSpec.name] = sliced; } export function removeDuplicatedColumnsInMultiDimensionAxis(config, axisSpec) { - if (isSingleDimensionAxis(axisSpec)) { return config } + if (isSingleDimensionAxis(axisSpec)) { + return config; + } - const columns = getCurrentChartAxis(config)[axisSpec.name] + const columns = getCurrentChartAxis(config)[axisSpec.name]; const uniqObject = columns.reduce((acc, col) => { - if (!acc[`${col.name}(${col.aggr})`]) { acc[`${col.name}(${col.aggr})`] = col } - return acc - }, {}) + if (!acc[`${col.name}(${col.aggr})`]) { + acc[`${col.name}(${col.aggr})`] = col; + } + return acc; + }, {}); - const filtered = [] + const filtered = []; for (let name in uniqObject) { - const col = uniqObject[name] - filtered.push(col) + if (uniqObject.hasOwnProperty(name)) { + const col = uniqObject[name]; + filtered.push(col); + } } - getCurrentChartAxis(config)[axisSpec.name] = filtered - return config + getCurrentChartAxis(config)[axisSpec.name] = filtered; + return config; } export function clearAxisConfig(config) { - delete config.axis /** Object: persisted axis for each chart */ - delete config.sharedAxis + delete config.axis; /** Object: persisted axis for each chart */ + delete config.sharedAxis; } export function initAxisConfig(config) { - if (!config.axis) { config.axis = {} } - if (!config.sharedAxis) { config.sharedAxis = {} } + if (!config.axis) { + config.axis = {}; + } + if (!config.sharedAxis) { + config.sharedAxis = {}; + } - const spec = config.spec - const availableCharts = getAvailableChartNames(spec.charts) + const spec = config.spec; + const availableCharts = getAvailableChartNames(spec.charts); - if (!config.axisSpecs) { config.axisSpecs = {} } + if (!config.axisSpecs) { + config.axisSpecs = {}; + } for (let i = 0; i < availableCharts.length; i++) { - const chartName = availableCharts[i] + const chartName = availableCharts[i]; if (!config.axis[chartName]) { - config.axis[chartName] = {} + config.axis[chartName] = {}; } - const axisSpecs = getSpecs(spec.charts[chartName].axis) + const axisSpecs = getSpecs(spec.charts[chartName].axis); if (!config.axisSpecs[chartName]) { - config.axisSpecs[chartName] = axisSpecs + config.axisSpecs[chartName] = axisSpecs; } /** initialize multi-dimension axes */ for (let i = 0; i < axisSpecs.length; i++) { - const axisSpec = axisSpecs[i] + const axisSpec = axisSpecs[i]; if (isSingleDimensionAxis(axisSpec)) { - continue + continue; } /** intentionally nested if-stmt is used because order of conditions matter here */ if (!useSharedAxis(config, chartName)) { if (!Array.isArray(config.axis[chartName][axisSpec.name])) { - config.axis[chartName][axisSpec.name] = [] + config.axis[chartName][axisSpec.name] = []; } } else if (useSharedAxis(config, chartName)) { /** @@ -235,180 +267,200 @@ export function initAxisConfig(config) { * all charts using shared axis have the same axis specs */ if (!Array.isArray(config.sharedAxis[axisSpec.name])) { - config.sharedAxis[axisSpec.name] = [] + config.sharedAxis[axisSpec.name] = []; } } } } /** this function should be called after initializing */ - serializeSharedAxes(config) + serializeSharedAxes(config); } export function resetAxisConfig(config) { - clearAxisConfig(config) - initAxisConfig(config) + clearAxisConfig(config); + initAxisConfig(config); } export function clearParameterConfig(config) { - delete config.parameter /** Object: persisted parameter for each chart */ + delete config.parameter; /** Object: persisted parameter for each chart */ } export function initParameterConfig(config) { - if (!config.parameter) { config.parameter = {} } + if (!config.parameter) { + config.parameter = {}; + } - const spec = config.spec - const availableCharts = getAvailableChartNames(spec.charts) + const spec = config.spec; + const availableCharts = getAvailableChartNames(spec.charts); - if (!config.paramSpecs) { config.paramSpecs = {} } + if (!config.paramSpecs) { + config.paramSpecs = {}; + } for (let i = 0; i < availableCharts.length; i++) { - const chartName = availableCharts[i] + const chartName = availableCharts[i]; - if (!config.parameter[chartName]) { config.parameter[chartName] = {} } - const paramSpecs = getSpecs(spec.charts[chartName].parameter) - if (!config.paramSpecs[chartName]) { config.paramSpecs[chartName] = paramSpecs } + if (!config.parameter[chartName]) { + config.parameter[chartName] = {}; + } + const paramSpecs = getSpecs(spec.charts[chartName].parameter); + if (!config.paramSpecs[chartName]) { + config.paramSpecs[chartName] = paramSpecs; + } for (let i = 0; i < paramSpecs.length; i++) { - const paramSpec = paramSpecs[i] + const paramSpec = paramSpecs[i]; if (!config.parameter[chartName][paramSpec.name]) { - config.parameter[chartName][paramSpec.name] = paramSpec.defaultValue + config.parameter[chartName][paramSpec.name] = paramSpec.defaultValue; } } } } export function resetParameterConfig(config) { - clearParameterConfig(config) - initParameterConfig(config) + clearParameterConfig(config); + initParameterConfig(config); } export function getSpecVersion(availableCharts, spec) { - const axisHash = {} - const paramHash = {} + const axisHash = {}; + const paramHash = {}; for (let i = 0; i < availableCharts.length; i++) { - const chartName = availableCharts[i] - const axisSpecs = getSpecs(spec.charts[chartName].axis) - axisHash[chartName] = axisSpecs + const chartName = availableCharts[i]; + const axisSpecs = getSpecs(spec.charts[chartName].axis); + axisHash[chartName] = axisSpecs; - const paramSpecs = getSpecs(spec.charts[chartName].parameter) - paramHash[chartName] = paramSpecs + const paramSpecs = getSpecs(spec.charts[chartName].parameter); + paramHash[chartName] = paramSpecs; } - return { axisVersion: JSON.stringify(axisHash), paramVersion: JSON.stringify(paramHash), } + return {axisVersion: JSON.stringify(axisHash), paramVersion: JSON.stringify(paramHash)}; } export function initializeConfig(config, spec) { - config.chartChanged = true - config.parameterChanged = false + config.chartChanged = true; + config.parameterChanged = false; - let updated = false + let updated = false; - const availableCharts = getAvailableChartNames(spec.charts) - const { axisVersion, paramVersion, } = getSpecVersion(availableCharts, spec) + const availableCharts = getAvailableChartNames(spec.charts); + const {axisVersion, paramVersion} = getSpecVersion(availableCharts, spec); if (!config.spec || !config.spec.version || !config.spec.version.axis || config.spec.version.axis !== axisVersion) { - spec.initialized = true - updated = true + spec.initialized = true; + updated = true; - delete config.chart /** Object: contains current, available chart */ - config.panel = { columnPanelOpened: true, parameterPanelOpened: false, } + delete config.chart; /** Object: contains current, available chart */ + config.panel = {columnPanelOpened: true, parameterPanelOpened: false}; - clearAxisConfig(config) - delete config.axisSpecs /** Object: persisted axisSpecs for each chart */ + clearAxisConfig(config); + delete config.axisSpecs; /** Object: persisted axisSpecs for each chart */ } if (!config.spec || !config.spec.version || !config.spec.version.parameter || config.spec.version.parameter !== paramVersion) { - updated = true + updated = true; - clearParameterConfig(config) - delete config.paramSpecs /** Object: persisted paramSpecs for each chart */ + clearParameterConfig(config); + delete config.paramSpecs; /** Object: persisted paramSpecs for each chart */ } - if (!spec.version) { spec.version = {} } - spec.version.axis = axisVersion - spec.version.parameter = paramVersion + if (!spec.version) { + spec.version = {}; + } + spec.version.axis = axisVersion; + spec.version.parameter = paramVersion; - if (!config.spec || updated) { config.spec = spec } + if (!config.spec || updated) { + config.spec = spec; + } if (!config.chart) { - config.chart = {} - config.chart.current = availableCharts[0] - config.chart.available = availableCharts + config.chart = {}; + config.chart.current = availableCharts[0]; + config.chart.available = availableCharts; } /** initialize config.axis, config.axisSpecs for each chart */ - initAxisConfig(config) + initAxisConfig(config); /** initialize config.parameter for each chart */ - initParameterConfig(config) - return config + initParameterConfig(config); + return config; } export function getColumnsForMultipleAxes(axisType, axisSpecs, axis) { - const axisNames = [] - let column = {} + const axisNames = []; + let column = {}; for (let i = 0; i < axisSpecs.length; i++) { - const axisSpec = axisSpecs[i] + const axisSpec = axisSpecs[i]; if (axisType === AxisType.KEY && isKeyAxis(axisSpec)) { - axisNames.push(axisSpec.name) + axisNames.push(axisSpec.name); } else if (axisType === AxisType.GROUP && isGroupAxis(axisSpec)) { - axisNames.push(axisSpec.name) + axisNames.push(axisSpec.name); } else if (axisType.AGGREGATOR && isAggregatorAxis(axisSpec)) { - axisNames.push(axisSpec.name) + axisNames.push(axisSpec.name); } } for (let axisName of axisNames) { - const columns = axis[axisName] - if (typeof axis[axisName] === 'undefined') { continue } - if (!column[axisName]) { column[axisName] = [] } - column[axisName] = column[axisName].concat(columns) + const columns = axis[axisName]; + if (typeof axis[axisName] === 'undefined') { + continue; + } + if (!column[axisName]) { + column[axisName] = []; + } + column[axisName] = column[axisName].concat(columns); } - return column + return column; } export function getColumnsFromAxis(axisSpecs, axis) { - const keyAxisNames = [] - const groupAxisNames = [] - const aggrAxisNames = [] + const keyAxisNames = []; + const groupAxisNames = []; + const aggrAxisNames = []; for (let i = 0; i < axisSpecs.length; i++) { - const axisSpec = axisSpecs[i] + const axisSpec = axisSpecs[i]; if (isKeyAxis(axisSpec)) { - keyAxisNames.push(axisSpec.name) + keyAxisNames.push(axisSpec.name); } else if (isGroupAxis(axisSpec)) { - groupAxisNames.push(axisSpec.name) + groupAxisNames.push(axisSpec.name); } else if (isAggregatorAxis(axisSpec)) { - aggrAxisNames.push(axisSpec.name) + aggrAxisNames.push(axisSpec.name); } } - let keyColumns = [] - let groupColumns = [] - let aggregatorColumns = [] - let customColumn = {} + let keyColumns = []; + let groupColumns = []; + let aggregatorColumns = []; + let customColumn = {}; for (let axisName in axis) { - const columns = axis[axisName] - if (keyAxisNames.includes(axisName)) { - keyColumns = keyColumns.concat(columns) - } else if (groupAxisNames.includes(axisName)) { - groupColumns = groupColumns.concat(columns) - } else if (aggrAxisNames.includes(axisName)) { - aggregatorColumns = aggregatorColumns.concat(columns) - } else { - const axisType = axisSpecs.filter(s => s.name === axisName)[0].axisType - if (!customColumn[axisType]) { customColumn[axisType] = [] } - customColumn[axisType] = customColumn[axisType].concat(columns) + if (axis.hasOwnProperty(axisName)) { + const columns = axis[axisName]; + if (keyAxisNames.includes(axisName)) { + keyColumns = keyColumns.concat(columns); + } else if (groupAxisNames.includes(axisName)) { + groupColumns = groupColumns.concat(columns); + } else if (aggrAxisNames.includes(axisName)) { + aggregatorColumns = aggregatorColumns.concat(columns); + } else { + const axisType = axisSpecs.filter((s) => s.name === axisName)[0].axisType; + if (!customColumn[axisType]) { + customColumn[axisType] = []; + } + customColumn[axisType] = customColumn[axisType].concat(columns); + } } } @@ -417,7 +469,7 @@ export function getColumnsFromAxis(axisSpecs, axis) { group: groupColumns, aggregator: aggregatorColumns, custom: customColumn, - } + }; } export const Aggregator = { @@ -426,7 +478,7 @@ export const Aggregator = { AVG: 'avg', MIN: 'min', MAX: 'max', -} +}; const TransformMethod = { /** @@ -449,38 +501,42 @@ const TransformMethod = { ARRAY: 'array', ARRAY_2_KEY: 'array:2-key', DRILL_DOWN: 'drill-down', -} +}; /** return function for lazy computation */ export function getTransformer(conf, rows, axisSpecs, axis) { - let transformer = () => {} + let transformer = () => {}; - const transformSpec = getCurrentChartTransform(conf) - if (!transformSpec) { return transformer } + const transformSpec = getCurrentChartTransform(conf); + if (!transformSpec) { + return transformer; + } - const method = transformSpec.method + const method = transformSpec.method; - const columns = getColumnsFromAxis(axisSpecs, axis) - const keyColumns = columns.key - const groupColumns = columns.group - const aggregatorColumns = columns.aggregator - const customColumns = columns.custom + const columns = getColumnsFromAxis(axisSpecs, axis); + const keyColumns = columns.key; + const groupColumns = columns.group; + const aggregatorColumns = columns.aggregator; + const customColumns = columns.custom; let column = { key: keyColumns, group: groupColumns, aggregator: aggregatorColumns, custom: customColumns, - } + }; if (method === TransformMethod.RAW) { - transformer = () => { return rows } + transformer = () => { + return rows; + }; } else if (method === TransformMethod.OBJECT) { transformer = () => { - const { cube, schema, keyColumnName, keyNames, groupNameSet, selectorNameWithIndex, } = - getKGACube(rows, keyColumns, groupColumns, aggregatorColumns) + const {cube, schema, keyColumnName, keyNames, groupNameSet, selectorNameWithIndex} = + getKGACube(rows, keyColumns, groupColumns, aggregatorColumns); const { - transformed, groupNames, sortedSelectors + transformed, groupNames, sortedSelectors, } = getObjectRowsFromKGACube(cube, schema, aggregatorColumns, - keyColumnName, keyNames, groupNameSet, selectorNameWithIndex) + keyColumnName, keyNames, groupNameSet, selectorNameWithIndex); return { rows: transformed, @@ -488,17 +544,17 @@ export function getTransformer(conf, rows, axisSpecs, axis) { keyNames, groupNames: groupNames, selectors: sortedSelectors, - } - } + }; + }; } else if (method === TransformMethod.ARRAY) { transformer = () => { - const { cube, schema, keyColumnName, keyNames, groupNameSet, selectorNameWithIndex, } = - getKGACube(rows, keyColumns, groupColumns, aggregatorColumns) + const {cube, schema, keyColumnName, keyNames, groupNameSet, selectorNameWithIndex} = + getKGACube(rows, keyColumns, groupColumns, aggregatorColumns); const { transformed, groupNames, sortedSelectors, } = getArrayRowsFromKGACube(cube, schema, aggregatorColumns, - keyColumnName, keyNames, groupNameSet, selectorNameWithIndex) + keyColumnName, keyNames, groupNameSet, selectorNameWithIndex); return { rows: transformed, @@ -506,34 +562,40 @@ export function getTransformer(conf, rows, axisSpecs, axis) { keyNames, groupNames: groupNames, selectors: sortedSelectors, - } - } + }; + }; } else if (method === TransformMethod.ARRAY_2_KEY) { - const keyAxisColumn = getColumnsForMultipleAxes(AxisType.KEY, axisSpecs, axis) - column.key = keyAxisColumn + const keyAxisColumn = getColumnsForMultipleAxes(AxisType.KEY, axisSpecs, axis); + column.key = keyAxisColumn; - let key1Columns = [] - let key2Columns = [] + let key1Columns = []; + let key2Columns = []; // since ARRAY_2_KEY :) - let i = 0 + let i = 0; for (let axisName in keyAxisColumn) { - if (i === 2) { break } + if (i === 2) { + break; + } - if (i === 0) { key1Columns = keyAxisColumn[axisName] } else if (i === 1) { key2Columns = keyAxisColumn[axisName] } - i++ + if (i === 0) { + key1Columns = keyAxisColumn[axisName]; + } else if (i === 1) { + key2Columns = keyAxisColumn[axisName]; + } + i++; } - const { cube, schema, + const {cube, schema, key1ColumnName, key1Names, key2ColumnName, key2Names, groupNameSet, selectorNameWithIndex, - } = getKKGACube(rows, key1Columns, key2Columns, groupColumns, aggregatorColumns) + } = getKKGACube(rows, key1Columns, key2Columns, groupColumns, aggregatorColumns); const { transformed, groupNames, sortedSelectors, key1NameWithIndex, key2NameWithIndex, } = getArrayRowsFromKKGACube(cube, schema, aggregatorColumns, - key1Names, key2Names, groupNameSet, selectorNameWithIndex) + key1Names, key2Names, groupNameSet, selectorNameWithIndex); transformer = () => { return { @@ -546,17 +608,17 @@ export function getTransformer(conf, rows, axisSpecs, axis) { key2NameWithIndex: key2NameWithIndex, groupNames: groupNames, selectors: sortedSelectors, - } - } + }; + }; } else if (method === TransformMethod.DRILL_DOWN) { transformer = () => { - const { cube, schema, keyColumnName, keyNames, groupNameSet, selectorNameWithIndex, } = - getKAGCube(rows, keyColumns, groupColumns, aggregatorColumns) + const {cube, schema, keyColumnName, keyNames, groupNameSet, selectorNameWithIndex} = + getKAGCube(rows, keyColumns, groupColumns, aggregatorColumns); const { transformed, groupNames, sortedSelectors, } = getDrilldownRowsFromKAGCube(cube, schema, aggregatorColumns, - keyColumnName, keyNames, groupNameSet, selectorNameWithIndex) + keyColumnName, keyNames, groupNameSet, selectorNameWithIndex); return { rows: transformed, @@ -564,48 +626,48 @@ export function getTransformer(conf, rows, axisSpecs, axis) { keyNames, groupNames: groupNames, selectors: sortedSelectors, - } - } + }; + }; } - return { transformer: transformer, column: column, } + return {transformer: transformer, column: column}; } const AggregatorFunctions = { sum: function(a, b) { - const varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0 - const varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0 - return varA + varB + const varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0; + const varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0; + return varA + varB; }, count: function(a, b) { - const varA = (a !== undefined) ? parseInt(a) : 0 - const varB = (b !== undefined) ? 1 : 0 - return varA + varB + const varA = (a !== undefined) ? parseInt(a) : 0; + const varB = (b !== undefined) ? 1 : 0; + return varA + varB; }, min: function(a, b) { - const varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0 - const varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0 - return Math.min(varA, varB) + const varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0; + const varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0; + return Math.min(varA, varB); }, max: function(a, b) { - const varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0 - const varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0 - return Math.max(varA, varB) + const varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0; + const varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0; + return Math.max(varA, varB); }, avg: function(a, b, c) { - const varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0 - const varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0 - return varA + varB - } -} + const varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0; + const varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0; + return varA + varB; + }, +}; const AggregatorFunctionDiv = { sum: false, min: false, max: false, count: false, - avg: true -} + avg: true, +}; /** nested cube `(key) -> (group) -> aggregator` */ export function getKGACube(rows, keyColumns, groupColumns, aggrColumns) { @@ -613,67 +675,75 @@ export function getKGACube(rows, keyColumns, groupColumns, aggrColumns) { key: keyColumns.length !== 0, group: groupColumns.length !== 0, aggregator: aggrColumns.length !== 0, - } + }; - let cube = {} - const entry = {} + let cube = {}; + const entry = {}; - const keyColumnName = keyColumns.map(c => c.name).join('.') - const groupNameSet = new Set() - const keyNameSet = new Set() - const selectorNameWithIndex = {} /** { selectorName: index } */ - let indexCounter = 0 + const keyColumnName = keyColumns.map((c) => c.name).join('.'); + const groupNameSet = new Set(); + const keyNameSet = new Set(); + const selectorNameWithIndex = {}; /** { selectorName: index } */ + let indexCounter = 0; for (let i = 0; i < rows.length; i++) { - const row = rows[i] - let e = entry - let c = cube + const row = rows[i]; + let e = entry; + let c = cube; // key: add to entry - let mergedKeyName + let mergedKeyName; if (schema.key) { - mergedKeyName = keyColumns.map(c => row[c.index]).join('.') - if (!e[mergedKeyName]) { e[mergedKeyName] = { children: {}, } } - e = e[mergedKeyName].children + mergedKeyName = keyColumns.map((c) => row[c.index]).join('.'); + if (!e[mergedKeyName]) { + e[mergedKeyName] = {children: {}}; + } + e = e[mergedKeyName].children; // key: add to row - if (!c[mergedKeyName]) { c[mergedKeyName] = {} } - c = c[mergedKeyName] + if (!c[mergedKeyName]) { + c[mergedKeyName] = {}; + } + c = c[mergedKeyName]; - keyNameSet.add(mergedKeyName) + keyNameSet.add(mergedKeyName); } - let mergedGroupName + let mergedGroupName; if (schema.group) { - mergedGroupName = groupColumns.map(c => row[c.index]).join('.') + mergedGroupName = groupColumns.map((c) => row[c.index]).join('.'); // add group to entry - if (!e[mergedGroupName]) { e[mergedGroupName] = { children: {}, } } - e = e[mergedGroupName].children + if (!e[mergedGroupName]) { + e[mergedGroupName] = {children: {}}; + } + e = e[mergedGroupName].children; // add group to row - if (!c[mergedGroupName]) { c[mergedGroupName] = {} } - c = c[mergedGroupName] - groupNameSet.add(mergedGroupName) + if (!c[mergedGroupName]) { + c[mergedGroupName] = {}; + } + c = c[mergedGroupName]; + groupNameSet.add(mergedGroupName); } for (let a = 0; a < aggrColumns.length; a++) { - const aggrColumn = aggrColumns[a] - const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})` + const aggrColumn = aggrColumns[a]; + const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})`; // update groupNameSet if (!mergedGroupName) { - groupNameSet.add(aggrName) /** aggr column name will be used as group name if group is empty */ + groupNameSet.add(aggrName); /** aggr column name will be used as group name if group is empty */ } // update selectorNameWithIndex - const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName) + const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName); if (typeof selectorNameWithIndex[selector] === 'undefined' /** value might be 0 */) { - selectorNameWithIndex[selector] = indexCounter - indexCounter = indexCounter + 1 + selectorNameWithIndex[selector] = indexCounter; + indexCounter = indexCounter + 1; } // add aggregator to entry if (!e[aggrName]) { - e[aggrName] = { type: 'aggregator', order: aggrColumn, index: aggrColumn.index, } + e[aggrName] = {type: 'aggregator', order: aggrColumn, index: aggrColumn.index}; } // add aggregatorName to row @@ -682,26 +752,26 @@ export function getKGACube(rows, keyColumns, groupColumns, aggrColumns) { aggr: aggrColumn.aggr, value: (aggrColumn.aggr !== 'count') ? row[aggrColumn.index] : 1, count: 1, - } + }; } else { const value = AggregatorFunctions[aggrColumn.aggr]( - c[aggrName].value, row[aggrColumn.index], c[aggrName].count + 1) + c[aggrName].value, row[aggrColumn.index], c[aggrName].count + 1); const count = (AggregatorFunctionDiv[aggrColumn.aggr]) - ? c[aggrName].count + 1 : c[aggrName].count + ? c[aggrName].count + 1 : c[aggrName].count; - c[aggrName].value = value - c[aggrName].count = count + c[aggrName].value = value; + c[aggrName].count = count; } } /** end loop for aggrColumns */ } - let keyNames = null + let keyNames = null; if (!schema.key) { - const mergedGroupColumnName = groupColumns.map(c => c.name).join('.') - cube = { [mergedGroupColumnName]: cube, } - keyNames = [ mergedGroupColumnName, ] + const mergedGroupColumnName = groupColumns.map((c) => c.name).join('.'); + cube = {[mergedGroupColumnName]: cube}; + keyNames = [mergedGroupColumnName]; } else { - keyNames = Object.keys(cube).sort() /** keys should be sorted */ + keyNames = Object.keys(cube).sort(); /** keys should be sorted */ } return { @@ -711,7 +781,7 @@ export function getKGACube(rows, keyColumns, groupColumns, aggrColumns) { keyNames: keyNames, groupNameSet: groupNameSet, selectorNameWithIndex: selectorNameWithIndex, - } + }; } /** nested cube `(key) -> aggregator -> (group)` for drill-down support */ @@ -720,98 +790,100 @@ export function getKAGCube(rows, keyColumns, groupColumns, aggrColumns) { key: keyColumns.length !== 0, group: groupColumns.length !== 0, aggregator: aggrColumns.length !== 0, - } + }; - let cube = {} + let cube = {}; - const keyColumnName = keyColumns.map(c => c.name).join('.') - const groupNameSet = new Set() - const keyNameSet = new Set() - const selectorNameWithIndex = {} /** { selectorName: index } */ - let indexCounter = 0 + const keyColumnName = keyColumns.map((c) => c.name).join('.'); + const groupNameSet = new Set(); + const keyNameSet = new Set(); + const selectorNameWithIndex = {}; /** { selectorName: index } */ + let indexCounter = 0; for (let i = 0; i < rows.length; i++) { - const row = rows[i] - let c = cube + const row = rows[i]; + let c = cube; // key: add to entry - let mergedKeyName + let mergedKeyName; if (schema.key) { - mergedKeyName = keyColumns.map(c => row[c.index]).join('.') + mergedKeyName = keyColumns.map((c) => row[c.index]).join('.'); // key: add to row - if (!c[mergedKeyName]) { c[mergedKeyName] = {} } - c = c[mergedKeyName] + if (!c[mergedKeyName]) { + c[mergedKeyName] = {}; + } + c = c[mergedKeyName]; - keyNameSet.add(mergedKeyName) + keyNameSet.add(mergedKeyName); } - let mergedGroupName + let mergedGroupName; if (schema.group) { - mergedGroupName = groupColumns.map(c => row[c.index]).join('.') - groupNameSet.add(mergedGroupName) + mergedGroupName = groupColumns.map((c) => row[c.index]).join('.'); + groupNameSet.add(mergedGroupName); } for (let a = 0; a < aggrColumns.length; a++) { - const aggrColumn = aggrColumns[a] - const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})` + const aggrColumn = aggrColumns[a]; + const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})`; // update groupNameSet if (!mergedGroupName) { - groupNameSet.add(aggrName) /** aggr column name will be used as group name if group is empty */ + groupNameSet.add(aggrName); /** aggr column name will be used as group name if group is empty */ } // update selectorNameWithIndex - const selector = getSelectorName(mergedKeyName, aggrColumns.length, aggrName) + const selector = getSelectorName(mergedKeyName, aggrColumns.length, aggrName); if (typeof selectorNameWithIndex[selector] === 'undefined' /** value might be 0 */) { - selectorNameWithIndex[selector] = indexCounter - indexCounter = indexCounter + 1 + selectorNameWithIndex[selector] = indexCounter; + indexCounter = indexCounter + 1; } // add aggregatorName to row if (!c[aggrName]) { - const value = (aggrColumn.aggr !== 'count') ? row[aggrColumn.index] : 1 - const count = 1 + const value = (aggrColumn.aggr !== 'count') ? row[aggrColumn.index] : 1; + const count = 1; - c[aggrName] = { aggr: aggrColumn.aggr, value: value, count: count, children: {}, } + c[aggrName] = {aggr: aggrColumn.aggr, value: value, count: count, children: {}}; } else { const value = AggregatorFunctions[aggrColumn.aggr]( - c[aggrName].value, row[aggrColumn.index], c[aggrName].count + 1) + c[aggrName].value, row[aggrColumn.index], c[aggrName].count + 1); const count = (AggregatorFunctionDiv[aggrColumn.aggr]) - ? c[aggrName].count + 1 : c[aggrName].count + ? c[aggrName].count + 1 : c[aggrName].count; - c[aggrName].value = value - c[aggrName].count = count + c[aggrName].value = value; + c[aggrName].count = count; } // add aggregated group (for drill-down) to row iff group is enabled if (mergedGroupName) { if (!c[aggrName].children[mergedGroupName]) { - const value = (aggrColumn.aggr !== 'count') ? row[aggrColumn.index] : 1 - const count = 1 + const value = (aggrColumn.aggr !== 'count') ? row[aggrColumn.index] : 1; + const count = 1; - c[aggrName].children[mergedGroupName] = { value: value, count: count, } + c[aggrName].children[mergedGroupName] = {value: value, count: count}; } else { - const drillDownedValue = c[aggrName].children[mergedGroupName].value - const drillDownedCount = c[aggrName].children[mergedGroupName].count + const drillDownedValue = c[aggrName].children[mergedGroupName].value; + const drillDownedCount = c[aggrName].children[mergedGroupName].count; const value = AggregatorFunctions[aggrColumn.aggr]( - drillDownedValue, row[aggrColumn.index], drillDownedCount + 1) + drillDownedValue, row[aggrColumn.index], drillDownedCount + 1); const count = (AggregatorFunctionDiv[aggrColumn.aggr]) - ? drillDownedCount + 1 : drillDownedCount + ? drillDownedCount + 1 : drillDownedCount; - c[aggrName].children[mergedGroupName].value = value - c[aggrName].children[mergedGroupName].count = count + c[aggrName].children[mergedGroupName].value = value; + c[aggrName].children[mergedGroupName].count = count; } } } /** end loop for aggrColumns */ } - let keyNames = null + let keyNames = null; if (!schema.key) { - const mergedGroupColumnName = groupColumns.map(c => c.name).join('.') - cube = { [mergedGroupColumnName]: cube, } - keyNames = [ mergedGroupColumnName, ] + const mergedGroupColumnName = groupColumns.map((c) => c.name).join('.'); + cube = {[mergedGroupColumnName]: cube}; + keyNames = [mergedGroupColumnName]; } else { - keyNames = Object.keys(cube).sort() /** keys should be sorted */ + keyNames = Object.keys(cube).sort(); /** keys should be sorted */ } return { @@ -821,7 +893,7 @@ export function getKAGCube(rows, keyColumns, groupColumns, aggrColumns) { keyNames: keyNames, groupNameSet: groupNameSet, selectorNameWithIndex: selectorNameWithIndex, - } + }; } /** nested cube `(key1) -> (key2) -> (group) -> aggregator` */ export function getKKGACube(rows, key1Columns, key2Columns, groupColumns, aggrColumns) { @@ -830,82 +902,98 @@ export function getKKGACube(rows, key1Columns, key2Columns, groupColumns, aggrCo key2: key2Columns.length !== 0, group: groupColumns.length !== 0, aggregator: aggrColumns.length !== 0, - } + }; - let cube = {} - const entry = {} + let cube = {}; + const entry = {}; - const key1ColumnName = key1Columns.map(c => c.name).join('.') - const key1NameSet = {} - const key2ColumnName = key2Columns.map(c => c.name).join('.') - const key2NameSet = {} - const groupNameSet = new Set() - const selectorNameWithIndex = {} /** { selectorName: index } */ - let indexCounter = 0 + const key1ColumnName = key1Columns.map((c) => c.name).join('.'); + const key1NameSet = {}; + const key2ColumnName = key2Columns.map((c) => c.name).join('.'); + const key2NameSet = {}; + const groupNameSet = new Set(); + const selectorNameWithIndex = {}; /** { selectorName: index } */ + let indexCounter = 0; for (let i = 0; i < rows.length; i++) { - const row = rows[i] - let e = entry - let c = cube + const row = rows[i]; + let e = entry; + let c = cube; // key1: add to entry - let mergedKey1Name + let mergedKey1Name; if (schema.key1) { - mergedKey1Name = key1Columns.map(c => row[c.index]).join('.') - if (!e[mergedKey1Name]) { e[mergedKey1Name] = { children: {}, } } - e = e[mergedKey1Name].children + mergedKey1Name = key1Columns.map((c) => row[c.index]).join('.'); + if (!e[mergedKey1Name]) { + e[mergedKey1Name] = {children: {}}; + } + e = e[mergedKey1Name].children; // key1: add to row - if (!c[mergedKey1Name]) { c[mergedKey1Name] = {} } - c = c[mergedKey1Name] + if (!c[mergedKey1Name]) { + c[mergedKey1Name] = {}; + } + c = c[mergedKey1Name]; - if (!key1NameSet[mergedKey1Name]) { key1NameSet[mergedKey1Name] = true } + if (!key1NameSet[mergedKey1Name]) { + key1NameSet[mergedKey1Name] = true; + } } // key2: add to entry - let mergedKey2Name + let mergedKey2Name; if (schema.key2) { - mergedKey2Name = key2Columns.map(c => row[c.index]).join('.') - if (!e[mergedKey2Name]) { e[mergedKey2Name] = { children: {}, } } - e = e[mergedKey2Name].children + mergedKey2Name = key2Columns.map((c) => row[c.index]).join('.'); + if (!e[mergedKey2Name]) { + e[mergedKey2Name] = {children: {}}; + } + e = e[mergedKey2Name].children; // key2: add to row - if (!c[mergedKey2Name]) { c[mergedKey2Name] = {} } - c = c[mergedKey2Name] + if (!c[mergedKey2Name]) { + c[mergedKey2Name] = {}; + } + c = c[mergedKey2Name]; - if (!key2NameSet[mergedKey2Name]) { key2NameSet[mergedKey2Name] = true } + if (!key2NameSet[mergedKey2Name]) { + key2NameSet[mergedKey2Name] = true; + } } - let mergedGroupName + let mergedGroupName; if (schema.group) { - mergedGroupName = groupColumns.map(c => row[c.index]).join('.') + mergedGroupName = groupColumns.map((c) => row[c.index]).join('.'); // add group to entry - if (!e[mergedGroupName]) { e[mergedGroupName] = { children: {}, } } - e = e[mergedGroupName].children + if (!e[mergedGroupName]) { + e[mergedGroupName] = {children: {}}; + } + e = e[mergedGroupName].children; // add group to row - if (!c[mergedGroupName]) { c[mergedGroupName] = {} } - c = c[mergedGroupName] - groupNameSet.add(mergedGroupName) + if (!c[mergedGroupName]) { + c[mergedGroupName] = {}; + } + c = c[mergedGroupName]; + groupNameSet.add(mergedGroupName); } for (let a = 0; a < aggrColumns.length; a++) { - const aggrColumn = aggrColumns[a] - const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})` + const aggrColumn = aggrColumns[a]; + const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})`; // update groupNameSet if (!mergedGroupName) { - groupNameSet.add(aggrName) /** aggr column name will be used as group name if group is empty */ + groupNameSet.add(aggrName); /** aggr column name will be used as group name if group is empty */ } // update selectorNameWithIndex - const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName) + const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName); if (typeof selectorNameWithIndex[selector] === 'undefined' /** value might be 0 */) { - selectorNameWithIndex[selector] = indexCounter - indexCounter = indexCounter + 1 + selectorNameWithIndex[selector] = indexCounter; + indexCounter = indexCounter + 1; } // add aggregator to entry if (!e[aggrName]) { - e[aggrName] = { type: 'aggregator', order: aggrColumn, index: aggrColumn.index, } + e[aggrName] = {type: 'aggregator', order: aggrColumn, index: aggrColumn.index}; } // add aggregatorName to row @@ -914,21 +1002,21 @@ export function getKKGACube(rows, key1Columns, key2Columns, groupColumns, aggrCo aggr: aggrColumn.aggr, value: (aggrColumn.aggr !== 'count') ? row[aggrColumn.index] : 1, count: 1, - } + }; } else { const value = AggregatorFunctions[aggrColumn.aggr]( - c[aggrName].value, row[aggrColumn.index], c[aggrName].count + 1) + c[aggrName].value, row[aggrColumn.index], c[aggrName].count + 1); const count = (AggregatorFunctionDiv[aggrColumn.aggr]) - ? c[aggrName].count + 1 : c[aggrName].count + ? c[aggrName].count + 1 : c[aggrName].count; - c[aggrName].value = value - c[aggrName].count = count + c[aggrName].value = value; + c[aggrName].count = count; } } /** end loop for aggrColumns */ } - let key1Names = Object.keys(key1NameSet).sort() /** keys should be sorted */ - let key2Names = Object.keys(key2NameSet).sort() /** keys should be sorted */ + let key1Names = Object.keys(key1NameSet).sort(); /** keys should be sorted */ + let key2Names = Object.keys(key2NameSet).sort(); /** keys should be sorted */ return { cube: cube, @@ -939,59 +1027,61 @@ export function getKKGACube(rows, key1Columns, key2Columns, groupColumns, aggrCo key2Names: key2Names, groupNameSet: groupNameSet, selectorNameWithIndex: selectorNameWithIndex, - } + }; } export function getSelectorName(mergedGroupName, aggrColumnLength, aggrColumnName) { if (!mergedGroupName) { - return aggrColumnName + return aggrColumnName; } else { return (aggrColumnLength > 1) - ? `${mergedGroupName} / ${aggrColumnName}` : mergedGroupName + ? `${mergedGroupName} / ${aggrColumnName}` : mergedGroupName; } } export function getCubeValue(obj, aggregator, aggrColumnName) { - let value = null /** default is null */ + let value = null; /** default is null */ try { /** if AVG or COUNT, calculate it now, previously we can't because we were doing accumulation */ if (aggregator === Aggregator.AVG) { - value = obj[aggrColumnName].value / obj[aggrColumnName].count + value = obj[aggrColumnName].value / obj[aggrColumnName].count; } else if (aggregator === Aggregator.COUNT) { - value = obj[aggrColumnName].value + value = obj[aggrColumnName].value; } else { - value = obj[aggrColumnName].value + value = obj[aggrColumnName].value; } - if (typeof value === 'undefined') { value = null } + if (typeof value === 'undefined') { + value = null; + } } catch (error) { /** iognore */ } - return value + return value; } export function getNameWithIndex(names) { - const nameWithIndex = {} + const nameWithIndex = {}; for (let i = 0; i < names.length; i++) { - const name = names[i] - nameWithIndex[name] = i + const name = names[i]; + nameWithIndex[name] = i; } - return nameWithIndex + return nameWithIndex; } export function getArrayRowsFromKKGACube(cube, schema, aggregatorColumns, key1Names, key2Names, groupNameSet, selectorNameWithIndex) { - const sortedSelectors = Object.keys(selectorNameWithIndex).sort() - const sortedSelectorNameWithIndex = getNameWithIndex(sortedSelectors) + const sortedSelectors = Object.keys(selectorNameWithIndex).sort(); + const sortedSelectorNameWithIndex = getNameWithIndex(sortedSelectors); - const selectorRows = new Array(sortedSelectors.length) - const key1NameWithIndex = getNameWithIndex(key1Names) - const key2NameWithIndex = getNameWithIndex(key2Names) + const selectorRows = new Array(sortedSelectors.length); + const key1NameWithIndex = getNameWithIndex(key1Names); + const key2NameWithIndex = getNameWithIndex(key2Names); fillSelectorRows(schema, cube, selectorRows, aggregatorColumns, sortedSelectorNameWithIndex, - key1Names, key2Names, key1NameWithIndex, key2NameWithIndex) + key1Names, key2Names, key1NameWithIndex, key2NameWithIndex); return { key1NameWithIndex: key1NameWithIndex, @@ -999,7 +1089,7 @@ export function getArrayRowsFromKKGACube(cube, schema, aggregatorColumns, transformed: selectorRows, groupNames: Array.from(groupNameSet).sort(), sortedSelectors: sortedSelectors, - } + }; } /** truly mutable style func. will return nothing */ @@ -1009,90 +1099,94 @@ export function fillSelectorRows(schema, cube, selectorRows, function fill(grouped, mergedGroupName, key1Name, key2Name) { // should iterate aggrColumns in the most nested loop to utilize memory locality for (let aggrColumn of aggrColumns) { - const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})` - const value = getCubeValue(grouped, aggrColumn.aggr, aggrName) - const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName) - const selectorIndex = selectorNameWithIndex[selector] + const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})`; + const value = getCubeValue(grouped, aggrColumn.aggr, aggrName); + const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName); + const selectorIndex = selectorNameWithIndex[selector]; if (typeof selectorRows[selectorIndex] === 'undefined') { - selectorRows[selectorIndex] = { selector: selector, value: [], } + selectorRows[selectorIndex] = {selector: selector, value: []}; } - const row = { aggregated: value, } + const row = {aggregated: value}; - if (typeof key1Name !== 'undefined') { row.key1 = key1Name } - if (typeof key2Name !== 'undefined') { row.key2 = key2Name } + if (typeof key1Name !== 'undefined') { + row.key1 = key1Name; + } + if (typeof key2Name !== 'undefined') { + row.key2 = key2Name; + } - selectorRows[selectorIndex].value.push(row) + selectorRows[selectorIndex].value.push(row); } } function iterateGroupNames(keyed, key1Name, key2Name) { if (!schema.group) { - fill(keyed, undefined, key1Name, key2Name) + fill(keyed, undefined, key1Name, key2Name); } else { // assuming sparse distribution (usual case) // otherwise we need to iterate using `groupNameSet` - const availableGroupNames = Object.keys(keyed) + const availableGroupNames = Object.keys(keyed); for (let groupName of availableGroupNames) { - const grouped = keyed[groupName] - fill(grouped, groupName, key1Name, key2Name) + const grouped = keyed[groupName]; + fill(grouped, groupName, key1Name, key2Name); } } } if (schema.key1 && schema.key2) { for (let key1Name of key1Names) { - const key1ed = cube[key1Name] + const key1ed = cube[key1Name]; // assuming sparse distribution (usual case) // otherwise we need to iterate using `key2Names` - const availableKey2Names = Object.keys(key1ed) + const availableKey2Names = Object.keys(key1ed); for (let key2Name of availableKey2Names) { - const keyed = key1ed[key2Name] - iterateGroupNames(keyed, key1Name, key2Name) + const keyed = key1ed[key2Name]; + iterateGroupNames(keyed, key1Name, key2Name); } } } else if (schema.key1 && !schema.key2) { for (let key1Name of key1Names) { - const keyed = cube[key1Name] - iterateGroupNames(keyed, key1Name, undefined) + const keyed = cube[key1Name]; + iterateGroupNames(keyed, key1Name, undefined); } } else if (!schema.key1 && schema.key2) { for (let key2Name of key2Names) { - const keyed = cube[key2Name] - iterateGroupNames(keyed, undefined, key2Name) + const keyed = cube[key2Name]; + iterateGroupNames(keyed, undefined, key2Name); } } else { - iterateGroupNames(cube, undefined, undefined) + iterateGroupNames(cube, undefined, undefined); } } export function getArrayRowsFromKGACube(cube, schema, aggregatorColumns, keyColumnName, keyNames, groupNameSet, selectorNameWithIndex) { - const sortedSelectors = Object.keys(selectorNameWithIndex).sort() - const sortedSelectorNameWithIndex = getNameWithIndex(sortedSelectors) + const sortedSelectors = Object.keys(selectorNameWithIndex).sort(); + const sortedSelectorNameWithIndex = getNameWithIndex(sortedSelectors); - const keyArrowRows = new Array(sortedSelectors.length) - const keyNameWithIndex = getNameWithIndex(keyNames) + const keyArrowRows = new Array(sortedSelectors.length); + const keyNameWithIndex = getNameWithIndex(keyNames); for (let i = 0; i < keyNames.length; i++) { - const key = keyNames[i] + const key = keyNames[i]; - const obj = cube[key] + const obj = cube[key]; fillArrayRow(schema, aggregatorColumns, obj, groupNameSet, sortedSelectorNameWithIndex, - key, keyNames, keyArrowRows, keyNameWithIndex) + key, keyNames, keyArrowRows, keyNameWithIndex); } return { transformed: keyArrowRows, groupNames: Array.from(groupNameSet).sort(), sortedSelectors: sortedSelectors, - } + }; } /** truly mutable style func. will return nothing, just modify `keyArrayRows` */ @@ -1100,34 +1194,34 @@ export function fillArrayRow(schema, aggrColumns, obj, groupNameSet, selectorNameWithIndex, keyName, keyNames, keyArrayRows, keyNameWithIndex) { function fill(target, mergedGroupName, aggr, aggrName) { - const value = getCubeValue(target, aggr, aggrName) - const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName) - const selectorIndex = selectorNameWithIndex[selector] - const keyIndex = keyNameWithIndex[keyName] + const value = getCubeValue(target, aggr, aggrName); + const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName); + const selectorIndex = selectorNameWithIndex[selector]; + const keyIndex = keyNameWithIndex[keyName]; if (typeof keyArrayRows[selectorIndex] === 'undefined') { keyArrayRows[selectorIndex] = { - selector: selector, value: new Array(keyNames.length) - } + selector: selector, value: new Array(keyNames.length), + }; } - keyArrayRows[selectorIndex].value[keyIndex] = value + keyArrayRows[selectorIndex].value[keyIndex] = value; } /** when group is empty */ if (!schema.group) { for (let i = 0; i < aggrColumns.length; i++) { - const aggrColumn = aggrColumns[i] - const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})` - fill(obj, undefined, aggrColumn.aggr, aggrName) + const aggrColumn = aggrColumns[i]; + const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})`; + fill(obj, undefined, aggrColumn.aggr, aggrName); } } else { for (let i = 0; i < aggrColumns.length; i++) { - const aggrColumn = aggrColumns[i] - const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})` + const aggrColumn = aggrColumns[i]; + const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})`; for (let groupName of groupNameSet) { - const grouped = obj[groupName] - fill(grouped, groupName, aggrColumn.aggr, aggrName) + const grouped = obj[groupName]; + fill(grouped, groupName, aggrColumn.aggr, aggrName); } } } @@ -1137,81 +1231,83 @@ export function getObjectRowsFromKGACube(cube, schema, aggregatorColumns, keyColumnName, keyNames, groupNameSet, selectorNameWithIndex) { const rows = keyNames.reduce((acc, key) => { - const obj = cube[key] - const row = getObjectRow(schema, aggregatorColumns, obj, groupNameSet) + const obj = cube[key]; + const row = getObjectRow(schema, aggregatorColumns, obj, groupNameSet); - if (schema.key) { row[keyColumnName] = key } - acc.push(row) + if (schema.key) { + row[keyColumnName] = key; + } + acc.push(row); - return acc - }, []) + return acc; + }, []); return { transformed: rows, sortedSelectors: Object.keys(selectorNameWithIndex).sort(), groupNames: Array.from(groupNameSet).sort(), - } + }; } export function getObjectRow(schema, aggrColumns, obj, groupNameSet) { - const row = {} + const row = {}; function fill(row, target, mergedGroupName, aggr, aggrName) { - const value = getCubeValue(target, aggr, aggrName) - const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName) - row[selector] = value + const value = getCubeValue(target, aggr, aggrName); + const selector = getSelectorName(mergedGroupName, aggrColumns.length, aggrName); + row[selector] = value; } /** when group is empty */ if (!schema.group) { for (let i = 0; i < aggrColumns.length; i++) { - const aggrColumn = aggrColumns[i] - const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})` + const aggrColumn = aggrColumns[i]; + const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})`; - fill(row, obj, undefined, aggrColumn.aggr, aggrName) + fill(row, obj, undefined, aggrColumn.aggr, aggrName); } - return row + return row; } /** when group is specified */ for (let i = 0; i < aggrColumns.length; i++) { - const aggrColumn = aggrColumns[i] - const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})` + const aggrColumn = aggrColumns[i]; + const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})`; for (let groupName of groupNameSet) { - const grouped = obj[groupName] + const grouped = obj[groupName]; if (grouped) { - fill(row, grouped, groupName, aggrColumn.aggr, aggrName) + fill(row, grouped, groupName, aggrColumn.aggr, aggrName); } } } - return row + return row; } export function getDrilldownRowsFromKAGCube(cube, schema, aggregatorColumns, keyColumnName, keyNames, groupNameSet, selectorNameWithIndex) { - const sortedSelectors = Object.keys(selectorNameWithIndex).sort() - const sortedSelectorNameWithIndex = getNameWithIndex(sortedSelectors) + const sortedSelectors = Object.keys(selectorNameWithIndex).sort(); + const sortedSelectorNameWithIndex = getNameWithIndex(sortedSelectors); - const rows = new Array(sortedSelectors.length) + const rows = new Array(sortedSelectors.length); - const groupNames = Array.from(groupNameSet).sort() + const groupNames = Array.from(groupNameSet).sort(); - keyNames.map(key => { - const obj = cube[key] + keyNames.map((key) => { + const obj = cube[key]; fillDrillDownRow(schema, obj, rows, key, - sortedSelectorNameWithIndex, aggregatorColumns, groupNames) - }) + sortedSelectorNameWithIndex, aggregatorColumns, groupNames); + }); return { transformed: rows, groupNames: groupNames, sortedSelectors: sortedSelectors, sortedSelectorNameWithIndex: sortedSelectorNameWithIndex, - } + }; } /** truly mutable style func. will return nothing, just modify `rows` */ @@ -1219,27 +1315,27 @@ export function fillDrillDownRow(schema, obj, rows, key, selectorNameWithIndex, aggrColumns, groupNames) { /** when group is empty */ for (let i = 0; i < aggrColumns.length; i++) { - const row = {} - const aggrColumn = aggrColumns[i] - const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})` + const row = {}; + const aggrColumn = aggrColumns[i]; + const aggrName = `${aggrColumn.name}(${aggrColumn.aggr})`; - const value = getCubeValue(obj, aggrColumn.aggr, aggrName) - const selector = getSelectorName((schema.key) ? key : undefined, aggrColumns.length, aggrName) + const value = getCubeValue(obj, aggrColumn.aggr, aggrName); + const selector = getSelectorName((schema.key) ? key : undefined, aggrColumns.length, aggrName); - const selectorIndex = selectorNameWithIndex[selector] - row.value = value - row.drillDown = [] - row.selector = selector + const selectorIndex = selectorNameWithIndex[selector]; + row.value = value; + row.drillDown = []; + row.selector = selector; if (schema.group) { - row.drillDown = [] + row.drillDown = []; for (let groupName of groupNames) { - const value = getCubeValue(obj[aggrName].children, aggrColumn.aggr, groupName) - row.drillDown.push({ group: groupName, value: value, }) + const value = getCubeValue(obj[aggrName].children, aggrColumn.aggr, groupName); + row.drillDown.push({group: groupName, value: value}); } } - rows[selectorIndex] = row + rows[selectorIndex] = row; } } diff --git a/zeppelin-web/src/app/tabledata/advanced-transformation-util.test.js b/zeppelin-web/src/app/tabledata/advanced-transformation-util.test.js index 90f569fade4..84ea4419dbc 100644 --- a/zeppelin-web/src/app/tabledata/advanced-transformation-util.test.js +++ b/zeppelin-web/src/app/tabledata/advanced-transformation-util.test.js @@ -12,1728 +12,1730 @@ * limitations under the License. */ -import * as Util from './advanced-transformation-util.js' +import * as Util from './advanced-transformation-util.js'; /* eslint-disable max-len */ const MockParameter = { - 'floatParam': { valueType: 'float', defaultValue: 10, description: '', }, - 'intParam': { valueType: 'int', defaultValue: 50, description: '', }, - 'jsonParam': { valueType: 'JSON', defaultValue: '', description: '', widget: 'textarea', }, - 'stringParam1': { valueType: 'string', defaultValue: '', description: '', }, - 'stringParam2': { valueType: 'string', defaultValue: '', description: '', widget: 'input', }, - 'boolParam': { valueType: 'boolean', defaultValue: false, description: '', widget: 'checkbox', }, - 'optionParam': { valueType: 'string', defaultValue: 'line', description: '', widget: 'option', optionValues: [ 'line', 'smoothedLine', ], }, -} + 'floatParam': {valueType: 'float', defaultValue: 10, description: ''}, + 'intParam': {valueType: 'int', defaultValue: 50, description: ''}, + 'jsonParam': {valueType: 'JSON', defaultValue: '', description: '', widget: 'textarea'}, + 'stringParam1': {valueType: 'string', defaultValue: '', description: ''}, + 'stringParam2': {valueType: 'string', defaultValue: '', description: '', widget: 'input'}, + 'boolParam': {valueType: 'boolean', defaultValue: false, description: '', widget: 'checkbox'}, + 'optionParam': {valueType: 'string', defaultValue: 'line', description: '', widget: 'option', optionValues: ['line', 'smoothedLine']}, +}; /* eslint-enable max-len */ const MockAxis1 = { - 'keyAxis': { dimension: 'multiple', axisType: 'key', }, - 'aggrAxis': { dimension: 'multiple', axisType: 'aggregator', }, - 'groupAxis': { dimension: 'multiple', axisType: 'group', }, -} + 'keyAxis': {dimension: 'multiple', axisType: 'key'}, + 'aggrAxis': {dimension: 'multiple', axisType: 'aggregator'}, + 'groupAxis': {dimension: 'multiple', axisType: 'group'}, +}; const MockAxis2 = { - 'singleKeyAxis': { dimension: 'single', axisType: 'key', }, - 'limitedAggrAxis': { dimension: 'multiple', axisType: 'aggregator', maxAxisCount: 2, }, - 'groupAxis': { dimension: 'multiple', axisType: 'group', }, -} + 'singleKeyAxis': {dimension: 'single', axisType: 'key'}, + 'limitedAggrAxis': {dimension: 'multiple', axisType: 'aggregator', maxAxisCount: 2}, + 'groupAxis': {dimension: 'multiple', axisType: 'group'}, +}; const MockAxis3 = { - 'customAxis1': { dimension: 'single', axisType: 'unique', }, - 'customAxis2': { dimension: 'multiple', axisType: 'value', }, -} + 'customAxis1': {dimension: 'single', axisType: 'unique'}, + 'customAxis2': {dimension: 'multiple', axisType: 'value'}, +}; const MockAxis4 = { - 'key1Axis': { dimension: 'multiple', axisType: 'key', }, - 'key2Axis': { dimension: 'multiple', axisType: 'key', }, - 'aggrAxis': { dimension: 'multiple', axisType: 'aggregator', }, - 'groupAxis': { dimension: 'multiple', axisType: 'group', }, -} + 'key1Axis': {dimension: 'multiple', axisType: 'key'}, + 'key2Axis': {dimension: 'multiple', axisType: 'key'}, + 'aggrAxis': {dimension: 'multiple', axisType: 'aggregator'}, + 'groupAxis': {dimension: 'multiple', axisType: 'group'}, +}; // test spec for axis, param, widget const MockSpec = { charts: { 'object-chart': { - transform: { method: 'object', }, + transform: {method: 'object'}, sharedAxis: true, axis: JSON.parse(JSON.stringify(MockAxis1)), parameter: MockParameter, }, 'array-chart': { - transform: { method: 'array', }, + transform: {method: 'array'}, sharedAxis: true, axis: JSON.parse(JSON.stringify(MockAxis1)), parameter: { - 'arrayChartParam0': { valueType: 'string', defaultValue: '', description: 'param0', }, + 'arrayChartParam0': {valueType: 'string', defaultValue: '', description: 'param0'}, }, }, 'drillDown-chart': { - transform: { method: 'drill-down', }, + transform: {method: 'drill-down'}, axis: JSON.parse(JSON.stringify(MockAxis2)), parameter: { - 'drillDownChartParam0': { valueType: 'string', defaultValue: '', description: 'param0', }, + 'drillDownChartParam0': {valueType: 'string', defaultValue: '', description: 'param0'}, }, }, 'raw-chart': { - transform: { method: 'raw', }, + transform: {method: 'raw'}, axis: JSON.parse(JSON.stringify(MockAxis3)), parameter: { - 'rawChartParam0': { valueType: 'string', defaultValue: '', description: 'param0', }, + 'rawChartParam0': {valueType: 'string', defaultValue: '', description: 'param0'}, }, }, }, -} +}; // test spec for transformation const MockSpec2 = { charts: { 'object-chart': { - transform: { method: 'object', }, + transform: {method: 'object'}, sharedAxis: false, axis: JSON.parse(JSON.stringify(MockAxis1)), parameter: MockParameter, }, 'array-chart': { - transform: { method: 'array', }, + transform: {method: 'array'}, sharedAxis: false, axis: JSON.parse(JSON.stringify(MockAxis1)), parameter: { - 'arrayChartParam0': { valueType: 'string', defaultValue: '', description: 'param0', }, + 'arrayChartParam0': {valueType: 'string', defaultValue: '', description: 'param0'}, }, }, 'drillDown-chart': { - transform: { method: 'drill-down', }, + transform: {method: 'drill-down'}, sharedAxis: false, axis: JSON.parse(JSON.stringify(MockAxis1)), parameter: { - 'drillDownChartParam0': { valueType: 'string', defaultValue: '', description: 'param0', }, + 'drillDownChartParam0': {valueType: 'string', defaultValue: '', description: 'param0'}, }, }, 'array2Key-chart': { - transform: { method: 'array:2-key', }, + transform: {method: 'array:2-key'}, sharedAxis: false, axis: JSON.parse(JSON.stringify(MockAxis4)), parameter: { - 'drillDownChartParam0': { valueType: 'string', defaultValue: '', description: 'param0', }, + 'drillDownChartParam0': {valueType: 'string', defaultValue: '', description: 'param0'}, }, }, 'raw-chart': { - transform: { method: 'raw', }, + transform: {method: 'raw'}, sharedAxis: false, axis: JSON.parse(JSON.stringify(MockAxis3)), parameter: { - 'rawChartParam0': { valueType: 'string', defaultValue: '', description: 'param0', }, + 'rawChartParam0': {valueType: 'string', defaultValue: '', description: 'param0'}, }, }, }, -} +}; /* eslint-disable max-len */ const MockTableDataColumn = [ - {'name': 'age', 'index': 0, 'aggr': 'sum', }, - {'name': 'job', 'index': 1, 'aggr': 'sum', }, - {'name': 'marital', 'index': 2, 'aggr': 'sum', }, - {'name': 'education', 'index': 3, 'aggr': 'sum', }, - {'name': 'default', 'index': 4, 'aggr': 'sum', }, - {'name': 'balance', 'index': 5, 'aggr': 'sum', }, - {'name': 'housing', 'index': 6, 'aggr': 'sum', }, - {'name': 'loan', 'index': 7, 'aggr': 'sum', }, - {'name': 'contact', 'index': 8, 'aggr': 'sum', }, - {'name': 'day', 'index': 9, 'aggr': 'sum', }, - {'name': 'month', 'index': 10, 'aggr': 'sum', }, - {'name': 'duration', 'index': 11, 'aggr': 'sum', }, - {'name': 'campaign', 'index': 12, 'aggr': 'sum', }, - {'name': 'pdays', 'index': 13, 'aggr': 'sum', }, - {'name': 'previous', 'index': 14, 'aggr': 'sum', }, - {'name': 'poutcome', 'index': 15, 'aggr': 'sum', }, - {'name': 'y', 'index': 16, 'aggr': 'sum', } -] + {'name': 'age', 'index': 0, 'aggr': 'sum'}, + {'name': 'job', 'index': 1, 'aggr': 'sum'}, + {'name': 'marital', 'index': 2, 'aggr': 'sum'}, + {'name': 'education', 'index': 3, 'aggr': 'sum'}, + {'name': 'default', 'index': 4, 'aggr': 'sum'}, + {'name': 'balance', 'index': 5, 'aggr': 'sum'}, + {'name': 'housing', 'index': 6, 'aggr': 'sum'}, + {'name': 'loan', 'index': 7, 'aggr': 'sum'}, + {'name': 'contact', 'index': 8, 'aggr': 'sum'}, + {'name': 'day', 'index': 9, 'aggr': 'sum'}, + {'name': 'month', 'index': 10, 'aggr': 'sum'}, + {'name': 'duration', 'index': 11, 'aggr': 'sum'}, + {'name': 'campaign', 'index': 12, 'aggr': 'sum'}, + {'name': 'pdays', 'index': 13, 'aggr': 'sum'}, + {'name': 'previous', 'index': 14, 'aggr': 'sum'}, + {'name': 'poutcome', 'index': 15, 'aggr': 'sum'}, + {'name': 'y', 'index': 16, 'aggr': 'sum'}, +]; const MockTableDataRows1 = [ - [ '44', 'services', 'single', 'tertiary', 'no', '106', 'no', 'no', 'unknown', '12', 'jun', '109', '2', '-1', '0', 'unknown', 'no' ], - [ '43', 'services', 'married', 'primary', 'no', '-88', 'yes', 'yes', 'cellular', '17', 'apr', '313', '1', '147', '2', 'failure', 'no' ], - [ '39', 'services', 'married', 'secondary', 'no', '9374', 'yes', 'no', 'unknown', '20', 'may', '273', '1', '-1', '0', 'unknown', 'no' ], - [ '33', 'services', 'single', 'tertiary', 'no', '4789', 'yes', 'yes', 'cellular', '11', 'may', '220', '1', '339', '4', 'failure', 'no' ], -] + ['44', 'services', 'single', 'tertiary', 'no', '106', 'no', 'no', 'unknown', '12', 'jun', '109', '2', '-1', '0', 'unknown', 'no'], + ['43', 'services', 'married', 'primary', 'no', '-88', 'yes', 'yes', 'cellular', '17', 'apr', '313', '1', '147', '2', 'failure', 'no'], + ['39', 'services', 'married', 'secondary', 'no', '9374', 'yes', 'no', 'unknown', '20', 'may', '273', '1', '-1', '0', 'unknown', 'no'], + ['33', 'services', 'single', 'tertiary', 'no', '4789', 'yes', 'yes', 'cellular', '11', 'may', '220', '1', '339', '4', 'failure', 'no'], +]; /* eslint-enable max-len */ describe('advanced-transformation-util', () => { describe('getCurrent* funcs', () => { it('should set return proper value of the current chart', () => { - const config = {} - const spec = JSON.parse(JSON.stringify(MockSpec)) - Util.initializeConfig(config, spec) - expect(Util.getCurrentChart(config)).toEqual('object-chart') - expect(Util.getCurrentChartTransform(config)).toEqual({method: 'object'}) + const config = {}; + const spec = JSON.parse(JSON.stringify(MockSpec)); + Util.initializeConfig(config, spec); + expect(Util.getCurrentChart(config)).toEqual('object-chart'); + expect(Util.getCurrentChartTransform(config)).toEqual({method: 'object'}); // use `toBe` to compare reference - expect(Util.getCurrentChartAxis(config)).toBe(config.axis['object-chart']) + expect(Util.getCurrentChartAxis(config)).toBe(config.axis['object-chart']); // use `toBe` to compare reference - expect(Util.getCurrentChartParam(config)).toBe(config.parameter['object-chart']) - }) - }) + expect(Util.getCurrentChartParam(config)).toBe(config.parameter['object-chart']); + }); + }); describe('useSharedAxis', () => { it('should set chartChanged for initial drawing', () => { - const config = {} - const spec = JSON.parse(JSON.stringify(MockSpec)) - Util.initializeConfig(config, spec) - expect(Util.useSharedAxis(config, 'object-chart')).toEqual(true) - expect(Util.useSharedAxis(config, 'array-chart')).toEqual(true) - expect(Util.useSharedAxis(config, 'drillDown-chart')).toBeUndefined() - expect(Util.useSharedAxis(config, 'raw-chart')).toBeUndefined() - }) - }) + const config = {}; + const spec = JSON.parse(JSON.stringify(MockSpec)); + Util.initializeConfig(config, spec); + expect(Util.useSharedAxis(config, 'object-chart')).toEqual(true); + expect(Util.useSharedAxis(config, 'array-chart')).toEqual(true); + expect(Util.useSharedAxis(config, 'drillDown-chart')).toBeUndefined(); + expect(Util.useSharedAxis(config, 'raw-chart')).toBeUndefined(); + }); + }); describe('initializeConfig', () => { - const config = {} - const spec = JSON.parse(JSON.stringify(MockSpec)) - Util.initializeConfig(config, spec) + const config = {}; + const spec = JSON.parse(JSON.stringify(MockSpec)); + Util.initializeConfig(config, spec); it('should set chartChanged for initial drawing', () => { - expect(config.chartChanged).toBe(true) - expect(config.parameterChanged).toBe(false) - }) + expect(config.chartChanged).toBe(true); + expect(config.parameterChanged).toBe(false); + }); it('should set panel toggles ', () => { - expect(config.panel.columnPanelOpened).toBe(true) - expect(config.panel.parameterPanelOpened).toBe(false) - }) + expect(config.panel.columnPanelOpened).toBe(true); + expect(config.panel.parameterPanelOpened).toBe(false); + }); it('should set version and initialized', () => { - expect(config.spec.version).toBeDefined() - expect(config.spec.initialized).toBe(true) - }) + expect(config.spec.version).toBeDefined(); + expect(config.spec.initialized).toBe(true); + }); it('should set chart', () => { - expect(config.chart.current).toBe('object-chart') + expect(config.chart.current).toBe('object-chart'); expect(config.chart.available).toEqual([ 'object-chart', 'array-chart', 'drillDown-chart', 'raw-chart', - ]) - }) + ]); + }); it('should set sharedAxis', () => { expect(config.sharedAxis).toEqual({ keyAxis: [], aggrAxis: [], groupAxis: [], - }) + }); // should use `toBe` to compare object reference - expect(config.sharedAxis).toBe(config.axis['object-chart']) + expect(config.sharedAxis).toBe(config.axis['object-chart']); // should use `toBe` to compare object reference - expect(config.sharedAxis).toBe(config.axis['array-chart']) - }) + expect(config.sharedAxis).toBe(config.axis['array-chart']); + }); it('should set paramSpecs', () => { - const expected = Util.getSpecs(MockParameter) - expect(config.paramSpecs['object-chart']).toEqual(expected) - expect(config.paramSpecs['array-chart'].length).toEqual(1) - expect(config.paramSpecs['drillDown-chart'].length).toEqual(1) - expect(config.paramSpecs['raw-chart'].length).toEqual(1) - }) + const expected = Util.getSpecs(MockParameter); + expect(config.paramSpecs['object-chart']).toEqual(expected); + expect(config.paramSpecs['array-chart'].length).toEqual(1); + expect(config.paramSpecs['drillDown-chart'].length).toEqual(1); + expect(config.paramSpecs['raw-chart'].length).toEqual(1); + }); it('should set parameter with default value', () => { - expect(Object.keys(MockParameter).length).toBeGreaterThan(0) // length > 0 + expect(Object.keys(MockParameter).length).toBeGreaterThan(0); // length > 0 for (let paramName in MockParameter) { - expect(config.parameter['object-chart'][paramName]) - .toEqual(MockParameter[paramName].defaultValue) + if (MockParameter.hasOwnProperty(paramName)) { + expect(config.parameter['object-chart'][paramName]) + .toEqual(MockParameter[paramName].defaultValue); + } } - }) + }); it('should set axisSpecs', () => { - const expected = Util.getSpecs(MockAxis1) - expect(config.axisSpecs['object-chart']).toEqual(expected) - expect(config.axisSpecs['array-chart'].length).toEqual(3) - expect(config.axisSpecs['drillDown-chart'].length).toEqual(3) - expect(config.axisSpecs['raw-chart'].length).toEqual(2) - }) + const expected = Util.getSpecs(MockAxis1); + expect(config.axisSpecs['object-chart']).toEqual(expected); + expect(config.axisSpecs['array-chart'].length).toEqual(3); + expect(config.axisSpecs['drillDown-chart'].length).toEqual(3); + expect(config.axisSpecs['raw-chart'].length).toEqual(2); + }); it('should prepare axis depending on dimension', () => { expect(config.axis['object-chart']).toEqual({ keyAxis: [], aggrAxis: [], groupAxis: [], - }) + }); expect(config.axis['array-chart']).toEqual({ keyAxis: [], aggrAxis: [], groupAxis: [], - }) + }); // it's ok not to set single dimension axis - expect(config.axis['drillDown-chart']).toEqual({ limitedAggrAxis: [], groupAxis: [], }) + expect(config.axis['drillDown-chart']).toEqual({limitedAggrAxis: [], groupAxis: []}); // it's ok not to set single dimension axis - expect(config.axis['raw-chart']).toEqual({ customAxis2: [], }) - }) - }) + expect(config.axis['raw-chart']).toEqual({customAxis2: []}); + }); + }); describe('axis', () => { - }) + }); describe('parameter:widget', () => { it('isInputWidget', () => { - expect(Util.isInputWidget(MockParameter.stringParam1)).toBe(true) - expect(Util.isInputWidget(MockParameter.stringParam2)).toBe(true) + expect(Util.isInputWidget(MockParameter.stringParam1)).toBe(true); + expect(Util.isInputWidget(MockParameter.stringParam2)).toBe(true); - expect(Util.isInputWidget(MockParameter.boolParam)).toBe(false) - expect(Util.isInputWidget(MockParameter.jsonParam)).toBe(false) - expect(Util.isInputWidget(MockParameter.optionParam)).toBe(false) - }) + expect(Util.isInputWidget(MockParameter.boolParam)).toBe(false); + expect(Util.isInputWidget(MockParameter.jsonParam)).toBe(false); + expect(Util.isInputWidget(MockParameter.optionParam)).toBe(false); + }); it('isOptionWidget', () => { - expect(Util.isOptionWidget(MockParameter.optionParam)).toBe(true) + expect(Util.isOptionWidget(MockParameter.optionParam)).toBe(true); - expect(Util.isOptionWidget(MockParameter.stringParam1)).toBe(false) - expect(Util.isOptionWidget(MockParameter.stringParam2)).toBe(false) - expect(Util.isOptionWidget(MockParameter.boolParam)).toBe(false) - expect(Util.isOptionWidget(MockParameter.jsonParam)).toBe(false) - }) + expect(Util.isOptionWidget(MockParameter.stringParam1)).toBe(false); + expect(Util.isOptionWidget(MockParameter.stringParam2)).toBe(false); + expect(Util.isOptionWidget(MockParameter.boolParam)).toBe(false); + expect(Util.isOptionWidget(MockParameter.jsonParam)).toBe(false); + }); it('isCheckboxWidget', () => { - expect(Util.isCheckboxWidget(MockParameter.boolParam)).toBe(true) + expect(Util.isCheckboxWidget(MockParameter.boolParam)).toBe(true); - expect(Util.isCheckboxWidget(MockParameter.stringParam1)).toBe(false) - expect(Util.isCheckboxWidget(MockParameter.stringParam2)).toBe(false) - expect(Util.isCheckboxWidget(MockParameter.jsonParam)).toBe(false) - expect(Util.isCheckboxWidget(MockParameter.optionParam)).toBe(false) - }) + expect(Util.isCheckboxWidget(MockParameter.stringParam1)).toBe(false); + expect(Util.isCheckboxWidget(MockParameter.stringParam2)).toBe(false); + expect(Util.isCheckboxWidget(MockParameter.jsonParam)).toBe(false); + expect(Util.isCheckboxWidget(MockParameter.optionParam)).toBe(false); + }); it('isTextareaWidget', () => { - expect(Util.isTextareaWidget(MockParameter.jsonParam)).toBe(true) + expect(Util.isTextareaWidget(MockParameter.jsonParam)).toBe(true); - expect(Util.isTextareaWidget(MockParameter.stringParam1)).toBe(false) - expect(Util.isTextareaWidget(MockParameter.stringParam2)).toBe(false) - expect(Util.isTextareaWidget(MockParameter.boolParam)).toBe(false) - expect(Util.isTextareaWidget(MockParameter.optionParam)).toBe(false) - }) - }) + expect(Util.isTextareaWidget(MockParameter.stringParam1)).toBe(false); + expect(Util.isTextareaWidget(MockParameter.stringParam2)).toBe(false); + expect(Util.isTextareaWidget(MockParameter.boolParam)).toBe(false); + expect(Util.isTextareaWidget(MockParameter.optionParam)).toBe(false); + }); + }); describe('parameter:parseParameter', () => { - const paramSpec = Util.getSpecs(MockParameter) + const paramSpec = Util.getSpecs(MockParameter); it('should parse number', () => { - const params = { intParam: '3', } - const parsed = Util.parseParameter(paramSpec, params) - expect(parsed.intParam).toBe(3) - }) + const params = {intParam: '3'}; + const parsed = Util.parseParameter(paramSpec, params); + expect(parsed.intParam).toBe(3); + }); it('should parse float', () => { - const params = { floatParam: '0.10', } - const parsed = Util.parseParameter(paramSpec, params) - expect(parsed.floatParam).toBe(0.10) - }) + const params = {floatParam: '0.10'}; + const parsed = Util.parseParameter(paramSpec, params); + expect(parsed.floatParam).toBe(0.10); + }); it('should parse boolean', () => { - const params1 = { boolParam: 'true', } - const parsed1 = Util.parseParameter(paramSpec, params1) - expect(typeof parsed1.boolParam).toBe('boolean') - expect(parsed1.boolParam).toBe(true) + const params1 = {boolParam: 'true'}; + const parsed1 = Util.parseParameter(paramSpec, params1); + expect(typeof parsed1.boolParam).toBe('boolean'); + expect(parsed1.boolParam).toBe(true); - const params2 = { boolParam: 'false', } - const parsed2 = Util.parseParameter(paramSpec, params2) - expect(typeof parsed2.boolParam).toBe('boolean') - expect(parsed2.boolParam).toBe(false) - }) + const params2 = {boolParam: 'false'}; + const parsed2 = Util.parseParameter(paramSpec, params2); + expect(typeof parsed2.boolParam).toBe('boolean'); + expect(parsed2.boolParam).toBe(false); + }); it('should parse JSON', () => { - const params = { jsonParam: '{ "a": 3 }', } - const parsed = Util.parseParameter(paramSpec, params) - expect(typeof parsed.jsonParam).toBe('object') - expect(JSON.stringify(parsed.jsonParam)).toBe('{"a":3}') - }) + const params = {jsonParam: '{ "a": 3 }'}; + const parsed = Util.parseParameter(paramSpec, params); + expect(typeof parsed.jsonParam).toBe('object'); + expect(JSON.stringify(parsed.jsonParam)).toBe('{"a":3}'); + }); it('should not parse string', () => { - const params = { stringParam: 'example', } - const parsed = Util.parseParameter(paramSpec, params) - expect(typeof parsed.stringParam).toBe('string') - expect(parsed.stringParam).toBe('example') - }) - }) + const params = {stringParam: 'example'}; + const parsed = Util.parseParameter(paramSpec, params); + expect(typeof parsed.stringParam).toBe('string'); + expect(parsed.stringParam).toBe('example'); + }); + }); describe('removeDuplicatedColumnsInMultiDimensionAxis', () => { - let config = {} + let config = {}; beforeEach(() => { - config = {} - const spec = JSON.parse(JSON.stringify(MockSpec)) - Util.initializeConfig(config, spec) - config.chart.current = 'drillDown-chart' // set non-sharedAxis chart - }) + config = {}; + const spec = JSON.parse(JSON.stringify(MockSpec)); + Util.initializeConfig(config, spec); + config.chart.current = 'drillDown-chart'; // set non-sharedAxis chart + }); it('should remove duplicated axis names in config when axis is not aggregator', () => { const addColumn = function(config, col) { - const axis = Util.getCurrentChartAxis(config)['groupAxis'] - axis.push(col) - const axisSpecs = Util.getCurrentChartAxisSpecs(config) - Util.removeDuplicatedColumnsInMultiDimensionAxis(config, axisSpecs[2]) - } + const axis = Util.getCurrentChartAxis(config)['groupAxis']; + axis.push(col); + const axisSpecs = Util.getCurrentChartAxisSpecs(config); + Util.removeDuplicatedColumnsInMultiDimensionAxis(config, axisSpecs[2]); + }; - addColumn(config, { name: 'columnA', aggr: 'sum', index: 0, }) - addColumn(config, { name: 'columnA', aggr: 'sum', index: 0, }) - addColumn(config, { name: 'columnA', aggr: 'sum', index: 0, }) + addColumn(config, {name: 'columnA', aggr: 'sum', index: 0}); + addColumn(config, {name: 'columnA', aggr: 'sum', index: 0}); + addColumn(config, {name: 'columnA', aggr: 'sum', index: 0}); - expect(Util.getCurrentChartAxis(config)['groupAxis'].length).toEqual(1) - }) + expect(Util.getCurrentChartAxis(config)['groupAxis'].length).toEqual(1); + }); it('should remove duplicated axis names in config when axis is aggregator', () => { const addColumn = function(config, value) { - const axis = Util.getCurrentChartAxis(config)['limitedAggrAxis'] - axis.push(value) - const axisSpecs = Util.getCurrentChartAxisSpecs(config) - Util.removeDuplicatedColumnsInMultiDimensionAxis(config, axisSpecs[1]) - } + const axis = Util.getCurrentChartAxis(config)['limitedAggrAxis']; + axis.push(value); + const axisSpecs = Util.getCurrentChartAxisSpecs(config); + Util.removeDuplicatedColumnsInMultiDimensionAxis(config, axisSpecs[1]); + }; - config.chart.current = 'drillDown-chart' // set non-sharedAxis chart - addColumn(config, { name: 'columnA', aggr: 'sum', index: 0, }) - addColumn(config, { name: 'columnA', aggr: 'aggr', index: 0, }) - addColumn(config, { name: 'columnA', aggr: 'sum', index: 0, }) + config.chart.current = 'drillDown-chart'; // set non-sharedAxis chart + addColumn(config, {name: 'columnA', aggr: 'sum', index: 0}); + addColumn(config, {name: 'columnA', aggr: 'aggr', index: 0}); + addColumn(config, {name: 'columnA', aggr: 'sum', index: 0}); - expect(Util.getCurrentChartAxis(config)['limitedAggrAxis'].length).toEqual(2) - }) - }) + expect(Util.getCurrentChartAxis(config)['limitedAggrAxis'].length).toEqual(2); + }); + }); describe('applyMaxAxisCount', () => { - const config = {} - const spec = JSON.parse(JSON.stringify(MockSpec)) - Util.initializeConfig(config, spec) + const config = {}; + const spec = JSON.parse(JSON.stringify(MockSpec)); + Util.initializeConfig(config, spec); const addColumn = function(config, value) { - const axis = Util.getCurrentChartAxis(config)['limitedAggrAxis'] - axis.push(value) - const axisSpecs = Util.getCurrentChartAxisSpecs(config) - Util.applyMaxAxisCount(config, axisSpecs[1]) - } + const axis = Util.getCurrentChartAxis(config)['limitedAggrAxis']; + axis.push(value); + const axisSpecs = Util.getCurrentChartAxisSpecs(config); + Util.applyMaxAxisCount(config, axisSpecs[1]); + }; it('should remove duplicated axis names in config', () => { - config.chart.current = 'drillDown-chart' // set non-sharedAxis chart + config.chart.current = 'drillDown-chart'; // set non-sharedAxis chart - addColumn(config, 'columnA') - addColumn(config, 'columnB') - addColumn(config, 'columnC') - addColumn(config, 'columnD') + addColumn(config, 'columnA'); + addColumn(config, 'columnB'); + addColumn(config, 'columnC'); + addColumn(config, 'columnD'); expect(Util.getCurrentChartAxis(config)['limitedAggrAxis']).toEqual([ 'columnC', 'columnD', - ]) - }) - }) + ]); + }); + }); describe('getColumnsFromAxis', () => { it('should return proper value for regular axis spec (key, aggr, group)', () => { - const config = {} - - const spec = JSON.parse(JSON.stringify(MockSpec)) - Util.initializeConfig(config, spec) - const chart = 'object-chart' - config.chart.current = chart - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - axis['keyAxis'].push('columnA') - axis['keyAxis'].push('columnB') - axis['aggrAxis'].push('columnC') - axis['groupAxis'].push('columnD') - axis['groupAxis'].push('columnE') - axis['groupAxis'].push('columnF') - - const column = Util.getColumnsFromAxis(axisSpecs, axis) - expect(column.key).toEqual([ 'columnA', 'columnB', ]) - expect(column.aggregator).toEqual([ 'columnC', ]) - expect(column.group).toEqual([ 'columnD', 'columnE', 'columnF', ]) - }) + const config = {}; + + const spec = JSON.parse(JSON.stringify(MockSpec)); + Util.initializeConfig(config, spec); + const chart = 'object-chart'; + config.chart.current = chart; + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + axis['keyAxis'].push('columnA'); + axis['keyAxis'].push('columnB'); + axis['aggrAxis'].push('columnC'); + axis['groupAxis'].push('columnD'); + axis['groupAxis'].push('columnE'); + axis['groupAxis'].push('columnF'); + + const column = Util.getColumnsFromAxis(axisSpecs, axis); + expect(column.key).toEqual(['columnA', 'columnB']); + expect(column.aggregator).toEqual(['columnC']); + expect(column.group).toEqual(['columnD', 'columnE', 'columnF']); + }); it('should return proper value for custom axis spec', () => { - const config = {} - const spec = JSON.parse(JSON.stringify(MockSpec)) - Util.initializeConfig(config, spec) - const chart = 'raw-chart' // for test custom columns - config.chart.current = chart - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - axis['customAxis1'] = ['columnA'] - axis['customAxis2'].push('columnB') - axis['customAxis2'].push('columnC') - axis['customAxis2'].push('columnD') - - const column = Util.getColumnsFromAxis(axisSpecs, axis) - expect(column.custom.unique).toEqual([ 'columnA', ]) - expect(column.custom.value).toEqual([ 'columnB', 'columnC', 'columnD', ]) - }) - }) + const config = {}; + const spec = JSON.parse(JSON.stringify(MockSpec)); + Util.initializeConfig(config, spec); + const chart = 'raw-chart'; // for test custom columns + config.chart.current = chart; + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + axis['customAxis1'] = ['columnA']; + axis['customAxis2'].push('columnB'); + axis['customAxis2'].push('columnC'); + axis['customAxis2'].push('columnD'); + + const column = Util.getColumnsFromAxis(axisSpecs, axis); + expect(column.custom.unique).toEqual(['columnA']); + expect(column.custom.value).toEqual(['columnB', 'columnC', 'columnD']); + }); + }); // it's hard to test all methods for transformation. // so let's do behavioral (black-box) test instead of describe('getTransformer', () => { describe('method: raw', () => { - let config = {} - const spec = JSON.parse(JSON.stringify(MockSpec2)) - Util.initializeConfig(config, spec) + let config = {}; + const spec = JSON.parse(JSON.stringify(MockSpec2)); + Util.initializeConfig(config, spec); it('should return original rows when transform.method is `raw`', () => { - const chart = 'raw-chart' - config.chart.current = chart + const chart = 'raw-chart'; + config.chart.current = chart; - const rows = [ { 'r1': 1, }, ] - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, rows, axisSpecs, axis).transformer - const transformed = transformer() + const rows = [{'r1': 1}]; + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, rows, axisSpecs, axis).transformer; + const transformed = transformer(); - expect(transformed).toBe(rows) - }) - }) + expect(transformed).toBe(rows); + }); + }); describe('array method', () => { - let config = {} - const chart = 'array-chart' - let ageColumn = null - let balanceColumn = null - let educationColumn = null - let martialColumn = null - let tableDataRows = [] + let config = {}; + const chart = 'array-chart'; + let ageColumn = null; + let balanceColumn = null; + let educationColumn = null; + let martialColumn = null; + let tableDataRows = []; beforeEach(() => { - const spec = JSON.parse(JSON.stringify(MockSpec2)) - config = {} - Util.initializeConfig(config, spec) - config.chart.current = chart - tableDataRows = JSON.parse(JSON.stringify(MockTableDataRows1)) - ageColumn = JSON.parse(JSON.stringify(MockTableDataColumn[0])) - balanceColumn = JSON.parse(JSON.stringify(MockTableDataColumn[5])) - educationColumn = JSON.parse(JSON.stringify(MockTableDataColumn[3])) - martialColumn = JSON.parse(JSON.stringify(MockTableDataColumn[2])) - }) + const spec = JSON.parse(JSON.stringify(MockSpec2)); + config = {}; + Util.initializeConfig(config, spec); + config.chart.current = chart; + tableDataRows = JSON.parse(JSON.stringify(MockTableDataRows1)); + ageColumn = JSON.parse(JSON.stringify(MockTableDataColumn[0])); + balanceColumn = JSON.parse(JSON.stringify(MockTableDataColumn[5])); + educationColumn = JSON.parse(JSON.stringify(MockTableDataColumn[3])); + martialColumn = JSON.parse(JSON.stringify(MockTableDataColumn[2])); + }); it('should transform properly: 0 key, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer - - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() - - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ '', ]) - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; + + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); + + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['']); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ - { selector: 'age(sum)', value: [ 159, ], } - ]) - }) + {selector: 'age(sum)', value: [159]}, + ]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(count)', () => { - ageColumn.aggr = 'count' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'count'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - let { rows, } = transformer() + let {rows} = transformer(); expect(rows).toEqual([ - { selector: 'age(count)', value: [ 4, ], } - ]) - }) + {selector: 'age(count)', value: [4]}, + ]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(avg)', () => { - ageColumn.aggr = 'avg' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'avg'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() + const {rows} = transformer(); expect(rows).toEqual([ - { selector: 'age(avg)', value: [ (44 + 43 + 39 + 33) / 4.0, ], } - ]) - }) + {selector: 'age(avg)', value: [(44 + 43 + 39 + 33) / 4.0]}, + ]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(max)', () => { - ageColumn.aggr = 'max' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'max'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() + const {rows} = transformer(); expect(rows).toEqual([ - { selector: 'age(max)', value: [ 44, ], } - ]) - }) + {selector: 'age(max)', value: [44]}, + ]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(min)', () => { - ageColumn.aggr = 'min' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'min'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() + const {rows} = transformer(); expect(rows).toEqual([ - { selector: 'age(min)', value: [ 33, ], } - ]) - }) + {selector: 'age(min)', value: [33]}, + ]); + }); it('should transform properly: 0 key, 0 group, 2 aggr(sum)', () => { - ageColumn.aggr = 'sum' - balanceColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].aggrAxis.push(balanceColumn) + ageColumn.aggr = 'sum'; + balanceColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].aggrAxis.push(balanceColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ '', ]) - expect(groupNames).toEqual([ 'age(sum)', 'balance(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', 'balance(sum)', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['']); + expect(groupNames).toEqual(['age(sum)', 'balance(sum)']); + expect(selectors).toEqual(['age(sum)', 'balance(sum)']); expect(rows).toEqual([ - { selector: 'age(sum)', value: [ 159, ], }, - { selector: 'balance(sum)', value: [ 14181, ], }, - ]) - }) + {selector: 'age(sum)', value: [159]}, + {selector: 'balance(sum)', value: [14181]}, + ]); + }); it('should transform properly: 0 key, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].groupAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].groupAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ 'marital', ]) - expect(groupNames).toEqual([ 'married', 'single', ]) - expect(selectors).toEqual([ 'married', 'single', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['marital']); + expect(groupNames).toEqual(['married', 'single']); + expect(selectors).toEqual(['married', 'single']); expect(rows).toEqual([ - { selector: 'married', value: [ 82, ], }, - { selector: 'single', value: [ 77, ], }, - ]) - }) + {selector: 'married', value: [82]}, + {selector: 'single', value: [77]}, + ]); + }); it('should transform properly: 0 key, 1 group, 2 aggr(sum)', () => { - ageColumn.aggr = 'sum' - balanceColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].aggrAxis.push(balanceColumn) - config.axis[chart].groupAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + balanceColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].aggrAxis.push(balanceColumn); + config.axis[chart].groupAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ 'marital', ]) - expect(groupNames).toEqual([ 'married', 'single', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['marital']); + expect(groupNames).toEqual(['married', 'single']); expect(selectors).toEqual([ 'married / age(sum)', 'married / balance(sum)', 'single / age(sum)', 'single / balance(sum)', - ]) + ]); expect(rows).toEqual([ - { selector: 'married / age(sum)', value: [ 82 ] }, - { selector: 'married / balance(sum)', value: [ 9286 ] }, - { selector: 'single / age(sum)', value: [ 77 ] }, - { selector: 'single / balance(sum)', value: [ 4895 ] }, - ]) - }) + {selector: 'married / age(sum)', value: [82]}, + {selector: 'married / balance(sum)', value: [9286]}, + {selector: 'single / age(sum)', value: [77]}, + {selector: 'single / balance(sum)', value: [4895]}, + ]); + }); it('should transform properly: 0 key, 2 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].groupAxis.push(martialColumn) - config.axis[chart].groupAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].groupAxis.push(martialColumn); + config.axis[chart].groupAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ 'marital.education', ]) - expect(groupNames).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) - expect(selectors).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['marital.education']); + expect(groupNames).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); + expect(selectors).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); expect(rows).toEqual([ - { selector: 'married.primary', value: [ '43' ] }, - { selector: 'married.secondary', value: [ '39' ] }, - { selector: 'single.tertiary', value: [ 77 ] }, - ]) - }) + {selector: 'married.primary', value: ['43']}, + {selector: 'married.secondary', value: ['39']}, + {selector: 'single.tertiary', value: [77]}, + ]); + }); it('should transform properly: 1 key, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].keyAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].keyAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('marital') - expect(keyNames).toEqual([ 'married', 'single', ]) - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + expect(keyColumnName).toEqual('marital'); + expect(keyNames).toEqual(['married', 'single']); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ - { selector: 'age(sum)', value: [ 82, 77, ] }, - ]) - }) + {selector: 'age(sum)', value: [82, 77]}, + ]); + }); it('should transform properly: 2 key, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].keyAxis.push(martialColumn) - config.axis[chart].keyAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].keyAxis.push(martialColumn); + config.axis[chart].keyAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('marital.education') - expect(keyNames).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + expect(keyColumnName).toEqual('marital.education'); + expect(keyNames).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ - { selector: 'age(sum)', value: [ '43', '39', 77, ] }, - ]) - }) + {selector: 'age(sum)', value: ['43', '39', 77]}, + ]); + }); it('should transform properly: 1 key, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].keyAxis.push(martialColumn) - config.axis[chart].groupAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].keyAxis.push(martialColumn); + config.axis[chart].groupAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('marital') - expect(keyNames).toEqual([ 'married', 'single', ]) - expect(groupNames).toEqual([ 'primary', 'secondary', 'tertiary', ]) - expect(selectors).toEqual([ 'primary', 'secondary', 'tertiary', ]) + expect(keyColumnName).toEqual('marital'); + expect(keyNames).toEqual(['married', 'single']); + expect(groupNames).toEqual(['primary', 'secondary', 'tertiary']); + expect(selectors).toEqual(['primary', 'secondary', 'tertiary']); expect(rows).toEqual([ - { selector: 'primary', value: [ '43', null, ] }, - { selector: 'secondary', value: [ '39', null, ] }, - { selector: 'tertiary', value: [ null, 77, ] }, - ]) - }) - }) // end: describe('method: array') + {selector: 'primary', value: ['43', null]}, + {selector: 'secondary', value: ['39', null]}, + {selector: 'tertiary', value: [null, 77]}, + ]); + }); + }); // end: describe('method: array') describe('method: object', () => { - let config = {} - const chart = 'object-chart' - let ageColumn = null - let balanceColumn = null - let educationColumn = null - let martialColumn = null - const tableDataRows = JSON.parse(JSON.stringify(MockTableDataRows1)) + let config = {}; + const chart = 'object-chart'; + let ageColumn = null; + let balanceColumn = null; + let educationColumn = null; + let martialColumn = null; + const tableDataRows = JSON.parse(JSON.stringify(MockTableDataRows1)); beforeEach(() => { - const spec = JSON.parse(JSON.stringify(MockSpec2)) - config = {} - Util.initializeConfig(config, spec) - config.chart.current = chart - ageColumn = JSON.parse(JSON.stringify(MockTableDataColumn[0])) - balanceColumn = JSON.parse(JSON.stringify(MockTableDataColumn[5])) - educationColumn = JSON.parse(JSON.stringify(MockTableDataColumn[3])) - martialColumn = JSON.parse(JSON.stringify(MockTableDataColumn[2])) - }) + const spec = JSON.parse(JSON.stringify(MockSpec2)); + config = {}; + Util.initializeConfig(config, spec); + config.chart.current = chart; + ageColumn = JSON.parse(JSON.stringify(MockTableDataColumn[0])); + balanceColumn = JSON.parse(JSON.stringify(MockTableDataColumn[5])); + educationColumn = JSON.parse(JSON.stringify(MockTableDataColumn[3])); + martialColumn = JSON.parse(JSON.stringify(MockTableDataColumn[2])); + }); it('should transform properly: 0 key, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ '', ]) - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', ]) - expect(rows).toEqual([{ 'age(sum)': 44 + 43 + 39 + 33, }]) - }) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['']); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['age(sum)']); + expect(rows).toEqual([{'age(sum)': 44 + 43 + 39 + 33}]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(count)', () => { - ageColumn.aggr = 'count' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'count'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() - expect(rows).toEqual([{ 'age(count)': 4, }]) - }) + const {rows} = transformer(); + expect(rows).toEqual([{'age(count)': 4}]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(avg)', () => { - ageColumn.aggr = 'avg' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'avg'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() + const {rows} = transformer(); expect(rows).toEqual([ - { 'age(avg)': (44 + 43 + 39 + 33) / 4.0, } - ]) - }) + {'age(avg)': (44 + 43 + 39 + 33) / 4.0}, + ]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(max)', () => { - ageColumn.aggr = 'max' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'max'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() - expect(rows).toEqual([{ 'age(max)': 44, }]) - }) + const {rows} = transformer(); + expect(rows).toEqual([{'age(max)': 44}]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(min)', () => { - ageColumn.aggr = 'min' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'min'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() - expect(rows).toEqual([{ 'age(min)': 33, }]) - }) + const {rows} = transformer(); + expect(rows).toEqual([{'age(min)': 33}]); + }); it('should transform properly: 0 key, 0 group, 2 aggr(sum)', () => { - ageColumn.aggr = 'sum' - balanceColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].aggrAxis.push(balanceColumn) + ageColumn.aggr = 'sum'; + balanceColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].aggrAxis.push(balanceColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ '', ]) - expect(groupNames).toEqual([ 'age(sum)', 'balance(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', 'balance(sum)', ]) - expect(rows).toEqual([{ 'age(sum)': 159, 'balance(sum)': 14181, }]) - }) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['']); + expect(groupNames).toEqual(['age(sum)', 'balance(sum)']); + expect(selectors).toEqual(['age(sum)', 'balance(sum)']); + expect(rows).toEqual([{'age(sum)': 159, 'balance(sum)': 14181}]); + }); it('should transform properly: 0 key, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].groupAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].groupAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ 'marital', ]) - expect(groupNames).toEqual([ 'married', 'single', ]) - expect(selectors).toEqual([ 'married', 'single', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['marital']); + expect(groupNames).toEqual(['married', 'single']); + expect(selectors).toEqual(['married', 'single']); expect(rows).toEqual([ - { single: 77, married: 82, } - ]) - }) + {single: 77, married: 82}, + ]); + }); it('should transform properly: 0 key, 1 group, 2 aggr(sum)', () => { - ageColumn.aggr = 'sum' - balanceColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].aggrAxis.push(balanceColumn) - config.axis[chart].groupAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + balanceColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].aggrAxis.push(balanceColumn); + config.axis[chart].groupAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ 'marital', ]) - expect(groupNames).toEqual([ 'married', 'single', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['marital']); + expect(groupNames).toEqual(['married', 'single']); expect(selectors).toEqual([ 'married / age(sum)', 'married / balance(sum)', 'single / age(sum)', 'single / balance(sum)', - ]) + ]); expect(rows).toEqual([{ 'married / age(sum)': 82, 'single / age(sum)': 77, 'married / balance(sum)': 9286, 'single / balance(sum)': 4895, - }]) - }) + }]); + }); it('should transform properly: 0 key, 2 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].groupAxis.push(martialColumn) - config.axis[chart].groupAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].groupAxis.push(martialColumn); + config.axis[chart].groupAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ 'marital.education', ]) - expect(groupNames).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) - expect(selectors).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['marital.education']); + expect(groupNames).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); + expect(selectors).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); expect(rows).toEqual([{ 'married.primary': '43', 'married.secondary': '39', 'single.tertiary': 77, - }]) - }) + }]); + }); it('should transform properly: 1 key, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].keyAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].keyAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('marital') - expect(keyNames).toEqual([ 'married', 'single', ]) - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + expect(keyColumnName).toEqual('marital'); + expect(keyNames).toEqual(['married', 'single']); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ - { 'age(sum)': 82, 'marital': 'married', }, - { 'age(sum)': 77, 'marital': 'single', }, - ]) - }) + {'age(sum)': 82, 'marital': 'married'}, + {'age(sum)': 77, 'marital': 'single'}, + ]); + }); it('should transform properly: 2 key, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].keyAxis.push(martialColumn) - config.axis[chart].keyAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].keyAxis.push(martialColumn); + config.axis[chart].keyAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('marital.education') - expect(keyNames).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + expect(keyColumnName).toEqual('marital.education'); + expect(keyNames).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ - { 'age(sum)': '43', 'marital.education': 'married.primary' }, - { 'age(sum)': '39', 'marital.education': 'married.secondary' }, - { 'age(sum)': 77, 'marital.education': 'single.tertiary' }, - ]) - }) + {'age(sum)': '43', 'marital.education': 'married.primary'}, + {'age(sum)': '39', 'marital.education': 'married.secondary'}, + {'age(sum)': 77, 'marital.education': 'single.tertiary'}, + ]); + }); it('should transform properly: 1 key, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].keyAxis.push(martialColumn) - config.axis[chart].groupAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].keyAxis.push(martialColumn); + config.axis[chart].groupAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('marital') - expect(keyNames).toEqual([ 'married', 'single', ]) - expect(groupNames).toEqual([ 'primary', 'secondary', 'tertiary', ]) - expect(selectors).toEqual([ 'primary', 'secondary', 'tertiary', ]) + expect(keyColumnName).toEqual('marital'); + expect(keyNames).toEqual(['married', 'single']); + expect(groupNames).toEqual(['primary', 'secondary', 'tertiary']); + expect(selectors).toEqual(['primary', 'secondary', 'tertiary']); expect(rows).toEqual([ - { primary: '43', secondary: '39', marital: 'married' }, - { tertiary: 44 + 33, marital: 'single' }, - ]) - }) - }) // end: describe('method: object') + {primary: '43', secondary: '39', marital: 'married'}, + {tertiary: 44 + 33, marital: 'single'}, + ]); + }); + }); // end: describe('method: object') describe('method: drill-down', () => { - let config = {} - const chart = 'drillDown-chart' - let ageColumn = null - let balanceColumn = null - let educationColumn = null - let martialColumn = null - const tableDataRows = JSON.parse(JSON.stringify(MockTableDataRows1)) + let config = {}; + const chart = 'drillDown-chart'; + let ageColumn = null; + let balanceColumn = null; + let educationColumn = null; + let martialColumn = null; + const tableDataRows = JSON.parse(JSON.stringify(MockTableDataRows1)); beforeEach(() => { - const spec = JSON.parse(JSON.stringify(MockSpec2)) - config = {} - Util.initializeConfig(config, spec) - config.chart.current = chart - ageColumn = JSON.parse(JSON.stringify(MockTableDataColumn[0])) - balanceColumn = JSON.parse(JSON.stringify(MockTableDataColumn[5])) - educationColumn = JSON.parse(JSON.stringify(MockTableDataColumn[3])) - martialColumn = JSON.parse(JSON.stringify(MockTableDataColumn[2])) - }) + const spec = JSON.parse(JSON.stringify(MockSpec2)); + config = {}; + Util.initializeConfig(config, spec); + config.chart.current = chart; + ageColumn = JSON.parse(JSON.stringify(MockTableDataColumn[0])); + balanceColumn = JSON.parse(JSON.stringify(MockTableDataColumn[5])); + educationColumn = JSON.parse(JSON.stringify(MockTableDataColumn[3])); + martialColumn = JSON.parse(JSON.stringify(MockTableDataColumn[2])); + }); it('should transform properly: 0 key, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ '', ]) - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['']); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ - { selector: 'age(sum)', value: 44 + 43 + 39 + 33, drillDown: [ ], }, - ]) - }) + {selector: 'age(sum)', value: 44 + 43 + 39 + 33, drillDown: []}, + ]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(count)', () => { - ageColumn.aggr = 'count' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'count'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() + const {rows} = transformer(); expect(rows).toEqual([ - { selector: 'age(count)', value: 4, drillDown: [ ], }, - ]) - }) + {selector: 'age(count)', value: 4, drillDown: []}, + ]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(avg)', () => { - ageColumn.aggr = 'avg' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'avg'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() + const {rows} = transformer(); expect(rows).toEqual([ - { selector: 'age(avg)', value: (44 + 43 + 39 + 33) / 4.0, drillDown: [ ], }, - ]) - }) + {selector: 'age(avg)', value: (44 + 43 + 39 + 33) / 4.0, drillDown: []}, + ]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(max)', () => { - ageColumn.aggr = 'max' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'max'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() + const {rows} = transformer(); expect(rows).toEqual([ - { selector: 'age(max)', value: 44, drillDown: [ ], }, - ]) - }) + {selector: 'age(max)', value: 44, drillDown: []}, + ]); + }); it('should transform properly: 0 key, 0 group, 1 aggr(min)', () => { - ageColumn.aggr = 'min' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'min'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, } = transformer() + const {rows} = transformer(); expect(rows).toEqual([ - { selector: 'age(min)', value: 33, drillDown: [ ], }, - ]) - }) + {selector: 'age(min)', value: 33, drillDown: []}, + ]); + }); it('should transform properly: 0 key, 0 group, 2 aggr(sum)', () => { - ageColumn.aggr = 'sum' - balanceColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].aggrAxis.push(balanceColumn) + ageColumn.aggr = 'sum'; + balanceColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].aggrAxis.push(balanceColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ '', ]) - expect(groupNames).toEqual([ 'age(sum)', 'balance(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', 'balance(sum)', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['']); + expect(groupNames).toEqual(['age(sum)', 'balance(sum)']); + expect(selectors).toEqual(['age(sum)', 'balance(sum)']); expect(rows).toEqual([ - { selector: 'age(sum)', value: 159, drillDown: [ ], }, - { selector: 'balance(sum)', value: 14181, drillDown: [ ], }, - ]) - }) + {selector: 'age(sum)', value: 159, drillDown: []}, + {selector: 'balance(sum)', value: 14181, drillDown: []}, + ]); + }); it('should transform properly: 0 key, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].groupAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].groupAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ 'marital', ]) - expect(groupNames).toEqual([ 'married', 'single', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['marital']); + expect(groupNames).toEqual(['married', 'single']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ { selector: 'age(sum)', value: 159, drillDown: [ - { group: 'married', value: 82 }, - { group: 'single', value: 77 }, + {group: 'married', value: 82}, + {group: 'single', value: 77}, ], }, - ]) - }) + ]); + }); it('should transform properly: 0 key, 1 group, 2 aggr(sum)', () => { - ageColumn.aggr = 'sum' - balanceColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].aggrAxis.push(balanceColumn) - config.axis[chart].groupAxis.push(martialColumn) - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer - - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() - - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ 'marital', ]) - expect(groupNames).toEqual([ 'married', 'single', ]) - expect(selectors).toEqual([ 'age(sum)', 'balance(sum)' ]) + ageColumn.aggr = 'sum'; + balanceColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].aggrAxis.push(balanceColumn); + config.axis[chart].groupAxis.push(martialColumn); + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; + + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); + + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['marital']); + expect(groupNames).toEqual(['married', 'single']); + expect(selectors).toEqual(['age(sum)', 'balance(sum)']); expect(rows).toEqual([ { selector: 'age(sum)', value: 159, drillDown: [ - { group: 'married', value: 82 }, - { group: 'single', value: 77 }, + {group: 'married', value: 82}, + {group: 'single', value: 77}, ], }, { selector: 'balance(sum)', value: 14181, drillDown: [ - { group: 'married', value: 9286 }, - { group: 'single', value: 4895 }, + {group: 'married', value: 9286}, + {group: 'single', value: 4895}, ], }, - ]) - }) + ]); + }); it('should transform properly: 0 key, 2 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].groupAxis.push(martialColumn) - config.axis[chart].groupAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].groupAxis.push(martialColumn); + config.axis[chart].groupAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('') - expect(keyNames).toEqual([ 'marital.education', ]) - expect(groupNames).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + expect(keyColumnName).toEqual(''); + expect(keyNames).toEqual(['marital.education']); + expect(groupNames).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ { selector: 'age(sum)', value: 159, drillDown: [ - { group: 'married.primary', value: '43' }, - { group: 'married.secondary', value: '39' }, - { group: 'single.tertiary', value: 77 }, + {group: 'married.primary', value: '43'}, + {group: 'married.secondary', value: '39'}, + {group: 'single.tertiary', value: 77}, ], }, - ]) - }) + ]); + }); it('should transform properly: 1 key, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].keyAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].keyAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('marital') - expect(keyNames).toEqual([ 'married', 'single', ]) - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'married', 'single', ]) + expect(keyColumnName).toEqual('marital'); + expect(keyNames).toEqual(['married', 'single']); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['married', 'single']); expect(rows).toEqual([ - { selector: 'married', value: 82, drillDown: [ ], }, - { selector: 'single', value: 77, drillDown: [ ], }, - ]) - }) + {selector: 'married', value: 82, drillDown: []}, + {selector: 'single', value: 77, drillDown: []}, + ]); + }); it('should transform properly: 2 key, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].keyAxis.push(martialColumn) - config.axis[chart].keyAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].keyAxis.push(martialColumn); + config.axis[chart].keyAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('marital.education') - expect(keyNames).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) + expect(keyColumnName).toEqual('marital.education'); + expect(keyNames).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); expect(rows).toEqual([ - { selector: 'married.primary', value: '43', drillDown: [ ], }, - { selector: 'married.secondary', value: '39', drillDown: [ ], }, - { selector: 'single.tertiary', value: 77, drillDown: [ ], }, - ]) - }) + {selector: 'married.primary', value: '43', drillDown: []}, + {selector: 'married.secondary', value: '39', drillDown: []}, + {selector: 'single.tertiary', value: 77, drillDown: []}, + ]); + }); it('should transform properly: 1 key, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].keyAxis.push(martialColumn) - config.axis[chart].groupAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].keyAxis.push(martialColumn); + config.axis[chart].groupAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, keyColumnName, keyNames, groupNames, selectors, } = transformer() + const {rows, keyColumnName, keyNames, groupNames, selectors} = transformer(); - expect(keyColumnName).toEqual('marital') - expect(keyNames).toEqual([ 'married', 'single', ]) - expect(groupNames).toEqual([ 'primary', 'secondary', 'tertiary', ]) - expect(selectors).toEqual([ 'married', 'single', ]) + expect(keyColumnName).toEqual('marital'); + expect(keyNames).toEqual(['married', 'single']); + expect(groupNames).toEqual(['primary', 'secondary', 'tertiary']); + expect(selectors).toEqual(['married', 'single']); expect(rows).toEqual([ { selector: 'married', value: 82, drillDown: [ - { group: 'primary', value: '43' }, - { group: 'secondary', value: '39' }, - { group: 'tertiary', value: null }, + {group: 'primary', value: '43'}, + {group: 'secondary', value: '39'}, + {group: 'tertiary', value: null}, ], }, { selector: 'single', value: 77, drillDown: [ - { group: 'primary', value: null }, - { group: 'secondary', value: null }, - { group: 'tertiary', value: 77 }, + {group: 'primary', value: null}, + {group: 'secondary', value: null}, + {group: 'tertiary', value: 77}, ], }, - ]) - }) - }) // end: describe('method: drill-down') + ]); + }); + }); // end: describe('method: drill-down') describe('method: array:2-key', () => { - let config = {} - const chart = 'array2Key-chart' - let ageColumn = null - let balanceColumn = null - let educationColumn = null - let martialColumn = null - let daysColumn = null - let pDaysColumn = null - const tableDataRows = JSON.parse(JSON.stringify(MockTableDataRows1)) + let config = {}; + const chart = 'array2Key-chart'; + let ageColumn = null; + let balanceColumn = null; + let educationColumn = null; + let martialColumn = null; + let daysColumn = null; + let pDaysColumn = null; + const tableDataRows = JSON.parse(JSON.stringify(MockTableDataRows1)); beforeEach(() => { - const spec = JSON.parse(JSON.stringify(MockSpec2)) - config = {} - Util.initializeConfig(config, spec) - config.chart.current = chart - ageColumn = JSON.parse(JSON.stringify(MockTableDataColumn[0])) - martialColumn = JSON.parse(JSON.stringify(MockTableDataColumn[2])) - educationColumn = JSON.parse(JSON.stringify(MockTableDataColumn[3])) - balanceColumn = JSON.parse(JSON.stringify(MockTableDataColumn[5])) - daysColumn = JSON.parse(JSON.stringify(MockTableDataColumn[9])) - pDaysColumn = JSON.parse(JSON.stringify(MockTableDataColumn[13])) - }) + const spec = JSON.parse(JSON.stringify(MockSpec2)); + config = {}; + Util.initializeConfig(config, spec); + config.chart.current = chart; + ageColumn = JSON.parse(JSON.stringify(MockTableDataColumn[0])); + martialColumn = JSON.parse(JSON.stringify(MockTableDataColumn[2])); + educationColumn = JSON.parse(JSON.stringify(MockTableDataColumn[3])); + balanceColumn = JSON.parse(JSON.stringify(MockTableDataColumn[5])); + daysColumn = JSON.parse(JSON.stringify(MockTableDataColumn[9])); + pDaysColumn = JSON.parse(JSON.stringify(MockTableDataColumn[13])); + }); it('should transform properly: 0 key1, 0 key2, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, key1Names, key2Names, selectors, } = transformer() + const {rows, key1Names, key2Names, selectors} = transformer(); - expect(key1Names).toEqual([]) - expect(key2Names).toEqual([]) - expect(selectors).toEqual([ 'age(sum)', ]) + expect(key1Names).toEqual([]); + expect(key2Names).toEqual([]); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ - { selector: 'age(sum)', value: [ { aggregated: 44 + 43 + 39 + 33, }, ] }, - ]) - }) + {selector: 'age(sum)', value: [{aggregated: 44 + 43 + 39 + 33}]}, + ]); + }); it('should transform properly: 0 key1, 0 key2, 0 group, 1 aggr(count)', () => { - ageColumn.aggr = 'count' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'count'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, key1Names, key2Names, selectors, } = transformer() + const {rows, key1Names, key2Names, selectors} = transformer(); - expect(key1Names).toEqual([]) - expect(key2Names).toEqual([]) - expect(selectors).toEqual([ 'age(count)', ]) + expect(key1Names).toEqual([]); + expect(key2Names).toEqual([]); + expect(selectors).toEqual(['age(count)']); expect(rows).toEqual([ - { selector: 'age(count)', value: [ { aggregated: 4, }, ] }, - ]) - }) + {selector: 'age(count)', value: [{aggregated: 4}]}, + ]); + }); it('should transform properly: 0 key1, 0 key2, 0 group, 1 aggr(avg)', () => { - ageColumn.aggr = 'avg' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'avg'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, key1Names, key2Names, selectors, } = transformer() + const {rows, key1Names, key2Names, selectors} = transformer(); - expect(key1Names).toEqual([]) - expect(key2Names).toEqual([]) - expect(selectors).toEqual([ 'age(avg)', ]) + expect(key1Names).toEqual([]); + expect(key2Names).toEqual([]); + expect(selectors).toEqual(['age(avg)']); expect(rows).toEqual([ - { selector: 'age(avg)', value: [ { aggregated: (44 + 43 + 39 + 33) / 4.0, }, ] }, - ]) - }) + {selector: 'age(avg)', value: [{aggregated: (44 + 43 + 39 + 33) / 4.0}]}, + ]); + }); it('should transform properly: 0 key1, 0 key2, 0 group, 1 aggr(max)', () => { - ageColumn.aggr = 'max' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'max'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, key1Names, key2Names, selectors, } = transformer() + const {rows, key1Names, key2Names, selectors} = transformer(); - expect(key1Names).toEqual([]) - expect(key2Names).toEqual([]) - expect(selectors).toEqual([ 'age(max)', ]) + expect(key1Names).toEqual([]); + expect(key2Names).toEqual([]); + expect(selectors).toEqual(['age(max)']); expect(rows).toEqual([ - { selector: 'age(max)', value: [ { aggregated: 44, }, ] }, - ]) - }) + {selector: 'age(max)', value: [{aggregated: 44}]}, + ]); + }); it('should transform properly: 0 key1, 0 key2, 0 group, 1 aggr(min)', () => { - ageColumn.aggr = 'min' - config.axis[chart].aggrAxis.push(ageColumn) + ageColumn.aggr = 'min'; + config.axis[chart].aggrAxis.push(ageColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, key1Names, key2Names, selectors, } = transformer() + const {rows, key1Names, key2Names, selectors} = transformer(); - expect(key1Names).toEqual([]) - expect(key2Names).toEqual([]) - expect(selectors).toEqual([ 'age(min)', ]) + expect(key1Names).toEqual([]); + expect(key2Names).toEqual([]); + expect(selectors).toEqual(['age(min)']); expect(rows).toEqual([ - { selector: 'age(min)', value: [ { aggregated: 33, }, ] }, - ]) - }) + {selector: 'age(min)', value: [{aggregated: 33}]}, + ]); + }); it('should transform properly: 0 key1, 0 key2, 0 group, 2 aggr(sum)', () => { - ageColumn.aggr = 'sum' - balanceColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].aggrAxis.push(balanceColumn) + ageColumn.aggr = 'sum'; + balanceColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].aggrAxis.push(balanceColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, groupNames, selectors, } = transformer() + const {rows, groupNames, selectors} = transformer(); - expect(groupNames).toEqual([ 'age(sum)', 'balance(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', 'balance(sum)', ]) + expect(groupNames).toEqual(['age(sum)', 'balance(sum)']); + expect(selectors).toEqual(['age(sum)', 'balance(sum)']); expect(rows).toEqual([ - { selector: 'age(sum)', value: [ { aggregated: 159 } ] }, - { selector: 'balance(sum)', value: [ { aggregated: 14181 }, ] }, - ]) - }) + {selector: 'age(sum)', value: [{aggregated: 159}]}, + {selector: 'balance(sum)', value: [{aggregated: 14181}]}, + ]); + }); it('should transform properly: 0 key1, 0 key2, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].groupAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].groupAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, groupNames, selectors, } = transformer() + const {rows, groupNames, selectors} = transformer(); - expect(groupNames).toEqual([ 'married', 'single', ]) - expect(selectors).toEqual([ 'married', 'single', ]) + expect(groupNames).toEqual(['married', 'single']); + expect(selectors).toEqual(['married', 'single']); expect(rows).toEqual([ - { selector: 'married', value: [ { aggregated: 82 }, ] }, - { selector: 'single', value: [ { aggregated: 77 }, ] }, - ]) - }) + {selector: 'married', value: [{aggregated: 82}]}, + {selector: 'single', value: [{aggregated: 77}]}, + ]); + }); it('should transform properly: 0 key1, 0 key2, 1 group, 2 aggr(sum)', () => { - ageColumn.aggr = 'sum' - balanceColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].aggrAxis.push(balanceColumn) - config.axis[chart].groupAxis.push(martialColumn) + ageColumn.aggr = 'sum'; + balanceColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].aggrAxis.push(balanceColumn); + config.axis[chart].groupAxis.push(martialColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, groupNames, selectors, } = transformer() + const {rows, groupNames, selectors} = transformer(); - expect(groupNames).toEqual([ 'married', 'single', ]) + expect(groupNames).toEqual(['married', 'single']); expect(selectors).toEqual([ 'married / age(sum)', 'married / balance(sum)', 'single / age(sum)', 'single / balance(sum)', - ]) + ]); expect(rows).toEqual([ - { selector: 'married / age(sum)', value: [ { aggregated: 82 }, ] }, - { selector: 'married / balance(sum)', value: [ { aggregated: 9286 }, ] }, - { selector: 'single / age(sum)', value: [ { aggregated: 77 }, ] }, - { selector: 'single / balance(sum)', value: [ { aggregated: 4895 }, ] }, - ]) - }) + {selector: 'married / age(sum)', value: [{aggregated: 82}]}, + {selector: 'married / balance(sum)', value: [{aggregated: 9286}]}, + {selector: 'single / age(sum)', value: [{aggregated: 77}]}, + {selector: 'single / balance(sum)', value: [{aggregated: 4895}]}, + ]); + }); it('should transform properly: 0 key1, 0 key2, 2 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].groupAxis.push(martialColumn) - config.axis[chart].groupAxis.push(educationColumn) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].groupAxis.push(martialColumn); + config.axis[chart].groupAxis.push(educationColumn); - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; - const { rows, groupNames, selectors, } = transformer() + const {rows, groupNames, selectors} = transformer(); - expect(groupNames).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) - expect(selectors).toEqual([ 'married.primary', 'married.secondary', 'single.tertiary', ]) + expect(groupNames).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); + expect(selectors).toEqual(['married.primary', 'married.secondary', 'single.tertiary']); expect(rows).toEqual([ - { selector: 'married.primary', value: [ { aggregated: '43' }, ] }, - { selector: 'married.secondary', value: [ { aggregated: '39' }, ] }, - { selector: 'single.tertiary', value: [ { aggregated: 77 }, ] }, - ]) - }) + {selector: 'married.primary', value: [{aggregated: '43'}]}, + {selector: 'married.secondary', value: [{aggregated: '39'}]}, + {selector: 'single.tertiary', value: [{aggregated: 77}]}, + ]); + }); it('should transform properly: 1 key1, 0 key2, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].key1Axis.push(balanceColumn) - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer - - const { rows, key1Names, key1ColumnName, - key2Names, key2ColumnName, groupNames, selectors, } = transformer() - - expect(key1Names).toEqual([ '-88', '106', '4789', '9374' ]) - expect(key1ColumnName).toEqual('balance') - expect(key2Names).toEqual([]) - expect(key2ColumnName).toEqual('') - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].key1Axis.push(balanceColumn); + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; + + const {rows, key1Names, key1ColumnName, + key2Names, key2ColumnName, groupNames, selectors} = transformer(); + + expect(key1Names).toEqual(['-88', '106', '4789', '9374']); + expect(key1ColumnName).toEqual('balance'); + expect(key2Names).toEqual([]); + expect(key2ColumnName).toEqual(''); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ { selector: 'age(sum)', value: [ - { aggregated: '43', key1: '-88' }, - { aggregated: '44', key1: '106' }, - { aggregated: '33', key1: '4789' }, - { aggregated: '39', key1: '9374' }, - ] - } - ]) - }) + {aggregated: '43', key1: '-88'}, + {aggregated: '44', key1: '106'}, + {aggregated: '33', key1: '4789'}, + {aggregated: '39', key1: '9374'}, + ], + }, + ]); + }); it('should transform properly: 0 key1, 1 key2, 0 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].key2Axis.push(balanceColumn) - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer - - const { rows, key1Names, key1ColumnName, - key2Names, key2ColumnName, groupNames, selectors, } = transformer() - - expect(key1Names).toEqual([]) - expect(key1ColumnName).toEqual('') - expect(key2Names).toEqual([ '-88', '106', '4789', '9374' ]) - expect(key2ColumnName).toEqual('balance') - expect(groupNames).toEqual([ 'age(sum)', ]) - expect(selectors).toEqual([ 'age(sum)', ]) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].key2Axis.push(balanceColumn); + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; + + const {rows, key1Names, key1ColumnName, + key2Names, key2ColumnName, groupNames, selectors} = transformer(); + + expect(key1Names).toEqual([]); + expect(key1ColumnName).toEqual(''); + expect(key2Names).toEqual(['-88', '106', '4789', '9374']); + expect(key2ColumnName).toEqual('balance'); + expect(groupNames).toEqual(['age(sum)']); + expect(selectors).toEqual(['age(sum)']); expect(rows).toEqual([ { selector: 'age(sum)', value: [ - { aggregated: '43', key2: '-88' }, - { aggregated: '44', key2: '106' }, - { aggregated: '33', key2: '4789' }, - { aggregated: '39', key2: '9374' }, - ] + {aggregated: '43', key2: '-88'}, + {aggregated: '44', key2: '106'}, + {aggregated: '33', key2: '4789'}, + {aggregated: '39', key2: '9374'}, + ], }, - ]) - }) + ]); + }); it('should transform properly: 1 key1, 0 key2, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].key1Axis.push(balanceColumn) - config.axis[chart].groupAxis.push(educationColumn) - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer - - const { rows, key1Names, key1ColumnName, - key2Names, key2ColumnName, groupNames, selectors, } = transformer() - - expect(key1Names).toEqual([ '-88', '106', '4789', '9374' ]) - expect(key1ColumnName).toEqual('balance') - expect(key2Names).toEqual([]) - expect(key2ColumnName).toEqual('') - expect(groupNames).toEqual([ 'primary', 'secondary', 'tertiary', ]) - expect(selectors).toEqual([ 'primary', 'secondary', 'tertiary', ]) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].key1Axis.push(balanceColumn); + config.axis[chart].groupAxis.push(educationColumn); + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; + + const {rows, key1Names, key1ColumnName, + key2Names, key2ColumnName, groupNames, selectors} = transformer(); + + expect(key1Names).toEqual(['-88', '106', '4789', '9374']); + expect(key1ColumnName).toEqual('balance'); + expect(key2Names).toEqual([]); + expect(key2ColumnName).toEqual(''); + expect(groupNames).toEqual(['primary', 'secondary', 'tertiary']); + expect(selectors).toEqual(['primary', 'secondary', 'tertiary']); expect(rows).toEqual([ - { selector: 'primary', value: [ { aggregated: '43', key1: '-88' }, ] }, - { selector: 'secondary', value: [ { aggregated: '39', key1: '9374' }, ] }, + {selector: 'primary', value: [{aggregated: '43', key1: '-88'}]}, + {selector: 'secondary', value: [{aggregated: '39', key1: '9374'}]}, { selector: 'tertiary', value: [ - { aggregated: '44', key1: '106' }, - { aggregated: '33', key1: '4789' }, - ] + {aggregated: '44', key1: '106'}, + {aggregated: '33', key1: '4789'}, + ], }, - ]) - }) + ]); + }); it('should transform properly: 0 key1, 1 key2, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].key2Axis.push(balanceColumn) - config.axis[chart].groupAxis.push(educationColumn) - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer - - const { rows, key1Names, key1ColumnName, - key2Names, key2ColumnName, groupNames, selectors, } = transformer() - - expect(key1Names).toEqual([]) - expect(key1ColumnName).toEqual('') - expect(key2Names).toEqual([ '-88', '106', '4789', '9374' ]) - expect(key2ColumnName).toEqual('balance') - expect(groupNames).toEqual([ 'primary', 'secondary', 'tertiary', ]) - expect(selectors).toEqual([ 'primary', 'secondary', 'tertiary', ]) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].key2Axis.push(balanceColumn); + config.axis[chart].groupAxis.push(educationColumn); + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; + + const {rows, key1Names, key1ColumnName, + key2Names, key2ColumnName, groupNames, selectors} = transformer(); + + expect(key1Names).toEqual([]); + expect(key1ColumnName).toEqual(''); + expect(key2Names).toEqual(['-88', '106', '4789', '9374']); + expect(key2ColumnName).toEqual('balance'); + expect(groupNames).toEqual(['primary', 'secondary', 'tertiary']); + expect(selectors).toEqual(['primary', 'secondary', 'tertiary']); expect(rows).toEqual([ - { selector: 'primary', value: [ { aggregated: '43', key2: '-88' }, ] }, - { selector: 'secondary', value: [ { aggregated: '39', key2: '9374' }, ] }, + {selector: 'primary', value: [{aggregated: '43', key2: '-88'}]}, + {selector: 'secondary', value: [{aggregated: '39', key2: '9374'}]}, { selector: 'tertiary', value: [ - { aggregated: '44', key2: '106' }, - { aggregated: '33', key2: '4789' }, - ] + {aggregated: '44', key2: '106'}, + {aggregated: '33', key2: '4789'}, + ], }, - ]) - }) + ]); + }); it('should transform properly: 1 key1, 1 key2, 1 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].key1Axis.push(pDaysColumn) - config.axis[chart].key2Axis.push(balanceColumn) - config.axis[chart].groupAxis.push(educationColumn) - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer - - const { rows, key1Names, key1ColumnName, - key2Names, key2ColumnName, groupNames, selectors, } = transformer() - - expect(key1Names).toEqual([ '-1', '147', '339', ]) - expect(key1ColumnName).toEqual('pdays') - expect(key2Names).toEqual([ '-88', '106', '4789', '9374' ]) - expect(key2ColumnName).toEqual('balance') - expect(groupNames).toEqual([ 'primary', 'secondary', 'tertiary', ]) - expect(selectors).toEqual([ 'primary', 'secondary', 'tertiary', ]) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].key1Axis.push(pDaysColumn); + config.axis[chart].key2Axis.push(balanceColumn); + config.axis[chart].groupAxis.push(educationColumn); + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; + + const {rows, key1Names, key1ColumnName, + key2Names, key2ColumnName, groupNames, selectors} = transformer(); + + expect(key1Names).toEqual(['-1', '147', '339']); + expect(key1ColumnName).toEqual('pdays'); + expect(key2Names).toEqual(['-88', '106', '4789', '9374']); + expect(key2ColumnName).toEqual('balance'); + expect(groupNames).toEqual(['primary', 'secondary', 'tertiary']); + expect(selectors).toEqual(['primary', 'secondary', 'tertiary']); expect(rows).toEqual([ { selector: 'primary', - value: [ { aggregated: '43', key1: '147', key2: '-88' }, ] + value: [{aggregated: '43', key1: '147', key2: '-88'}], }, { selector: 'secondary', - value: [ { aggregated: '39', key1: '-1', key2: '9374' }, ] + value: [{aggregated: '39', key1: '-1', key2: '9374'}], }, { selector: 'tertiary', value: [ - { aggregated: '44', key1: '-1', key2: '106' }, - { aggregated: '33', key1: '339', key2: '4789' }, - ] + {aggregated: '44', key1: '-1', key2: '106'}, + {aggregated: '33', key1: '339', key2: '4789'}, + ], }, - ]) - }) + ]); + }); it('should transform properly: 1 key1, 1 key2, 2 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'sum' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].key1Axis.push(pDaysColumn) - config.axis[chart].key2Axis.push(balanceColumn) - config.axis[chart].groupAxis.push(educationColumn) - config.axis[chart].groupAxis.push(martialColumn) - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer - - const { rows, key1Names, key1ColumnName, - key2Names, key2ColumnName, groupNames, selectors, } = transformer() - - expect(key1Names).toEqual([ '-1', '147', '339', ]) - expect(key1ColumnName).toEqual('pdays') - expect(key2Names).toEqual([ '-88', '106', '4789', '9374' ]) - expect(key2ColumnName).toEqual('balance') - expect(groupNames).toEqual([ 'primary.married', 'secondary.married', 'tertiary.single', ]) - expect(selectors).toEqual([ 'primary.married', 'secondary.married', 'tertiary.single', ]) + ageColumn.aggr = 'sum'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].key1Axis.push(pDaysColumn); + config.axis[chart].key2Axis.push(balanceColumn); + config.axis[chart].groupAxis.push(educationColumn); + config.axis[chart].groupAxis.push(martialColumn); + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; + + const {rows, key1Names, key1ColumnName, + key2Names, key2ColumnName, groupNames, selectors} = transformer(); + + expect(key1Names).toEqual(['-1', '147', '339']); + expect(key1ColumnName).toEqual('pdays'); + expect(key2Names).toEqual(['-88', '106', '4789', '9374']); + expect(key2ColumnName).toEqual('balance'); + expect(groupNames).toEqual(['primary.married', 'secondary.married', 'tertiary.single']); + expect(selectors).toEqual(['primary.married', 'secondary.married', 'tertiary.single']); expect(rows).toEqual([ { selector: 'primary.married', - value: [ { aggregated: '43', key1: '147', key2: '-88'}, ] + value: [{aggregated: '43', key1: '147', key2: '-88'}], }, { selector: 'secondary.married', - value: [ { aggregated: '39', key1: '-1', key2: '9374' }, ] + value: [{aggregated: '39', key1: '-1', key2: '9374'}], }, { selector: 'tertiary.single', value: [ - { aggregated: '44', key1: '-1', key2: '106' }, - { aggregated: '33', key1: '339', key2: '4789' }, - ] + {aggregated: '44', key1: '-1', key2: '106'}, + {aggregated: '33', key1: '339', key2: '4789'}, + ], }, - ]) - }) + ]); + }); it('should transform properly: 1 key1, 1 key2, 2 group, 1 aggr(sum)', () => { - ageColumn.aggr = 'min' - daysColumn.aggr = 'max' - config.axis[chart].aggrAxis.push(ageColumn) - config.axis[chart].aggrAxis.push(daysColumn) - config.axis[chart].key1Axis.push(pDaysColumn) - config.axis[chart].key2Axis.push(balanceColumn) - config.axis[chart].groupAxis.push(martialColumn) - - const axisSpecs = config.axisSpecs[chart] - const axis = config.axis[chart] - const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer - - const { rows, key1Names, key1ColumnName, - key2Names, key2ColumnName, groupNames, selectors, } = transformer() - - expect(key1Names).toEqual([ '-1', '147', '339', ]) - expect(key1ColumnName).toEqual('pdays') - expect(key2Names).toEqual([ '-88', '106', '4789', '9374' ]) - expect(key2ColumnName).toEqual('balance') - expect(groupNames).toEqual([ 'married', 'single', ]) + ageColumn.aggr = 'min'; + daysColumn.aggr = 'max'; + config.axis[chart].aggrAxis.push(ageColumn); + config.axis[chart].aggrAxis.push(daysColumn); + config.axis[chart].key1Axis.push(pDaysColumn); + config.axis[chart].key2Axis.push(balanceColumn); + config.axis[chart].groupAxis.push(martialColumn); + + const axisSpecs = config.axisSpecs[chart]; + const axis = config.axis[chart]; + const transformer = Util.getTransformer(config, tableDataRows, axisSpecs, axis).transformer; + + const {rows, key1Names, key1ColumnName, + key2Names, key2ColumnName, groupNames, selectors} = transformer(); + + expect(key1Names).toEqual(['-1', '147', '339']); + expect(key1ColumnName).toEqual('pdays'); + expect(key2Names).toEqual(['-88', '106', '4789', '9374']); + expect(key2ColumnName).toEqual('balance'); + expect(groupNames).toEqual(['married', 'single']); expect(selectors).toEqual( - [ 'married / age(min)', 'married / day(max)', 'single / age(min)', 'single / day(max)', ] - ) + ['married / age(min)', 'married / day(max)', 'single / age(min)', 'single / day(max)'] + ); expect(rows).toEqual([ { selector: 'married / age(min)', value: [ - { aggregated: '39', key1: '-1', key2: '9374' }, - { aggregated: '43', key1: '147', key2: '-88' }, - ] + {aggregated: '39', key1: '-1', key2: '9374'}, + {aggregated: '43', key1: '147', key2: '-88'}, + ], }, { selector: 'married / day(max)', value: [ - { aggregated: '20', key1: '-1', key2: '9374' }, - { aggregated: '17', key1: '147', key2: '-88' }, - ] + {aggregated: '20', key1: '-1', key2: '9374'}, + {aggregated: '17', key1: '147', key2: '-88'}, + ], }, { selector: 'single / age(min)', value: [ - { aggregated: '44', key1: '-1', key2: '106' }, - { aggregated: '33', key1: '339', key2: '4789' }, - ] + {aggregated: '44', key1: '-1', key2: '106'}, + {aggregated: '33', key1: '339', key2: '4789'}, + ], }, { selector: 'single / day(max)', value: [ - { aggregated: '12', key1: '-1', key2: '106' }, - { aggregated: '11', key1: '339', key2: '4789' }, - ] + {aggregated: '12', key1: '-1', key2: '106'}, + {aggregated: '11', key1: '339', key2: '4789'}, + ], }, - ]) - }) - }) // end: describe('method: array:2-key') - }) // end: describe('getTransformer') -}) + ]); + }); + }); // end: describe('method: array:2-key') + }); // end: describe('getTransformer') +}); diff --git a/zeppelin-web/src/app/tabledata/advanced-transformation.js b/zeppelin-web/src/app/tabledata/advanced-transformation.js index 8650de53046..7420bede811 100644 --- a/zeppelin-web/src/app/tabledata/advanced-transformation.js +++ b/zeppelin-web/src/app/tabledata/advanced-transformation.js @@ -12,7 +12,7 @@ * limitations under the License. */ -import Transformation from './transformation' +import Transformation from './transformation'; import { getCurrentChart, getCurrentChartAxis, getCurrentChartParam, @@ -23,46 +23,46 @@ import { removeDuplicatedColumnsInMultiDimensionAxis, applyMaxAxisCount, isInputWidget, isOptionWidget, isCheckboxWidget, isTextareaWidget, parseParameter, getTransformer, -} from './advanced-transformation-util' +} from './advanced-transformation-util'; -const SETTING_TEMPLATE = 'app/tabledata/advanced-transformation-setting.html' +const SETTING_TEMPLATE = 'app/tabledata/advanced-transformation-setting.html'; export default class AdvancedTransformation extends Transformation { constructor(config, spec) { - super(config) + super(config); - this.columns = [] /** [{ name, index, comment }] */ - this.props = {} - this.spec = spec + this.columns = []; /** [{ name, index, comment }] */ + this.props = {}; + this.spec = spec; - initializeConfig(config, spec) + initializeConfig(config, spec); } emitConfigChange(conf) { - conf.chartChanged = false - conf.parameterChanged = false - this.emitConfig(conf) + conf.chartChanged = false; + conf.parameterChanged = false; + this.emitConfig(conf); } emitChartChange(conf) { - conf.chartChanged = true - conf.parameterChanged = false - this.emitConfig(conf) + conf.chartChanged = true; + conf.parameterChanged = false; + this.emitConfig(conf); } emitParameterChange(conf) { - conf.chartChanged = false - conf.parameterChanged = true - this.emitConfig(conf) + conf.chartChanged = false; + conf.parameterChanged = true; + this.emitConfig(conf); } getSetting() { - const self = this /** for closure */ - const configInstance = self.config /** for closure */ + const self = this; /** for closure */ + const configInstance = self.config; /** for closure */ if (self.spec.initialized) { - self.spec.initialized = false - self.emitConfig(configInstance) + self.spec.initialized = false; + self.emitConfig(configInstance); } return { @@ -71,148 +71,174 @@ export default class AdvancedTransformation extends Transformation { config: configInstance, columns: self.columns, resetAxisConfig: () => { - resetAxisConfig(configInstance) - self.emitChartChange(configInstance) + resetAxisConfig(configInstance); + self.emitChartChange(configInstance); }, resetParameterConfig: () => { - resetParameterConfig(configInstance) - self.emitParameterChange(configInstance) + resetParameterConfig(configInstance); + self.emitParameterChange(configInstance); }, toggleColumnPanel: () => { - configInstance.panel.columnPanelOpened = !configInstance.panel.columnPanelOpened - self.emitConfigChange(configInstance) + configInstance.panel.columnPanelOpened = !configInstance.panel.columnPanelOpened; + self.emitConfigChange(configInstance); }, toggleParameterPanel: () => { - configInstance.panel.parameterPanelOpened = !configInstance.panel.parameterPanelOpened - self.emitConfigChange(configInstance) + configInstance.panel.parameterPanelOpened = !configInstance.panel.parameterPanelOpened; + self.emitConfigChange(configInstance); }, getAxisAnnotation: (axisSpec) => { - let anno = `${axisSpec.name}` + let anno = `${axisSpec.name}`; if (axisSpec.valueType) { - anno = `${anno} (${axisSpec.valueType})` + anno = `${anno} (${axisSpec.valueType})`; } - return anno + return anno; }, getAxisTypeAnnotation: (axisSpec) => { - let anno = '' + let anno = ''; - let minAxisCount = axisSpec.minAxisCount - let maxAxisCount = axisSpec.maxAxisCount + let minAxisCount = axisSpec.minAxisCount; + let maxAxisCount = axisSpec.maxAxisCount; if (isSingleDimensionAxis(axisSpec)) { - maxAxisCount = 1 + maxAxisCount = 1; } - let comment = '' - if (minAxisCount) { comment = `min: ${minAxisCount}` } - if (minAxisCount && maxAxisCount) { comment = `${comment}, ` } - if (maxAxisCount) { comment = `${comment}max: ${maxAxisCount}` } + let comment = ''; + if (minAxisCount) { + comment = `min: ${minAxisCount}`; + } + if (minAxisCount && maxAxisCount) { + comment = `${comment}, `; + } + if (maxAxisCount) { + comment = `${comment}max: ${maxAxisCount}`; + } if (comment !== '') { - anno = `${anno} (${comment})` + anno = `${anno} (${comment})`; } - return anno + return anno; }, getAxisAnnotationColor: (axisSpec) => { if (isAggregatorAxis(axisSpec)) { - return { 'background-color': '#5782bd' } + return {'background-color': '#5782bd'}; } else if (isGroupAxis(axisSpec)) { - return { 'background-color': '#cd5c5c' } + return {'background-color': '#cd5c5c'}; } else if (isKeyAxis(axisSpec)) { - return { 'background-color': '#906ebd' } + return {'background-color': '#906ebd'}; } else { - return { 'background-color': '#62bda9' } + return {'background-color': '#62bda9'}; } }, - useSharedAxis: (chartName) => { return useSharedAxis(configInstance, chartName) }, - isGroupAxis: (axisSpec) => { return isGroupAxis(axisSpec) }, - isKeyAxis: (axisSpec) => { return isKeyAxis(axisSpec) }, - isAggregatorAxis: (axisSpec) => { return isAggregatorAxis(axisSpec) }, - isSingleDimensionAxis: (axisSpec) => { return isSingleDimensionAxis(axisSpec) }, - getSingleDimensionAxis: (axisSpec) => { return getCurrentChartAxis(configInstance)[axisSpec.name] }, + useSharedAxis: (chartName) => { + return useSharedAxis(configInstance, chartName); + }, + isGroupAxis: (axisSpec) => { + return isGroupAxis(axisSpec); + }, + isKeyAxis: (axisSpec) => { + return isKeyAxis(axisSpec); + }, + isAggregatorAxis: (axisSpec) => { + return isAggregatorAxis(axisSpec); + }, + isSingleDimensionAxis: (axisSpec) => { + return isSingleDimensionAxis(axisSpec); + }, + getSingleDimensionAxis: (axisSpec) => { + return getCurrentChartAxis(configInstance)[axisSpec.name]; + }, chartChanged: (selected) => { - configInstance.chart.current = selected - self.emitChartChange(configInstance) + configInstance.chart.current = selected; + self.emitChartChange(configInstance); }, axisChanged: function(e, ui, axisSpec) { - removeDuplicatedColumnsInMultiDimensionAxis(configInstance, axisSpec) - applyMaxAxisCount(configInstance, axisSpec) + removeDuplicatedColumnsInMultiDimensionAxis(configInstance, axisSpec); + applyMaxAxisCount(configInstance, axisSpec); - self.emitChartChange(configInstance) + self.emitChartChange(configInstance); }, aggregatorChanged: (colIndex, axisSpec, aggregator) => { if (isSingleDimensionAxis(axisSpec)) { - getCurrentChartAxis(configInstance)[axisSpec.name].aggr = aggregator + getCurrentChartAxis(configInstance)[axisSpec.name].aggr = aggregator; } else { - getCurrentChartAxis(configInstance)[axisSpec.name][colIndex].aggr = aggregator - removeDuplicatedColumnsInMultiDimensionAxis(configInstance, axisSpec) + getCurrentChartAxis(configInstance)[axisSpec.name][colIndex].aggr = aggregator; + removeDuplicatedColumnsInMultiDimensionAxis(configInstance, axisSpec); } - self.emitChartChange(configInstance) + self.emitChartChange(configInstance); }, removeFromAxis: function(colIndex, axisSpec) { if (isSingleDimensionAxis(axisSpec)) { - getCurrentChartAxis(configInstance)[axisSpec.name] = null + getCurrentChartAxis(configInstance)[axisSpec.name] = null; } else { - getCurrentChartAxis(configInstance)[axisSpec.name].splice(colIndex, 1) + getCurrentChartAxis(configInstance)[axisSpec.name].splice(colIndex, 1); } - self.emitChartChange(configInstance) + self.emitChartChange(configInstance); }, - isInputWidget: function(paramSpec) { return isInputWidget(paramSpec) }, - isCheckboxWidget: function(paramSpec) { return isCheckboxWidget(paramSpec) }, - isOptionWidget: function(paramSpec) { return isOptionWidget(paramSpec) }, - isTextareaWidget: function(paramSpec) { return isTextareaWidget(paramSpec) }, + isInputWidget: function(paramSpec) { + return isInputWidget(paramSpec); + }, + isCheckboxWidget: function(paramSpec) { + return isCheckboxWidget(paramSpec); + }, + isOptionWidget: function(paramSpec) { + return isOptionWidget(paramSpec); + }, + isTextareaWidget: function(paramSpec) { + return isTextareaWidget(paramSpec); + }, parameterChanged: (paramSpec) => { - configInstance.chartChanged = false - configInstance.parameterChanged = true - self.emitParameterChange(configInstance) + configInstance.chartChanged = false; + configInstance.parameterChanged = true; + self.emitParameterChange(configInstance); }, parameterOnKeyDown: function(event, paramSpec) { - const code = event.keyCode || event.which + const code = event.keyCode || event.which; if (code === 13 && isInputWidget(paramSpec)) { - self.emitParameterChange(configInstance) + self.emitParameterChange(configInstance); } else if (code === 13 && event.shiftKey && isTextareaWidget(paramSpec)) { - self.emitParameterChange(configInstance) + self.emitParameterChange(configInstance); } - event.stopPropagation() /** avoid to conflict with paragraph shortcuts */ + event.stopPropagation(); /** avoid to conflict with paragraph shortcuts */ }, - } - } + }, + }; } transform(tableData) { - this.columns = tableData.columns /** used in `getSetting` */ + this.columns = tableData.columns; /** used in `getSetting` */ /** initialize in `transform` instead of `getSetting` because this method is called before */ - serializeSharedAxes(this.config) + serializeSharedAxes(this.config); - const conf = this.config - const chart = getCurrentChart(conf) - const axis = getCurrentChartAxis(conf) - const axisSpecs = getCurrentChartAxisSpecs(conf) - const param = getCurrentChartParam(conf) - const paramSpecs = getCurrentChartParamSpecs(conf) - const parsedParam = parseParameter(paramSpecs, param) + const conf = this.config; + const chart = getCurrentChart(conf); + const axis = getCurrentChartAxis(conf); + const axisSpecs = getCurrentChartAxisSpecs(conf); + const param = getCurrentChartParam(conf); + const paramSpecs = getCurrentChartParamSpecs(conf); + const parsedParam = parseParameter(paramSpecs, param); - let { transformer, column, } = getTransformer(conf, tableData.rows, axisSpecs, axis) + let {transformer, column} = getTransformer(conf, tableData.rows, axisSpecs, axis); return { chartChanged: conf.chartChanged, @@ -224,6 +250,6 @@ export default class AdvancedTransformation extends Transformation { column: column, transformer: transformer, - } + }; } } diff --git a/zeppelin-web/src/app/tabledata/columnselector.js b/zeppelin-web/src/app/tabledata/columnselector.js index 9fcf2f17369..1998f062b09 100644 --- a/zeppelin-web/src/app/tabledata/columnselector.js +++ b/zeppelin-web/src/app/tabledata/columnselector.js @@ -12,7 +12,7 @@ * limitations under the License. */ -import Transformation from './transformation' +import Transformation from './transformation'; /** * select columns @@ -26,55 +26,55 @@ import Transformation from './transformation' * ] */ export default class ColumnselectorTransformation extends Transformation { - constructor (config, columnSelectorProp) { - super(config) - this.props = columnSelectorProp + constructor(config, columnSelectorProp) { + super(config); + this.props = columnSelectorProp; } - getSetting () { - let self = this - let configObj = self.config + getSetting() { + let self = this; + let configObj = self.config; return { template: 'app/tabledata/columnselector_settings.html', scope: { config: self.config, props: self.props, tableDataColumns: self.tableDataColumns, - save: function () { - self.emitConfig(configObj) + save: function() { + self.emitConfig(configObj); }, - remove: function (selectorName) { - configObj[selectorName] = null - self.emitConfig(configObj) - } - } - } + remove: function(selectorName) { + configObj[selectorName] = null; + self.emitConfig(configObj); + }, + }, + }; } /** * Method will be invoked when tableData or config changes */ - transform (tableData) { - this.tableDataColumns = tableData.columns - this.removeUnknown() - return tableData + transform(tableData) { + this.tableDataColumns = tableData.columns; + this.removeUnknown(); + return tableData; } - removeUnknown () { - let fields = this.config + removeUnknown() { + let fields = this.config; for (let f in fields) { if (fields[f]) { - let found = false + let found = false; for (let i = 0; i < this.tableDataColumns.length; i++) { - let a = fields[f] - let b = this.tableDataColumns[i] + let a = fields[f]; + let b = this.tableDataColumns[i]; if (a.index === b.index && a.name === b.name) { - found = true - break + found = true; + break; } } if (!found && (fields[f] instanceof Object) && !(fields[f] instanceof Array)) { - fields[f] = null + fields[f] = null; } } } diff --git a/zeppelin-web/src/app/tabledata/dataset.js b/zeppelin-web/src/app/tabledata/dataset.js index 762e3008906..ba3ee7d3957 100644 --- a/zeppelin-web/src/app/tabledata/dataset.js +++ b/zeppelin-web/src/app/tabledata/dataset.js @@ -30,7 +30,7 @@ class Dataset { */ const DatasetType = Object.freeze({ NETWORK: 'NETWORK', - TABLE: 'TABLE' -}) + TABLE: 'TABLE', +}); -export {Dataset, DatasetType} +export {Dataset, DatasetType}; diff --git a/zeppelin-web/src/app/tabledata/datasetfactory.js b/zeppelin-web/src/app/tabledata/datasetfactory.js index f2f69c90856..6d19a989c82 100644 --- a/zeppelin-web/src/app/tabledata/datasetfactory.js +++ b/zeppelin-web/src/app/tabledata/datasetfactory.js @@ -12,9 +12,9 @@ * limitations under the License. */ -import TableData from './tabledata' -import NetworkData from './networkdata' -import {DatasetType} from './dataset' +import TableData from './tabledata'; +import NetworkData from './networkdata'; +import {DatasetType} from './dataset'; /** * Create table data object from paragraph table type result @@ -23,11 +23,11 @@ export default class DatasetFactory { createDataset(datasetType) { switch (datasetType) { case DatasetType.NETWORK: - return new NetworkData() + return new NetworkData(); case DatasetType.TABLE: - return new TableData() + return new TableData(); default: - throw new Error('Dataset type not found') + throw new Error('Dataset type not found'); } } } diff --git a/zeppelin-web/src/app/tabledata/datasetfactory.test.js b/zeppelin-web/src/app/tabledata/datasetfactory.test.js index 0beb137e020..807456ab4d2 100644 --- a/zeppelin-web/src/app/tabledata/datasetfactory.test.js +++ b/zeppelin-web/src/app/tabledata/datasetfactory.test.js @@ -12,35 +12,37 @@ * limitations under the License. */ -import NetworkData from './networkdata.js' -import TableData from './tabledata.js' -import {DatasetType} from './dataset.js' -import DatasetFactory from './datasetfactory.js' +import NetworkData from './networkdata.js'; +import TableData from './tabledata.js'; +import {DatasetType} from './dataset.js'; +import DatasetFactory from './datasetfactory.js'; describe('DatasetFactory build', function() { - let df + let df; beforeAll(function() { - df = new DatasetFactory() - }) + df = new DatasetFactory(); + }); it('should create a TableData instance', function() { - let td = df.createDataset(DatasetType.TABLE) - expect(td instanceof TableData).toBeTruthy() - expect(td.columns.length).toBe(0) - expect(td.rows.length).toBe(0) - }) + let td = df.createDataset(DatasetType.TABLE); + expect(td instanceof TableData).toBeTruthy(); + expect(td.columns.length).toBe(0); + expect(td.rows.length).toBe(0); + }); it('should create a NetworkData instance', function() { - let nd = df.createDataset(DatasetType.NETWORK) - expect(nd instanceof NetworkData).toBeTruthy() - expect(nd.columns.length).toBe(0) - expect(nd.rows.length).toBe(0) - expect(nd.graph).toEqual({}) - }) + let nd = df.createDataset(DatasetType.NETWORK); + expect(nd instanceof NetworkData).toBeTruthy(); + expect(nd.columns.length).toBe(0); + expect(nd.rows.length).toBe(0); + expect(nd.graph).toEqual({}); + }); it('should thrown an Error', function() { - expect(function() { df.createDataset('text') }) - .toThrow(new Error('Dataset type not found')) - }) -}) + expect(function() { + df.createDataset('text'); + }) + .toThrow(new Error('Dataset type not found')); + }); +}); diff --git a/zeppelin-web/src/app/tabledata/network.js b/zeppelin-web/src/app/tabledata/network.js index 403ea5b64d5..3566722930d 100644 --- a/zeppelin-web/src/app/tabledata/network.js +++ b/zeppelin-web/src/app/tabledata/network.js @@ -12,37 +12,37 @@ * limitations under the License. */ -import Transformation from './transformation' +import Transformation from './transformation'; /** * trasformation settings for network visualization */ export default class NetworkTransformation extends Transformation { getSetting() { - let self = this - let configObj = self.config + let self = this; + let configObj = self.config; return { template: 'app/tabledata/network_settings.html', scope: { config: configObj, isEmptyObject: function(obj) { - obj = obj || {} - return angular.equals(obj, {}) + obj = obj || {}; + return angular.equals(obj, {}); }, setNetworkLabel: function(label, value) { - configObj.properties[label].selected = value + configObj.properties[label].selected = value; }, saveConfig: function() { - self.emitConfig(configObj) - } - } - } + self.emitConfig(configObj); + }, + }, + }; } setConfig(config) { } transform(networkData) { - return networkData + return networkData; } } diff --git a/zeppelin-web/src/app/tabledata/networkdata.js b/zeppelin-web/src/app/tabledata/networkdata.js index 70cd86ba5b4..368254d3f22 100644 --- a/zeppelin-web/src/app/tabledata/networkdata.js +++ b/zeppelin-web/src/app/tabledata/networkdata.js @@ -12,74 +12,74 @@ * limitations under the License. */ -import TableData from './tabledata' -import {DatasetType} from './dataset' +import TableData from './tabledata'; +import {DatasetType} from './dataset'; /** * Create network data object from paragraph graph type result */ export default class NetworkData extends TableData { constructor(graph) { - super() - this.graph = graph || {} + super(); + this.graph = graph || {}; if (this.graph.nodes) { - this.loadParagraphResult({msg: JSON.stringify(graph), type: DatasetType.NETWORK}) + this.loadParagraphResult({msg: JSON.stringify(graph), type: DatasetType.NETWORK}); } } loadParagraphResult(paragraphResult) { if (!paragraphResult || paragraphResult.type !== DatasetType.NETWORK) { - console.log('Can not load paragraph result') - return + console.log('Can not load paragraph result'); + return; } - this.graph = JSON.parse(paragraphResult.msg.trim() || '{}') + this.graph = JSON.parse(paragraphResult.msg.trim() || '{}'); if (!this.graph.nodes) { - console.log('Graph result is empty') - return + console.log('Graph result is empty'); + return; } - this.graph.edges = this.graph.edges || [] + this.graph.edges = this.graph.edges || []; this.networkNodes = angular.equals({}, this.graph.labels || {}) - ? null : {count: this.graph.nodes.length, labels: this.graph.labels} + ? null : {count: this.graph.nodes.length, labels: this.graph.labels}; this.networkRelationships = angular.equals([], this.graph.types || []) - ? null : {count: this.graph.edges.length, types: this.graph.types} + ? null : {count: this.graph.edges.length, types: this.graph.types}; - const rows = [] - const comment = '' - const entities = this.graph.nodes.concat(this.graph.edges) - const baseColumnNames = [{name: 'id', index: 0, aggr: 'sum'}] - const containsLabelField = _.find(entities, (entity) => 'label' in entity) != null + const rows = []; + const comment = ''; + const entities = this.graph.nodes.concat(this.graph.edges); + const baseColumnNames = [{name: 'id', index: 0, aggr: 'sum'}]; + const containsLabelField = _.find(entities, (entity) => 'label' in entity) !== undefined; if (this.graph.labels || this.graph.types || containsLabelField) { - baseColumnNames.push({name: 'label', index: 1, aggr: 'sum'}) + baseColumnNames.push({name: 'label', index: 1, aggr: 'sum'}); } const internalFieldsToJump = ['count', 'size', 'totalCount', - 'data', 'x', 'y', 'labels', 'source', 'target'] - const baseCols = _.map(baseColumnNames, (col) => col.name) - let keys = _.map(entities, (elem) => Object.keys(elem.data || {})) - keys = _.flatten(keys) - keys = _.uniq(keys).filter((key) => baseCols.indexOf(key) === -1) + 'data', 'x', 'y', 'labels', 'source', 'target']; + const baseCols = _.map(baseColumnNames, (col) => col.name); + let keys = _.map(entities, (elem) => Object.keys(elem.data || {})); + keys = _.flatten(keys); + keys = _.uniq(keys).filter((key) => baseCols.indexOf(key) === -1); const entityColumnNames = _.map(keys, (elem, i) => { - return {name: elem, index: i + baseColumnNames.length, aggr: 'sum'} - }) - const columnNames = baseColumnNames.concat(entityColumnNames) + return {name: elem, index: i + baseColumnNames.length, aggr: 'sum'}; + }); + const columnNames = baseColumnNames.concat(entityColumnNames); for (let i = 0; i < entities.length; i++) { - const entity = entities[i] - const col = [] - entity.data = entity.data || {} + const entity = entities[i]; + const col = []; + entity.data = entity.data || {}; for (let j = 0; j < columnNames.length; j++) { - const name = columnNames[j].name + const name = columnNames[j].name; const value = name in entity && internalFieldsToJump.indexOf(name) === -1 - ? entity[name] : entity.data[name] - const parsedValue = value === null || value === undefined ? '' : value - col.push(parsedValue) + ? entity[name] : entity.data[name]; + const parsedValue = value === null || value === undefined ? '' : value; + col.push(parsedValue); } - rows.push(col) + rows.push(col); } - this.comment = comment - this.columns = columnNames - this.rows = rows + this.comment = comment; + this.columns = columnNames; + this.rows = rows; } } diff --git a/zeppelin-web/src/app/tabledata/networkdata.test.js b/zeppelin-web/src/app/tabledata/networkdata.test.js index 739ac19fe36..cd3a12f29c8 100644 --- a/zeppelin-web/src/app/tabledata/networkdata.test.js +++ b/zeppelin-web/src/app/tabledata/networkdata.test.js @@ -12,56 +12,56 @@ * limitations under the License. */ -import NetworkData from './networkdata.js' -import {DatasetType} from './dataset.js' +import NetworkData from './networkdata.js'; +import {DatasetType} from './dataset.js'; describe('NetworkData build', function() { - let nd + let nd; beforeEach(function() { - nd = new NetworkData() - }) + nd = new NetworkData(); + }); it('should initialize the default value', function() { - expect(nd.columns.length).toBe(0) - expect(nd.rows.length).toBe(0) - expect(nd.graph).toEqual({}) - }) + expect(nd.columns.length).toBe(0); + expect(nd.rows.length).toBe(0); + expect(nd.graph).toEqual({}); + }); it('should able to create NetowkData from paragraph result', function() { - let jsonExpected = {nodes: [{id: 1}, {id: 2}], edges: [{source: 2, target: 1, id: 1}]} + let jsonExpected = {nodes: [{id: 1}, {id: 2}], edges: [{source: 2, target: 1, id: 1}]}; nd.loadParagraphResult({ type: DatasetType.NETWORK, - msg: JSON.stringify(jsonExpected) - }) + msg: JSON.stringify(jsonExpected), + }); - expect(nd.columns.length).toBe(1) - expect(nd.rows.length).toBe(3) - expect(nd.graph.nodes[0].id).toBe(jsonExpected.nodes[0].id) - expect(nd.graph.nodes[1].id).toBe(jsonExpected.nodes[1].id) - expect(nd.graph.edges[0].id).toBe(jsonExpected.edges[0].id) - expect(nd.graph.edges[0].source).toBe(jsonExpected.edges[0].source) - expect(nd.graph.edges[0].target).toBe(jsonExpected.edges[0].target) - }) + expect(nd.columns.length).toBe(1); + expect(nd.rows.length).toBe(3); + expect(nd.graph.nodes[0].id).toBe(jsonExpected.nodes[0].id); + expect(nd.graph.nodes[1].id).toBe(jsonExpected.nodes[1].id); + expect(nd.graph.edges[0].id).toBe(jsonExpected.edges[0].id); + expect(nd.graph.edges[0].source).toBe(jsonExpected.edges[0].source); + expect(nd.graph.edges[0].target).toBe(jsonExpected.edges[0].target); + }); it('should able to show data fields source and target', function() { let jsonExpected = {nodes: [{id: 1, data: {source: 'Source'}}, {id: 2, data: {target: 'Target'}}], - edges: [{source: 2, target: 1, id: 1, data: {source: 'Source Edge Data', target: 'Target Edge Data'}}]} + edges: [{source: 2, target: 1, id: 1, data: {source: 'Source Edge Data', target: 'Target Edge Data'}}]}; nd.loadParagraphResult({ type: DatasetType.NETWORK, - msg: JSON.stringify(jsonExpected) - }) + msg: JSON.stringify(jsonExpected), + }); - expect(nd.columns.length).toBe(3) - expect(nd.rows.length).toBe(3) - expect(nd.graph.nodes[0].id).toBe(jsonExpected.nodes[0].id) - expect(nd.graph.nodes[1].id).toBe(jsonExpected.nodes[1].id) - expect(nd.graph.edges[0].id).toBe(jsonExpected.edges[0].id) - expect(nd.graph.edges[0].source).toBe(jsonExpected.edges[0].source) - expect(nd.graph.edges[0].target).toBe(jsonExpected.edges[0].target) - expect(nd.graph.nodes[0].data.source).toBe(jsonExpected.nodes[0].data.source) - expect(nd.graph.nodes[1].data.target).toBe(jsonExpected.nodes[1].data.target) - expect(nd.graph.edges[0].data.source).toBe(jsonExpected.edges[0].data.source) - expect(nd.graph.edges[0].data.target).toBe(jsonExpected.edges[0].data.target) - }) -}) + expect(nd.columns.length).toBe(3); + expect(nd.rows.length).toBe(3); + expect(nd.graph.nodes[0].id).toBe(jsonExpected.nodes[0].id); + expect(nd.graph.nodes[1].id).toBe(jsonExpected.nodes[1].id); + expect(nd.graph.edges[0].id).toBe(jsonExpected.edges[0].id); + expect(nd.graph.edges[0].source).toBe(jsonExpected.edges[0].source); + expect(nd.graph.edges[0].target).toBe(jsonExpected.edges[0].target); + expect(nd.graph.nodes[0].data.source).toBe(jsonExpected.nodes[0].data.source); + expect(nd.graph.nodes[1].data.target).toBe(jsonExpected.nodes[1].data.target); + expect(nd.graph.edges[0].data.source).toBe(jsonExpected.edges[0].data.source); + expect(nd.graph.edges[0].data.target).toBe(jsonExpected.edges[0].data.target); + }); +}); diff --git a/zeppelin-web/src/app/tabledata/passthrough.js b/zeppelin-web/src/app/tabledata/passthrough.js index e376c43a64b..772b7bee5db 100644 --- a/zeppelin-web/src/app/tabledata/passthrough.js +++ b/zeppelin-web/src/app/tabledata/passthrough.js @@ -12,21 +12,21 @@ * limitations under the License. */ -import Transformation from './transformation' +import Transformation from './transformation'; /** * passthough the data */ export default class PassthroughTransformation extends Transformation { // eslint-disable-next-line no-useless-constructor - constructor (config) { - super(config) + constructor(config) { + super(config); } /** * Method will be invoked when tableData or config changes */ - transform (tableData) { - return tableData + transform(tableData) { + return tableData; } } diff --git a/zeppelin-web/src/app/tabledata/pivot.js b/zeppelin-web/src/app/tabledata/pivot.js index da2990043b0..2baa6b5c8d8 100644 --- a/zeppelin-web/src/app/tabledata/pivot.js +++ b/zeppelin-web/src/app/tabledata/pivot.js @@ -12,192 +12,192 @@ * limitations under the License. */ -import Transformation from './transformation' +import Transformation from './transformation'; /** * pivot table data and return d3 chart data */ export default class PivotTransformation extends Transformation { // eslint-disable-next-line no-useless-constructor - constructor (config) { - super(config) + constructor(config) { + super(config); } - getSetting () { - let self = this + getSetting() { + let self = this; - let configObj = self.config - console.log('getSetting', configObj) + let configObj = self.config; + console.log('getSetting', configObj); return { template: 'app/tabledata/pivot_settings.html', scope: { config: configObj.common.pivot, tableDataColumns: self.tableDataColumns, - save: function () { - self.emitConfig(configObj) + save: function() { + self.emitConfig(configObj); }, - removeKey: function (idx) { - configObj.common.pivot.keys.splice(idx, 1) - self.emitConfig(configObj) + removeKey: function(idx) { + configObj.common.pivot.keys.splice(idx, 1); + self.emitConfig(configObj); }, - removeGroup: function (idx) { - configObj.common.pivot.groups.splice(idx, 1) - self.emitConfig(configObj) + removeGroup: function(idx) { + configObj.common.pivot.groups.splice(idx, 1); + self.emitConfig(configObj); }, - removeValue: function (idx) { - configObj.common.pivot.values.splice(idx, 1) - self.emitConfig(configObj) + removeValue: function(idx) { + configObj.common.pivot.values.splice(idx, 1); + self.emitConfig(configObj); }, - setValueAggr: function (idx, aggr) { - configObj.common.pivot.values[idx].aggr = aggr - self.emitConfig(configObj) - } - } - } + setValueAggr: function(idx, aggr) { + configObj.common.pivot.values[idx].aggr = aggr; + self.emitConfig(configObj); + }, + }, + }; } /** * Method will be invoked when tableData or config changes */ - transform (tableData) { - this.tableDataColumns = tableData.columns - this.config.common = this.config.common || {} - this.config.common.pivot = this.config.common.pivot || {} - let config = this.config.common.pivot - let firstTime = (!config.keys && !config.groups && !config.values) + transform(tableData) { + this.tableDataColumns = tableData.columns; + this.config.common = this.config.common || {}; + this.config.common.pivot = this.config.common.pivot || {}; + let config = this.config.common.pivot; + let firstTime = (!config.keys && !config.groups && !config.values); - config.keys = config.keys || [] - config.groups = config.groups || [] - config.values = config.values || [] + config.keys = config.keys || []; + config.groups = config.groups || []; + config.values = config.values || []; - this.removeUnknown() + this.removeUnknown(); if (firstTime) { - this.selectDefault() + this.selectDefault(); } return this.pivot( tableData, config.keys, config.groups, - config.values) + config.values); } - removeUnknown () { - let config = this.config.common.pivot - let tableDataColumns = this.tableDataColumns - let unique = function (list) { + removeUnknown() { + let config = this.config.common.pivot; + let tableDataColumns = this.tableDataColumns; + let unique = function(list) { for (let i = 0; i < list.length; i++) { for (let j = i + 1; j < list.length; j++) { if (angular.equals(list[i], list[j])) { - list.splice(j, 1) - j-- + list.splice(j, 1); + j--; } } } - } + }; - let removeUnknown = function (list) { + let removeUnknown = function(list) { for (let i = 0; i < list.length; i++) { // remove non existing column - let found = false + let found = false; for (let j = 0; j < tableDataColumns.length; j++) { - let a = list[i] - let b = tableDataColumns[j] + let a = list[i]; + let b = tableDataColumns[j]; if (a.index === b.index && a.name === b.name) { - found = true - break + found = true; + break; } } if (!found) { - list.splice(i, 1) + list.splice(i, 1); } } - } + }; - unique(config.keys) - removeUnknown(config.keys) - unique(config.groups) - removeUnknown(config.groups) - removeUnknown(config.values) + unique(config.keys); + removeUnknown(config.keys); + unique(config.groups); + removeUnknown(config.groups); + removeUnknown(config.values); } - selectDefault () { - let config = this.config.common.pivot + selectDefault() { + let config = this.config.common.pivot; if (config.keys.length === 0 && config.groups.length === 0 && config.values.length === 0) { if (config.keys.length === 0 && this.tableDataColumns.length > 0) { - config.keys.push(this.tableDataColumns[0]) + config.keys.push(this.tableDataColumns[0]); } if (config.values.length === 0 && this.tableDataColumns.length > 1) { - config.values.push(this.tableDataColumns[1]) + config.values.push(this.tableDataColumns[1]); } } } - pivot (data, keys, groups, values) { + pivot(data, keys, groups, values) { let aggrFunc = { - sum: function (a, b) { - let varA = (a !== undefined) ? (isNaN(a) ? 0 : parseFloat(a)) : 0 - let varB = (b !== undefined) ? (isNaN(b) ? 0 : parseFloat(b)) : 0 - return varA + varB + sum: function(a, b) { + let varA = (a !== undefined) ? (isNaN(a) ? 0 : parseFloat(a)) : 0; + let varB = (b !== undefined) ? (isNaN(b) ? 0 : parseFloat(b)) : 0; + return varA + varB; }, - count: function (a, b) { - let varA = (a !== undefined) ? parseInt(a) : 0 - let varB = (b !== undefined) ? 1 : 0 - return varA + varB + count: function(a, b) { + let varA = (a !== undefined) ? parseInt(a) : 0; + let varB = (b !== undefined) ? 1 : 0; + return varA + varB; }, - min: function (a, b) { - let aIsValid = isValidNumber(a) - let bIsValid = isValidNumber(b) + min: function(a, b) { + let aIsValid = isValidNumber(a); + let bIsValid = isValidNumber(b); if (!aIsValid) { - return parseFloat(b) + return parseFloat(b); } else if (!bIsValid) { - return parseFloat(a) + return parseFloat(a); } else { - return Math.min(parseFloat(a), parseFloat(b)) + return Math.min(parseFloat(a), parseFloat(b)); } }, - max: function (a, b) { - let aIsValid = isValidNumber(a) - let bIsValid = isValidNumber(b) + max: function(a, b) { + let aIsValid = isValidNumber(a); + let bIsValid = isValidNumber(b); if (!aIsValid) { - return parseFloat(b) + return parseFloat(b); } else if (!bIsValid) { - return parseFloat(a) + return parseFloat(a); } else { - return Math.max(parseFloat(a), parseFloat(b)) + return Math.max(parseFloat(a), parseFloat(b)); } }, - avg: function (a, b, c) { - let varA = (a !== undefined) ? (isNaN(a) ? 0 : parseFloat(a)) : 0 - let varB = (b !== undefined) ? (isNaN(b) ? 0 : parseFloat(b)) : 0 - return varA + varB - } - } + avg: function(a, b, c) { + let varA = (a !== undefined) ? (isNaN(a) ? 0 : parseFloat(a)) : 0; + let varB = (b !== undefined) ? (isNaN(b) ? 0 : parseFloat(b)) : 0; + return varA + varB; + }, + }; let isValidNumber = function(num) { - return num !== undefined && !isNaN(num) - } + return num !== undefined && !isNaN(num); + }; let aggrFuncDiv = { sum: false, count: false, min: false, max: false, - avg: true - } + avg: true, + }; - let schema = {} - let rows = {} + let schema = {}; + let rows = {}; for (let i = 0; i < data.rows.length; i++) { - let row = data.rows[i] - let s = schema - let p = rows + let row = data.rows[i]; + let s = schema; + let p = rows; for (let k = 0; k < keys.length; k++) { - let key = keys[k] + let key = keys[k]; // add key to schema if (!s[key.name]) { @@ -205,22 +205,22 @@ export default class PivotTransformation extends Transformation { order: k, index: key.index, type: 'key', - children: {} - } + children: {}, + }; } - s = s[key.name].children + s = s[key.name].children; // add key to row - let keyKey = row[key.index] + let keyKey = row[key.index]; if (!p[keyKey]) { - p[keyKey] = {} + p[keyKey] = {}; } - p = p[keyKey] + p = p[keyKey]; } for (let g = 0; g < groups.length; g++) { - let group = groups[g] - let groupKey = row[group.index] + let group = groups[g]; + let groupKey = row[group.index]; // add group to schema if (!s[groupKey]) { @@ -228,42 +228,42 @@ export default class PivotTransformation extends Transformation { order: g, index: group.index, type: 'group', - children: {} - } + children: {}, + }; } - s = s[groupKey].children + s = s[groupKey].children; // add key to row if (!p[groupKey]) { - p[groupKey] = {} + p[groupKey] = {}; } - p = p[groupKey] + p = p[groupKey]; } for (let v = 0; v < values.length; v++) { - let value = values[v] - let valueKey = value.name + '(' + value.aggr + ')' + let value = values[v]; + let valueKey = value.name + '(' + value.aggr + ')'; // add value to schema if (!s[valueKey]) { s[valueKey] = { type: 'value', order: v, - index: value.index - } + index: value.index, + }; } // add value to row if (!p[valueKey]) { p[valueKey] = { value: (value.aggr !== 'count') ? row[value.index] : 1, - count: 1 - } + count: 1, + }; } else { p[valueKey] = { value: aggrFunc[value.aggr](p[valueKey].value, row[value.index], p[valueKey].count + 1), - count: (aggrFuncDiv[value.aggr]) ? p[valueKey].count + 1 : p[valueKey].count - } + count: (aggrFuncDiv[value.aggr]) ? p[valueKey].count + 1 : p[valueKey].count, + }; } } } @@ -274,7 +274,7 @@ export default class PivotTransformation extends Transformation { groups: groups, values: values, schema: schema, - rows: rows - } + rows: rows, + }; } } diff --git a/zeppelin-web/src/app/tabledata/tabledata.js b/zeppelin-web/src/app/tabledata/tabledata.js index 3fe01b7791c..745ab179050 100644 --- a/zeppelin-web/src/app/tabledata/tabledata.js +++ b/zeppelin-web/src/app/tabledata/tabledata.js @@ -11,65 +11,65 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {Dataset, DatasetType} from './dataset' +import {Dataset, DatasetType} from './dataset'; /** * Create table data object from paragraph table type result */ export default class TableData extends Dataset { - constructor (columns, rows, comment) { - super() - this.columns = columns || [] - this.rows = rows || [] - this.comment = comment || '' + constructor(columns, rows, comment) { + super(); + this.columns = columns || []; + this.rows = rows || []; + this.comment = comment || ''; } - loadParagraphResult (paragraphResult) { + loadParagraphResult(paragraphResult) { if (!paragraphResult || paragraphResult.type !== DatasetType.TABLE) { - console.log('Can not load paragraph result') - return + console.log('Can not load paragraph result'); + return; } - let columnNames = [] - let rows = [] - let array = [] - let textRows = paragraphResult.msg.split('\n') - let comment = '' - let commentRow = false + let columnNames = []; + let rows = []; + let array = []; + let textRows = paragraphResult.msg.split('\n'); + let comment = ''; + let commentRow = false; for (let i = 0; i < textRows.length; i++) { - let textRow = textRows[i] + let textRow = textRows[i]; if (commentRow) { - comment += textRow - continue + comment += textRow; + continue; } if (textRow === '' || textRow === '') { if (rows.length > 0) { - commentRow = true + commentRow = true; } - continue + continue; } - let textCols = textRow.split('\t') - let cols = [] - let cols2 = [] + let textCols = textRow.split('\t'); + let cols = []; + let cols2 = []; for (let j = 0; j < textCols.length; j++) { - let col = textCols[j] + let col = textCols[j]; if (i === 0) { - columnNames.push({name: col, index: j, aggr: 'sum'}) + columnNames.push({name: col, index: j, aggr: 'sum'}); } else { - cols.push(col) - cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: col}) + cols.push(col); + cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: col}); } } if (i !== 0) { - rows.push(cols) - array.push(cols2) + rows.push(cols); + array.push(cols2); } } - this.comment = comment - this.columns = columnNames - this.rows = rows + this.comment = comment; + this.columns = columnNames; + this.rows = rows; } } diff --git a/zeppelin-web/src/app/tabledata/tabledata.test.js b/zeppelin-web/src/app/tabledata/tabledata.test.js index e24b0733924..931cc2d7801 100644 --- a/zeppelin-web/src/app/tabledata/tabledata.test.js +++ b/zeppelin-web/src/app/tabledata/tabledata.test.js @@ -12,42 +12,42 @@ * limitations under the License. */ -import TableData from './tabledata.js' -import PivotTransformation from './pivot.js' +import TableData from './tabledata.js'; +import PivotTransformation from './pivot.js'; -describe('TableData build', function () { - let td +describe('TableData build', function() { + let td; - beforeEach(function () { - console.log(TableData) - td = new TableData() - }) + beforeEach(function() { + console.log(TableData); + td = new TableData(); + }); - it('should initialize the default value', function () { - expect(td.columns.length).toBe(0) - expect(td.rows.length).toBe(0) - expect(td.comment).toBe('') - }) + it('should initialize the default value', function() { + expect(td.columns.length).toBe(0); + expect(td.rows.length).toBe(0); + expect(td.comment).toBe(''); + }); - it('should able to create Tabledata from paragraph result', function () { + it('should able to create Tabledata from paragraph result', function() { td.loadParagraphResult({ type: 'TABLE', - msg: 'key\tvalue\na\t10\nb\t20\n\nhello' - }) + msg: 'key\tvalue\na\t10\nb\t20\n\nhello', + }); - expect(td.columns.length).toBe(2) - expect(td.rows.length).toBe(2) - expect(td.comment).toBe('hello') - }) -}) + expect(td.columns.length).toBe(2); + expect(td.rows.length).toBe(2); + expect(td.comment).toBe('hello'); + }); +}); describe('PivotTransformation build', function() { - let pt + let pt; - beforeEach(function () { - console.log(PivotTransformation) - pt = new PivotTransformation() - }) + beforeEach(function() { + console.log(PivotTransformation); + pt = new PivotTransformation(); + }); it('check the result of keys, groups and values unique', function() { // set inited mock data @@ -63,33 +63,33 @@ describe('PivotTransformation build', function() { {index: 3, name: '3'}, {index: 5, name: '5'}], groups: [], - values: [] - } - } - } + values: [], + }, + }, + }; pt.tableDataColumns = [ {index: 1, name: '1'}, {index: 2, name: '2'}, {index: 3, name: '3'}, {index: 4, name: '4'}, - {index: 5, name: '5'}] + {index: 5, name: '5'}]; - pt.setConfig(config) + pt.setConfig(config); - pt.removeUnknown() + pt.removeUnknown(); - expect(config.common.pivot.keys.length).toBe(3) - expect(config.common.pivot.keys[0].index).toBe(4) - expect(config.common.pivot.keys[1].index).toBe(3) - expect(config.common.pivot.keys[2].index).toBe(5) - }) + expect(config.common.pivot.keys.length).toBe(3); + expect(config.common.pivot.keys[0].index).toBe(4); + expect(config.common.pivot.keys[1].index).toBe(3); + expect(config.common.pivot.keys[2].index).toBe(5); + }); it('should aggregate values correctly', function() { - let td = new TableData() + let td = new TableData(); td.loadParagraphResult({ type: 'TABLE', - msg: 'key\tvalue\na\t10\na\tnull\na\t0\na\t1\n' - }) + msg: 'key\tvalue\na\t10\na\tnull\na\t0\na\t1\n', + }); let config = { common: { @@ -98,34 +98,34 @@ describe('PivotTransformation build', function() { { 'name': 'key', 'index': 0.0, - } + }, ], groups: [], values: [ { 'name': 'value', 'index': 1.0, - 'aggr': 'sum' - } - ] - } - } - } - - pt.setConfig(config) - let transformed = pt.transform(td) - expect(transformed.rows['a']['value(sum)'].value).toBe(11) - - pt.config.common.pivot.values[0].aggr = 'max' - transformed = pt.transform(td) - expect(transformed.rows['a']['value(max)'].value).toBe(10) - - pt.config.common.pivot.values[0].aggr = 'min' - transformed = pt.transform(td) - expect(transformed.rows['a']['value(min)'].value).toBe(0) - - pt.config.common.pivot.values[0].aggr = 'count' - transformed = pt.transform(td) - expect(transformed.rows['a']['value(count)'].value).toBe(4) - }) -}) + 'aggr': 'sum', + }, + ], + }, + }, + }; + + pt.setConfig(config); + let transformed = pt.transform(td); + expect(transformed.rows['a']['value(sum)'].value).toBe(11); + + pt.config.common.pivot.values[0].aggr = 'max'; + transformed = pt.transform(td); + expect(transformed.rows['a']['value(max)'].value).toBe(10); + + pt.config.common.pivot.values[0].aggr = 'min'; + transformed = pt.transform(td); + expect(transformed.rows['a']['value(min)'].value).toBe(0); + + pt.config.common.pivot.values[0].aggr = 'count'; + transformed = pt.transform(td); + expect(transformed.rows['a']['value(count)'].value).toBe(4); + }); +}); diff --git a/zeppelin-web/src/app/tabledata/transformation.js b/zeppelin-web/src/app/tabledata/transformation.js index f142618283c..a15e12b3a51 100644 --- a/zeppelin-web/src/app/tabledata/transformation.js +++ b/zeppelin-web/src/app/tabledata/transformation.js @@ -16,9 +16,9 @@ * Base class for visualization */ export default class Transformation { - constructor (config) { - this.config = config - this._emitter = () => {} + constructor(config) { + this.config = config; + this._emitter = () => {}; } /** @@ -27,77 +27,81 @@ export default class Transformation { * scope : an object to bind to template scope * } */ - getSetting () { + getSetting() { // override this } /** * Method will be invoked when tableData or config changes */ - transform (tableData) { + transform(tableData) { // override this } /** * render setting */ - renderSetting (targetEl) { - let setting = this.getSetting() + renderSetting(targetEl) { + let setting = this.getSetting(); if (!setting) { - return + return; } // already readered if (this._scope) { - let self = this - this._scope.$apply(function () { + let self = this; + this._scope.$apply(function() { for (let k in setting.scope) { - self._scope[k] = setting.scope[k] + if(setting.scope.hasOwnProperty(k)) { + self._scope[k] = setting.scope[k]; + } } for (let k in self._prevSettingScope) { if (!setting.scope[k]) { - self._scope[k] = setting.scope[k] + self._scope[k] = setting.scope[k]; } } - }) - return + }); + return; } else { - this._prevSettingScope = setting.scope + this._prevSettingScope = setting.scope; } - let scope = this._createNewScope() + let scope = this._createNewScope(); for (let k in setting.scope) { - scope[k] = setting.scope[k] + if(setting.scope.hasOwnProperty(k)) { + scope[k] = setting.scope[k]; + } } - let template = setting.template + let template = setting.template; if (template.split('\n').length === 1 && template.endsWith('.html')) { // template is url - let self = this - this._templateRequest(template).then(function (t) { - self._render(targetEl, t, scope) - }) + let self = this; + this._templateRequest(template).then(function(t) { + self._render(targetEl, t, scope); + }); } else { - this._render(targetEl, template, scope) + this._render(targetEl, template, scope); } } - _render (targetEl, template, scope) { - this._targetEl = targetEl - targetEl.html(template) - this._compile(targetEl.contents())(scope) - this._scope = scope + _render(targetEl, template, scope) { + this._targetEl = targetEl; + targetEl.html(template); + this._compile(targetEl.contents())(scope); + this._scope = scope; } - setConfig (config) { - this.config = config + setConfig(config) { + this.config = config; } /** * Emit config. config will sent to server and saved. */ - emitConfig (config) { - this._emitter(config) + emitConfig(config) { + this._emitter(config); } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-areachart.js b/zeppelin-web/src/app/visualization/builtins/visualization-areachart.js index 494f8ae67f7..886aec9a339 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-areachart.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-areachart.js @@ -12,34 +12,34 @@ * limitations under the License. */ -import Nvd3ChartVisualization from './visualization-nvd3chart' -import PivotTransformation from '../../tabledata/pivot' +import Nvd3ChartVisualization from './visualization-nvd3chart'; +import PivotTransformation from '../../tabledata/pivot'; /** * Visualize data in area chart */ export default class AreachartVisualization extends Nvd3ChartVisualization { - constructor (targetEl, config) { - super(targetEl, config) + constructor(targetEl, config) { + super(targetEl, config); - this.pivot = new PivotTransformation(config) + this.pivot = new PivotTransformation(config); try { - this.config.rotate = {degree: config.rotate.degree} + this.config.rotate = {degree: config.rotate.degree}; } catch (e) { - this.config.rotate = {degree: '-45'} + this.config.rotate = {degree: '-45'}; } } - type () { - return 'stackedAreaChart' + type() { + return 'stackedAreaChart'; } - getTransformation () { - return this.pivot + getTransformation() { + return this.pivot; } - render (pivot) { + render(pivot) { let d3Data = this.d3DataFromPivot( pivot.schema, pivot.rows, @@ -48,108 +48,112 @@ export default class AreachartVisualization extends Nvd3ChartVisualization { pivot.values, false, true, - false) + false); - this.xLabels = d3Data.xLabels - super.render(d3Data) - this.config.changeXLabel(this.config.xLabelStatus) + this.xLabels = d3Data.xLabels; + super.render(d3Data); + this.config.changeXLabel(this.config.xLabelStatus); } /** * Set new config */ - setConfig (config) { - super.setConfig(config) - this.pivot.setConfig(config) + setConfig(config) { + super.setConfig(config); + this.pivot.setConfig(config); } - configureChart (chart) { - let self = this - let configObj = self.config + configureChart(chart) { + let self = this; + let configObj = self.config; - chart.xAxis.tickFormat(function (d) { return self.xAxisTickFormat(d, self.xLabels) }) - chart.yAxis.tickFormat(function (d) { return self.yAxisTickFormat(d) }) - chart.yAxis.axisLabelDistance(50) - chart.useInteractiveGuideline(true) // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691) + chart.xAxis.tickFormat(function(d) { + return self.xAxisTickFormat(d, self.xLabels); + }); + chart.yAxis.tickFormat(function(d) { + return self.yAxisTickFormat(d); + }); + chart.yAxis.axisLabelDistance(50); + chart.useInteractiveGuideline(true); // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691) self.config.changeXLabel = function(type) { switch (type) { case 'default': - self.chart._options['showXAxis'] = true - self.chart._options['margin'] = {bottom: 50} - self.chart.xAxis.rotateLabels(0) - configObj.xLabelStatus = 'default' - break + self.chart._options['showXAxis'] = true; + self.chart._options['margin'] = {bottom: 50}; + self.chart.xAxis.rotateLabels(0); + configObj.xLabelStatus = 'default'; + break; case 'rotate': - self.chart._options['showXAxis'] = true - self.chart._options['margin'] = {bottom: 140} - self.chart.xAxis.rotateLabels(configObj.rotate.degree) - configObj.xLabelStatus = 'rotate' - break + self.chart._options['showXAxis'] = true; + self.chart._options['margin'] = {bottom: 140}; + self.chart.xAxis.rotateLabels(configObj.rotate.degree); + configObj.xLabelStatus = 'rotate'; + break; case 'hide': - self.chart._options['showXAxis'] = false - self.chart._options['margin'] = {bottom: 50} - d3.select('#' + self.targetEl[0].id + '> svg').select('g.nv-axis.nv-x').selectAll('*').remove() - configObj.xLabelStatus = 'hide' - break + self.chart._options['showXAxis'] = false; + self.chart._options['margin'] = {bottom: 50}; + d3.select('#' + self.targetEl[0].id + '> svg').select('g.nv-axis.nv-x').selectAll('*').remove(); + configObj.xLabelStatus = 'hide'; + break; } - self.emitConfig(configObj) - } + self.emitConfig(configObj); + }; self.config.isXLabelStatus = function(type) { if (configObj.xLabelStatus === type) { - return true + return true; } else { - return false + return false; } - } + }; self.config.setDegree = function(type) { - configObj.rotate.degree = type - self.chart.xAxis.rotateLabels(type) - self.emitConfig(configObj) - } + configObj.rotate.degree = type; + self.chart.xAxis.rotateLabels(type); + self.emitConfig(configObj); + }; self.config.isDegreeEmpty = function() { if (configObj.rotate.degree.length > 0) { - return true + return true; } else { - configObj.rotate.degree = '-45' - self.emitConfig(configObj) - return false + configObj.rotate.degree = '-45'; + self.emitConfig(configObj); + return false; } - } + }; - this.chart.style(this.config.style || 'stack') + this.chart.style(this.config.style || 'stack'); - this.chart.dispatch.on('stateChange', function (s) { - self.config.style = s.style + this.chart.dispatch.on('stateChange', function(s) { + self.config.style = s.style; // give some time to animation finish - setTimeout(function () { - self.emitConfig(self.config) - }, 500) - }) + setTimeout(function() { + self.emitConfig(self.config); + }, 500); + }); } getSetting(chart) { - let self = this - let configObj = self.config + let self = this; + let configObj = self.config; // default to visualize xLabel if (typeof (configObj.xLabelStatus) === 'undefined') { - configObj.changeXLabel('default') + configObj.changeXLabel('default'); } if (typeof (configObj.rotate.degree) === 'undefined' || configObj.rotate.degree === '') { - configObj.rotate.degree = '-45' - self.emitConfig(configObj) + configObj.rotate.degree = '-45'; + self.emitConfig(configObj); } return { template: 'app/visualization/builtins/visualization-displayXAxis.html', scope: { - config: configObj - } - } + config: configObj, + }, + }; } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-barchart.js b/zeppelin-web/src/app/visualization/builtins/visualization-barchart.js index 2653af21e7f..e0279d99759 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-barchart.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-barchart.js @@ -12,34 +12,34 @@ * limitations under the License. */ -import Nvd3ChartVisualization from './visualization-nvd3chart' -import PivotTransformation from '../../tabledata/pivot' +import Nvd3ChartVisualization from './visualization-nvd3chart'; +import PivotTransformation from '../../tabledata/pivot'; /** * Visualize data in bar char */ export default class BarchartVisualization extends Nvd3ChartVisualization { - constructor (targetEl, config) { - super(targetEl, config) + constructor(targetEl, config) { + super(targetEl, config); - this.pivot = new PivotTransformation(config) + this.pivot = new PivotTransformation(config); try { - this.config.rotate = {degree: config.rotate.degree} + this.config.rotate = {degree: config.rotate.degree}; } catch (e) { - this.config.rotate = {degree: '-45'} + this.config.rotate = {degree: '-45'}; } } - type () { - return 'multiBarChart' + type() { + return 'multiBarChart'; } - getTransformation () { - return this.pivot + getTransformation() { + return this.pivot; } - render (pivot) { + render(pivot) { let d3Data = this.d3DataFromPivot( pivot.schema, pivot.rows, @@ -48,96 +48,98 @@ export default class BarchartVisualization extends Nvd3ChartVisualization { pivot.values, true, true, - true) + true); - super.render(d3Data) - this.config.changeXLabel(this.config.xLabelStatus) + super.render(d3Data); + this.config.changeXLabel(this.config.xLabelStatus); } /** * Set new config */ - setConfig (config) { - super.setConfig(config) - this.pivot.setConfig(config) + setConfig(config) { + super.setConfig(config); + this.pivot.setConfig(config); } - configureChart (chart) { - let self = this - let configObj = self.config + configureChart(chart) { + let self = this; + let configObj = self.config; - chart.yAxis.axisLabelDistance(50) - chart.yAxis.tickFormat(function (d) { return self.yAxisTickFormat(d) }) + chart.yAxis.axisLabelDistance(50); + chart.yAxis.tickFormat(function(d) { + return self.yAxisTickFormat(d); + }); - self.chart.stacked(this.config.stacked) + self.chart.stacked(this.config.stacked); self.config.changeXLabel = function(type) { switch (type) { case 'default': - self.chart._options['showXAxis'] = true - self.chart._options['margin'] = {bottom: 50} - self.chart.xAxis.rotateLabels(0) - configObj.xLabelStatus = 'default' - break + self.chart._options['showXAxis'] = true; + self.chart._options['margin'] = {bottom: 50}; + self.chart.xAxis.rotateLabels(0); + configObj.xLabelStatus = 'default'; + break; case 'rotate': - self.chart._options['showXAxis'] = true - self.chart._options['margin'] = {bottom: 140} - self.chart.xAxis.rotateLabels(configObj.rotate.degree) - configObj.xLabelStatus = 'rotate' - break + self.chart._options['showXAxis'] = true; + self.chart._options['margin'] = {bottom: 140}; + self.chart.xAxis.rotateLabels(configObj.rotate.degree); + configObj.xLabelStatus = 'rotate'; + break; case 'hide': - self.chart._options['showXAxis'] = false - self.chart._options['margin'] = {bottom: 50} - d3.select('#' + self.targetEl[0].id + '> svg').select('g.nv-axis.nv-x').selectAll('*').remove() - configObj.xLabelStatus = 'hide' - break + self.chart._options['showXAxis'] = false; + self.chart._options['margin'] = {bottom: 50}; + d3.select('#' + self.targetEl[0].id + '> svg').select('g.nv-axis.nv-x').selectAll('*').remove(); + configObj.xLabelStatus = 'hide'; + break; } - self.emitConfig(configObj) - } + self.emitConfig(configObj); + }; self.config.isXLabelStatus = function(type) { if (configObj.xLabelStatus === type) { - return true + return true; } else { - return false + return false; } - } + }; self.config.setDegree = function(type) { - configObj.rotate.degree = type - self.chart.xAxis.rotateLabels(type) - self.emitConfig(configObj) - } + configObj.rotate.degree = type; + self.chart.xAxis.rotateLabels(type); + self.emitConfig(configObj); + }; this.chart.dispatch.on('stateChange', function(s) { - configObj.stacked = s.stacked + configObj.stacked = s.stacked; // give some time to animation finish setTimeout(function() { - self.emitConfig(configObj) - }, 500) - }) + self.emitConfig(configObj); + }, 500); + }); } getSetting(chart) { - let self = this - let configObj = self.config + let self = this; + let configObj = self.config; // default to visualize xLabel if (typeof (configObj.xLabelStatus) === 'undefined') { - configObj.changeXLabel('default') + configObj.changeXLabel('default'); } if (typeof (configObj.rotate.degree) === 'undefined' || configObj.rotate.degree === '') { - configObj.rotate.degree = '-45' - self.emitConfig(configObj) + configObj.rotate.degree = '-45'; + self.emitConfig(configObj); } return { template: 'app/visualization/builtins/visualization-displayXAxis.html', scope: { - config: configObj - } - } + config: configObj, + }, + }; } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js b/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js index 46ee25168d9..749e4344dca 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js @@ -12,18 +12,18 @@ * limitations under the License. */ -import Visualization from '../visualization' -import NetworkTransformation from '../../tabledata/network' +import Visualization from '../visualization'; +import NetworkTransformation from '../../tabledata/network'; /** * Visualize data in network format */ export default class NetworkVisualization extends Visualization { constructor(targetEl, config) { - super(targetEl, config) - console.log('Init network viz') + super(targetEl, config); + console.log('Init network viz'); if (!config.properties) { - config.properties = {} + config.properties = {}; } if (!config.d3Graph) { config.d3Graph = { @@ -33,101 +33,101 @@ export default class NetworkVisualization extends Visualization { linkDistance: 80, }, zoom: { - minScale: 1.3 - } - } + minScale: 1.3, + }, + }; } - this.targetEl.addClass('network') - this.containerId = this.targetEl.prop('id') - this.force = null - this.svg = null - this.$timeout = angular.injector(['ng']).get('$timeout') - this.$interpolate = angular.injector(['ng']).get('$interpolate') - this.transformation = new NetworkTransformation(config) + this.targetEl.addClass('network'); + this.containerId = this.targetEl.prop('id'); + this.force = null; + this.svg = null; + this.$timeout = angular.injector(['ng']).get('$timeout'); + this.$interpolate = angular.injector(['ng']).get('$interpolate'); + this.transformation = new NetworkTransformation(config); } refresh() { - console.log('refresh') + console.log('refresh'); } render(networkData) { if (!('graph' in networkData)) { - console.log('graph not found') - return + console.log('graph not found'); + return; } if (!networkData.isRendered) { - networkData.isRendered = true + networkData.isRendered = true; } else { - return + return; } - console.log('Rendering the graph') + console.log('Rendering the graph'); if (networkData.graph.edges.length && !networkData.isDefaultSet) { - networkData.isDefaultSet = true - this._setEdgesDefaults(networkData.graph) + networkData.isDefaultSet = true; + this._setEdgesDefaults(networkData.graph); } - const transformationConfig = this.transformation.getSetting().scope.config - console.log('cfg', transformationConfig) + const transformationConfig = this.transformation.getSetting().scope.config; + console.log('cfg', transformationConfig); if (transformationConfig && angular.equals({}, transformationConfig.properties)) { - transformationConfig.properties = this.getNetworkProperties(networkData.graph) + transformationConfig.properties = this.getNetworkProperties(networkData.graph); } - this.targetEl.empty().append('') + this.targetEl.empty().append(''); - const width = this.targetEl.width() - const height = this.targetEl.height() - const self = this - const defaultOpacity = 0 - const nodeSize = 10 - const textOffset = 3 - const linkSize = 10 + const width = this.targetEl.width(); + const height = this.targetEl.height(); + const self = this; + const defaultOpacity = 0; + const nodeSize = 10; + const textOffset = 3; + const linkSize = 10; const arcPath = (leftHand, d) => { - let start = leftHand ? d.source : d.target - let end = leftHand ? d.target : d.source - let dx = end.x - start.x - let dy = end.y - start.y + let start = leftHand ? d.source : d.target; + let end = leftHand ? d.target : d.source; + let dx = end.x - start.x; + let dy = end.y - start.y; let dr = d.totalCount === 1 - ? 0 : Math.sqrt(Math.pow(dx, 2) + Math.pow(dy, 2)) / (1 + (1 / d.totalCount) * (d.count - 1)) - let sweep = leftHand ? 0 : 1 - return `M${start.x},${start.y}A${dr},${dr} 0 0,${sweep} ${end.x},${end.y}` - } + ? 0 : Math.sqrt(Math.pow(dx, 2) + Math.pow(dy, 2)) / (1 + (1 / d.totalCount) * (d.count - 1)); + let sweep = leftHand ? 0 : 1; + return `M${start.x},${start.y}A${dr},${dr} 0 0,${sweep} ${end.x},${end.y}`; + }; // Use elliptical arc path segments to doubly-encode directionality. const tick = () => { // Links linkPath.attr('d', function(d) { - return arcPath(true, d) - }) + return arcPath(true, d); + }); textPath.attr('d', function(d) { - return arcPath(d.source.x < d.target.x, d) - }) + return arcPath(d.source.x < d.target.x, d); + }); // Nodes - circle.attr('transform', (d) => `translate(${d.x},${d.y})`) - text.attr('transform', (d) => `translate(${d.x},${d.y})`) - } + circle.attr('transform', (d) => `translate(${d.x},${d.y})`); + text.attr('transform', (d) => `translate(${d.x},${d.y})`); + }; const setOpacity = (scale) => { - let opacity = scale >= +transformationConfig.d3Graph.zoom.minScale ? 1 : 0 + let opacity = scale >= +transformationConfig.d3Graph.zoom.minScale ? 1 : 0; this.svg.selectAll('.nodeLabel') - .style('opacity', opacity) + .style('opacity', opacity); this.svg.selectAll('textPath') - .style('opacity', opacity) - } + .style('opacity', opacity); + }; const zoom = d3.behavior.zoom() .scaleExtent([1, 10]) .on('zoom', () => { - console.log('zoom') - setOpacity(d3.event.scale) - container.attr('transform', `translate(${d3.event.translate})scale(${d3.event.scale})`) - }) + console.log('zoom'); + setOpacity(d3.event.scale); + container.attr('transform', `translate(${d3.event.translate})scale(${d3.event.scale})`); + }); this.svg = d3.select(`#${this.containerId} svg`) .attr('width', width) .attr('height', height) - .call(zoom) + .call(zoom); this.force = d3.layout.force() .charge(transformationConfig.d3Graph.forceLayout.charge) @@ -137,54 +137,56 @@ export default class NetworkVisualization extends Visualization { .links(networkData.graph.edges) .size([width, height]) .on('start', () => { - console.log('force layout start') - this.$timeout(() => { this.force.stop() }, transformationConfig.d3Graph.forceLayout.timeout) + console.log('force layout start'); + this.$timeout(() => { + this.force.stop(); + }, transformationConfig.d3Graph.forceLayout.timeout); }) .on('end', () => { - console.log('force layout stop') - setOpacity(zoom.scale()) + console.log('force layout stop'); + setOpacity(zoom.scale()); }) - .start() + .start(); const renderFooterOnClick = (entity, type) => { - const footerId = this.containerId + '_footer' - const obj = {id: entity.id, label: entity.defaultLabel || entity.label, type: type} - let html = [`
  • ${obj.type}_id: ${obj.id}
    • `] + const footerId = this.containerId + '_footer'; + const obj = {id: entity.id, label: entity.defaultLabel || entity.label, type: type}; + let html = [`
  • ${obj.type}_id: ${obj.id}
    • `]; if (obj.label) { - html.push(`
  • ${obj.type}_type: ${obj.label}
    • `) + html.push(`
  • ${obj.type}_type: ${obj.label}
    • `); } html = html.concat(_.map(entity.data, (v, k) => { - return `
  • ${k}: ${v}
    • ` - })) + return `
  • ${k}: ${v}
    • `; + })); angular.element('#' + footerId) .find('.list-inline') .empty() - .append(html.join('')) - } + .append(html.join('')); + }; const drag = d3.behavior.drag() .origin((d) => d) .on('dragstart', function(d) { - console.log('dragstart') - d3.event.sourceEvent.stopPropagation() - d3.select(this).classed('dragging', true) - self.force.stop() + console.log('dragstart'); + d3.event.sourceEvent.stopPropagation(); + d3.select(this).classed('dragging', true); + self.force.stop(); }) .on('drag', function(d) { - console.log('drag') - d.px += d3.event.dx - d.py += d3.event.dy - d.x += d3.event.dx - d.y += d3.event.dy + console.log('drag'); + d.px += d3.event.dx; + d.py += d3.event.dy; + d.x += d3.event.dx; + d.y += d3.event.dy; }) .on('dragend', function(d) { - console.log('dragend') - d.fixed = true - d3.select(this).classed('dragging', false) - self.force.resume() - }) + console.log('dragend'); + d.fixed = true; + d3.select(this).classed('dragging', false); + self.force.resume(); + }); - const container = this.svg.append('g') + const container = this.svg.append('g'); if (networkData.graph.directed) { container.append('svg:defs').selectAll('marker') .data(['arrowMarker-' + this.containerId]) @@ -198,26 +200,26 @@ export default class NetworkVisualization extends Visualization { .attr('markerHeight', 4) .attr('orient', 'auto') .append('svg:path') - .attr('d', 'M0,-5L10,0L0,5') + .attr('d', 'M0,-5L10,0L0,5'); } // Links const link = container.append('svg:g') .on('click', () => { - renderFooterOnClick(d3.select(d3.event.target).datum(), 'edge') + renderFooterOnClick(d3.select(d3.event.target).datum(), 'edge'); }) .selectAll('g.link') .data(self.force.links()) .enter() - .append('g') - const getPathId = (d) => this.containerId + '_' + d.source.index + '_' + d.target.index + '_' + d.count - const showLabel = (d) => this._showNodeLabel(d) + .append('g'); + const getPathId = (d) => this.containerId + '_' + d.source.index + '_' + d.target.index + '_' + d.count; + const showLabel = (d) => this._showNodeLabel(d); const linkPath = link.append('svg:path') .attr('class', 'link') .attr('size', linkSize) - .attr('marker-end', `url(#arrowMarker-${this.containerId})`) + .attr('marker-end', `url(#arrowMarker-${this.containerId})`); const textPath = link.append('svg:path') .attr('id', getPathId) - .attr('class', 'textpath') + .attr('class', 'textpath'); container.append('svg:g') .selectAll('.pathLabel') .data(self.force.links()) @@ -229,11 +231,11 @@ export default class NetworkVisualization extends Visualization { .attr('text-anchor', 'middle') .attr('xlink:href', (d) => '#' + getPathId(d)) .text((d) => d.label) - .style('opacity', defaultOpacity) + .style('opacity', defaultOpacity); // Nodes const circle = container.append('svg:g') .on('click', () => { - renderFooterOnClick(d3.select(d3.event.target).datum(), 'node') + renderFooterOnClick(d3.select(d3.event.target).datum(), 'node'); }) .selectAll('circle') .data(self.force.nodes()) @@ -241,37 +243,37 @@ export default class NetworkVisualization extends Visualization { .attr('r', (d) => nodeSize) .attr('fill', (d) => networkData.graph.labels && d.label in networkData.graph.labels ? networkData.graph.labels[d.label] : '#000000') - .call(drag) + .call(drag); const text = container.append('svg:g').selectAll('g') .data(self.force.nodes()) - .enter().append('svg:g') + .enter().append('svg:g'); text.append('svg:text') .attr('x', (d) => nodeSize + textOffset) .attr('size', nodeSize) .attr('y', '.31em') .attr('class', (d) => 'nodeLabel shadow label-' + d.label) .text(showLabel) - .style('opacity', defaultOpacity) + .style('opacity', defaultOpacity); text.append('svg:text') .attr('x', (d) => nodeSize + textOffset) .attr('size', nodeSize) .attr('y', '.31em') .attr('class', (d) => 'nodeLabel label-' + d.label) .text(showLabel) - .style('opacity', defaultOpacity) + .style('opacity', defaultOpacity); } destroy() { } _showNodeLabel(d) { - const transformationConfig = this.transformation.getSetting().scope.config - const selectedLabel = (transformationConfig.properties[d.label] || {selected: 'label'}).selected - return d.data[selectedLabel] || d[selectedLabel] + const transformationConfig = this.transformation.getSetting().scope.config; + const selectedLabel = (transformationConfig.properties[d.label] || {selected: 'label'}).selected; + return d.data[selectedLabel] || d[selectedLabel]; } getTransformation() { - return this.transformation + return this.transformation; } setNodesDefaults() { @@ -281,56 +283,56 @@ export default class NetworkVisualization extends Visualization { graph.edges .sort((a, b) => { if (a.source > b.source) { - return 1 + return 1; } else if (a.source < b.source) { - return -1 + return -1; } else if (a.target > b.target) { - return 1 + return 1; } else if (a.target < b.target) { - return -1 + return -1; } else { - return 0 + return 0; } - }) + }); graph.edges .forEach((edge, index) => { - let prevEdge = graph.edges[index - 1] + let prevEdge = graph.edges[index - 1]; edge.count = (index > 0 && +edge.source === +prevEdge.source && +edge.target === +prevEdge.target - ? prevEdge.count : 0) + 1 + ? prevEdge.count : 0) + 1; edge.totalCount = graph.edges .filter((innerEdge) => +edge.source === +innerEdge.source && +edge.target === +innerEdge.target) - .length - }) + .length; + }); graph.edges .forEach((edge) => { if (typeof +edge.source === 'number') { // edge.source = graph.nodes.filter((node) => +edge.source === +node.id)[0] || null - edge.source = _.find(graph.nodes, (node) => +edge.source === +node.id) + edge.source = _.find(graph.nodes, (node) => +edge.source === +node.id); } if (typeof +edge.target === 'number') { // edge.target = graph.nodes.filter((node) => +edge.target === +node.id)[0] || null - edge.target = _.find(graph.nodes, (node) => +edge.target === +node.id) + edge.target = _.find(graph.nodes, (node) => +edge.target === +node.id); } - }) + }); } getNetworkProperties(graph) { - const baseCols = ['id', 'label'] - const properties = {} + const baseCols = ['id', 'label']; + const properties = {}; graph.nodes.forEach(function(node) { - const hasLabel = 'label' in node && node.label !== '' + const hasLabel = 'label' in node && node.label !== ''; if (!hasLabel) { - return + return; } - const label = node.label - const hasKey = hasLabel && label in properties + const label = node.label; + const hasKey = hasLabel && label in properties; const keys = _.uniq(Object.keys(node.data || {}) - .concat(hasKey ? properties[label].keys : baseCols)) + .concat(hasKey ? properties[label].keys : baseCols)); if (!hasKey) { - properties[label] = {selected: 'label'} + properties[label] = {selected: 'label'}; } - properties[label].keys = keys - }) - return properties + properties[label].keys = keys; + }); + return properties; } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-linechart.js b/zeppelin-web/src/app/visualization/builtins/visualization-linechart.js index 6d47a9e8d66..df161b98aa8 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-linechart.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-linechart.js @@ -12,39 +12,39 @@ * limitations under the License. */ -import Nvd3ChartVisualization from './visualization-nvd3chart' -import PivotTransformation from '../../tabledata/pivot' -import moment from 'moment' +import Nvd3ChartVisualization from './visualization-nvd3chart'; +import PivotTransformation from '../../tabledata/pivot'; +import moment from 'moment'; /** * Visualize data in line chart */ export default class LinechartVisualization extends Nvd3ChartVisualization { - constructor (targetEl, config) { - super(targetEl, config) + constructor(targetEl, config) { + super(targetEl, config); - this.pivot = new PivotTransformation(config) + this.pivot = new PivotTransformation(config); try { - this.config.rotate = {degree: config.rotate.degree} + this.config.rotate = {degree: config.rotate.degree}; } catch (e) { - this.config.rotate = {degree: '-45'} + this.config.rotate = {degree: '-45'}; } } - type () { + type() { if (this.config.lineWithFocus) { - return 'lineWithFocusChart' + return 'lineWithFocusChart'; } else { - return 'lineChart' + return 'lineChart'; } } - getTransformation () { - return this.pivot + getTransformation() { + return this.pivot; } - render (pivot) { + render(pivot) { let d3Data = this.d3DataFromPivot( pivot.schema, pivot.rows, @@ -53,113 +53,113 @@ export default class LinechartVisualization extends Nvd3ChartVisualization { pivot.values, false, true, - false) + false); - this.xLabels = d3Data.xLabels - super.render(d3Data) - this.config.changeXLabel(this.config.xLabelStatus) + this.xLabels = d3Data.xLabels; + super.render(d3Data); + this.config.changeXLabel(this.config.xLabelStatus); } /** * Set new config */ - setConfig (config) { - super.setConfig(config) - this.pivot.setConfig(config) + setConfig(config) { + super.setConfig(config); + this.pivot.setConfig(config); // change mode if (this.currentMode !== config.lineWithFocus) { - super.destroy() - this.currentMode = config.lineWithFocus + super.destroy(); + this.currentMode = config.lineWithFocus; } } - configureChart (chart) { - let self = this - let configObj = self.config + configureChart(chart) { + let self = this; + let configObj = self.config; - chart.xAxis.tickFormat(function (d) { + chart.xAxis.tickFormat(function(d) { if (self.config.isDateFormat) { if (self.config.dateFormat) { - return moment(new Date(self.xAxisTickFormat(d, self.xLabels))).format(self.config.dateFormat) + return moment(new Date(self.xAxisTickFormat(d, self.xLabels))).format(self.config.dateFormat); } else { - return moment(new Date(self.xAxisTickFormat(d, self.xLabels))).format('YYYY-MM-DD HH:mm:ss') + return moment(new Date(self.xAxisTickFormat(d, self.xLabels))).format('YYYY-MM-DD HH:mm:ss'); } } - return self.xAxisTickFormat(d, self.xLabels) - }) - chart.yAxis.tickFormat(function (d) { + return self.xAxisTickFormat(d, self.xLabels); + }); + chart.yAxis.tickFormat(function(d) { if (d === undefined) { - return 'N/A' + return 'N/A'; } - return self.yAxisTickFormat(d, self.xLabels) - }) - chart.yAxis.axisLabelDistance(50) + return self.yAxisTickFormat(d, self.xLabels); + }); + chart.yAxis.axisLabelDistance(50); if (chart.useInteractiveGuideline) { // lineWithFocusChart hasn't got useInteractiveGuideline - chart.useInteractiveGuideline(true) // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691) + chart.useInteractiveGuideline(true); // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691) } if (this.config.forceY) { - chart.forceY([0]) // force y-axis minimum to 0 for line chart. + chart.forceY([0]); // force y-axis minimum to 0 for line chart. } else { - chart.forceY([]) + chart.forceY([]); } self.config.changeXLabel = function(type) { switch (type) { case 'default': - self.chart._options['showXAxis'] = true - self.chart._options['margin'] = {bottom: 50} - self.chart.xAxis.rotateLabels(0) - configObj.xLabelStatus = 'default' - break + self.chart._options['showXAxis'] = true; + self.chart._options['margin'] = {bottom: 50}; + self.chart.xAxis.rotateLabels(0); + configObj.xLabelStatus = 'default'; + break; case 'rotate': - self.chart._options['showXAxis'] = true - self.chart._options['margin'] = {bottom: 140} - self.chart.xAxis.rotateLabels(configObj.rotate.degree) - configObj.xLabelStatus = 'rotate' - break + self.chart._options['showXAxis'] = true; + self.chart._options['margin'] = {bottom: 140}; + self.chart.xAxis.rotateLabels(configObj.rotate.degree); + configObj.xLabelStatus = 'rotate'; + break; case 'hide': - self.chart._options['showXAxis'] = false - self.chart._options['margin'] = {bottom: 50} - d3.select('#' + self.targetEl[0].id + '> svg').select('g.nv-axis.nv-x').selectAll('*').remove() - configObj.xLabelStatus = 'hide' - break + self.chart._options['showXAxis'] = false; + self.chart._options['margin'] = {bottom: 50}; + d3.select('#' + self.targetEl[0].id + '> svg').select('g.nv-axis.nv-x').selectAll('*').remove(); + configObj.xLabelStatus = 'hide'; + break; } - self.emitConfig(configObj) - } + self.emitConfig(configObj); + }; self.config.isXLabelStatus = function(type) { if (configObj.xLabelStatus === type) { - return true + return true; } else { - return false + return false; } - } + }; self.config.setDegree = function(type) { - configObj.rotate.degree = type - self.chart.xAxis.rotateLabels(type) - self.emitConfig(configObj) - } - - self.config.setDateFormat = function (format) { - configObj.dateFormat = format - self.emitConfig(configObj) - } + configObj.rotate.degree = type; + self.chart.xAxis.rotateLabels(type); + self.emitConfig(configObj); + }; + + self.config.setDateFormat = function(format) { + configObj.dateFormat = format; + self.emitConfig(configObj); + }; } - getSetting (chart) { - let self = this - let configObj = self.config + getSetting(chart) { + let self = this; + let configObj = self.config; // default to visualize xLabel if (typeof (configObj.xLabelStatus) === 'undefined') { - configObj.changeXLabel('default') + configObj.changeXLabel('default'); } if (typeof (configObj.rotate.degree) === 'undefined' || configObj.rotate.degree === '') { - configObj.rotate.degree = '-45' - self.emitConfig(configObj) + configObj.rotate.degree = '-45'; + self.emitConfig(configObj); } return { @@ -199,14 +199,14 @@ export default class LinechartVisualization extends Nvd3ChartVisualization { `, scope: { config: configObj, - save: function () { - self.emitConfig(configObj) - } - } - } + save: function() { + self.emitConfig(configObj); + }, + }, + }; } - defaultY () { - return undefined + defaultY() { + return undefined; } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-nvd3chart.js b/zeppelin-web/src/app/visualization/builtins/visualization-nvd3chart.js index f99fa3da7e8..b3e6ec654e2 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-nvd3chart.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-nvd3chart.js @@ -12,42 +12,42 @@ * limitations under the License. */ -import Visualization from '../visualization' +import Visualization from '../visualization'; /** * Visualize data in table format */ export default class Nvd3ChartVisualization extends Visualization { - constructor (targetEl, config) { - super(targetEl, config) - this.targetEl.append('') + constructor(targetEl, config) { + super(targetEl, config); + this.targetEl.append(''); } - refresh () { + refresh() { if (this.chart) { - this.chart.update() + this.chart.update(); } } - render (data) { - let type = this.type() - let d3g = data.d3g + render(data) { + let type = this.type(); + let d3g = data.d3g; if (!this.chart) { - this.chart = nv.models[type]() + this.chart = nv.models[type](); } - this.configureChart(this.chart) + this.configureChart(this.chart); - let animationDuration = 300 - let numberOfDataThreshold = 150 - let height = this.targetEl.height() + let animationDuration = 300; + let numberOfDataThreshold = 150; + let height = this.targetEl.height(); // turn off animation when dataset is too large. (for performance issue) // still, since dataset is large, the chart content sequentially appears like animated try { if (d3g[0].values.length > numberOfDataThreshold) { - animationDuration = 0 + animationDuration = 0; } } catch (err) { /** ignore */ } @@ -56,206 +56,214 @@ export default class Nvd3ChartVisualization extends Visualization { .datum(d3g) .transition() .duration(animationDuration) - .call(this.chart) - d3.select('#' + this.targetEl[0].id + ' svg').style.height = height + 'px' + .call(this.chart); + d3.select('#' + this.targetEl[0].id + ' svg').style.height = height + 'px'; } - type () { + type() { // override this and return chart type } - configureChart (chart) { + configureChart(chart) { // override this to configure chart } - groupedThousandsWith3DigitsFormatter (x) { - return d3.format(',')(d3.round(x, 3)) + groupedThousandsWith3DigitsFormatter(x) { + return d3.format(',')(d3.round(x, 3)); } - customAbbrevFormatter (x) { - let s = d3.format('.3s')(x) + customAbbrevFormatter(x) { + let s = d3.format('.3s')(x); switch (s[s.length - 1]) { - case 'G': return s.slice(0, -1) + 'B' + case 'G': return s.slice(0, -1) + 'B'; } - return s + return s; } - defaultY () { - return 0 + defaultY() { + return 0; } - xAxisTickFormat (d, xLabels) { + xAxisTickFormat(d, xLabels) { if (xLabels[d] && (isNaN(parseFloat(xLabels[d])) || !isFinite(xLabels[d]))) { // to handle string type xlabel - return xLabels[d] + return xLabels[d]; } else { - return d + return d; } } - yAxisTickFormat (d) { + yAxisTickFormat(d) { if (Math.abs(d) >= Math.pow(10, 6)) { - return this.customAbbrevFormatter(d) + return this.customAbbrevFormatter(d); } - return this.groupedThousandsWith3DigitsFormatter(d) + return this.groupedThousandsWith3DigitsFormatter(d); } - d3DataFromPivot ( + d3DataFromPivot( schema, rows, keys, groups, values, allowTextXAxis, fillMissingValues, multiBarChart) { - let self = this + let self = this; // construct table data - let d3g = [] + let d3g = []; - let concat = function (o, n) { + let concat = function(o, n) { if (!o) { - return n + return n; } else { - return o + '.' + n + return o + '.' + n; } - } + }; - const getSchemaUnderKey = function (key, s) { + const getSchemaUnderKey = function(key, s) { for (let c in key.children) { - s[c] = {} - getSchemaUnderKey(key.children[c], s[c]) + if(key.children.hasOwnProperty(c)) { + s[c] = {}; + getSchemaUnderKey(key.children[c], s[c]); + } } - } + }; - const traverse = function (sKey, s, rKey, r, func, rowName, rowValue, colName) { + const traverse = function(sKey, s, rKey, r, func, rowName, rowValue, colName) { // console.log("TRAVERSE sKey=%o, s=%o, rKey=%o, r=%o, rowName=%o, rowValue=%o, colName=%o", sKey, s, rKey, r, rowName, rowValue, colName); if (s.type === 'key') { - rowName = concat(rowName, sKey) - rowValue = concat(rowValue, rKey) + rowName = concat(rowName, sKey); + rowValue = concat(rowValue, rKey); } else if (s.type === 'group') { - colName = concat(colName, rKey) + colName = concat(colName, rKey); } else if (s.type === 'value' && sKey === rKey || valueOnly) { - colName = concat(colName, rKey) - func(rowName, rowValue, colName, r) + colName = concat(colName, rKey); + func(rowName, rowValue, colName, r); } for (let c in s.children) { if (fillMissingValues && s.children[c].type === 'group' && r[c] === undefined) { - let cs = {} - getSchemaUnderKey(s.children[c], cs) - traverse(c, s.children[c], c, cs, func, rowName, rowValue, colName) - continue + let cs = {}; + getSchemaUnderKey(s.children[c], cs); + traverse(c, s.children[c], c, cs, func, rowName, rowValue, colName); + continue; } for (let j in r) { if (s.children[c].type === 'key' || c === j) { - traverse(c, s.children[c], j, r[j], func, rowName, rowValue, colName) + traverse(c, s.children[c], j, r[j], func, rowName, rowValue, colName); } } } - } + }; - const valueOnly = (keys.length === 0 && groups.length === 0 && values.length > 0) - let noKey = (keys.length === 0) - let isMultiBarChart = multiBarChart + const valueOnly = (keys.length === 0 && groups.length === 0 && values.length > 0); + let noKey = (keys.length === 0); + let isMultiBarChart = multiBarChart; - let sKey = Object.keys(schema)[0] + let sKey = Object.keys(schema)[0]; - let rowNameIndex = {} - let rowIdx = 0 - let colNameIndex = {} - let colIdx = 0 - let rowIndexValue = {} + let rowNameIndex = {}; + let rowIdx = 0; + let colNameIndex = {}; + let colIdx = 0; + let rowIndexValue = {}; for (let k in rows) { - traverse(sKey, schema[sKey], k, rows[k], function (rowName, rowValue, colName, value) { - // console.log("RowName=%o, row=%o, col=%o, value=%o", rowName, rowValue, colName, value); - if (rowNameIndex[rowValue] === undefined) { - rowIndexValue[rowIdx] = rowValue - rowNameIndex[rowValue] = rowIdx++ - } + if (rows.hasOwnProperty(k)) { + traverse(sKey, schema[sKey], k, rows[k], function(rowName, rowValue, colName, value) { + // console.log("RowName=%o, row=%o, col=%o, value=%o", rowName, rowValue, colName, value); + if (rowNameIndex[rowValue] === undefined) { + rowIndexValue[rowIdx] = rowValue; + rowNameIndex[rowValue] = rowIdx++; + } - if (colNameIndex[colName] === undefined) { - colNameIndex[colName] = colIdx++ - } - let i = colNameIndex[colName] - if (noKey && isMultiBarChart) { - i = 0 - } + if (colNameIndex[colName] === undefined) { + colNameIndex[colName] = colIdx++; + } + let i = colNameIndex[colName]; + if (noKey && isMultiBarChart) { + i = 0; + } - if (!d3g[i]) { - d3g[i] = { - values: [], - key: (noKey && isMultiBarChart) ? 'values' : colName + if (!d3g[i]) { + d3g[i] = { + values: [], + key: (noKey && isMultiBarChart) ? 'values' : colName, + }; } - } - let xVar = isNaN(rowValue) ? ((allowTextXAxis) ? rowValue : rowNameIndex[rowValue]) : parseFloat(rowValue) - let yVar = self.defaultY() - if (xVar === undefined) { xVar = colName } - if (value !== undefined) { - yVar = isNaN(value.value) ? self.defaultY() : parseFloat(value.value) / parseFloat(value.count) - } - d3g[i].values.push({ - x: xVar, - y: yVar - }) - }) + let xVar = isNaN(rowValue) ? ((allowTextXAxis) ? rowValue : rowNameIndex[rowValue]) : parseFloat(rowValue); + let yVar = self.defaultY(); + if (xVar === undefined) { + xVar = colName; + } + if (value !== undefined) { + yVar = isNaN(value.value) ? self.defaultY() : parseFloat(value.value) / parseFloat(value.count); + } + d3g[i].values.push({ + x: xVar, + y: yVar, + }); + }); + } } // clear aggregation name, if possible - let namesWithoutAggr = {} - let colName - let withoutAggr + let namesWithoutAggr = {}; + let colName; + let withoutAggr; // TODO - This part could use som refactoring - Weird if/else with similar actions and variable names for (colName in colNameIndex) { - withoutAggr = colName.substring(0, colName.lastIndexOf('(')) - if (!namesWithoutAggr[withoutAggr]) { - namesWithoutAggr[withoutAggr] = 1 - } else { - namesWithoutAggr[withoutAggr]++ + if (colNameIndex.hasOwnProperty(colName)) { + withoutAggr = colName.substring(0, colName.lastIndexOf('(')); + if (!namesWithoutAggr[withoutAggr]) { + namesWithoutAggr[withoutAggr] = 1; + } else { + namesWithoutAggr[withoutAggr]++; + } } } if (valueOnly) { for (let valueIndex = 0; valueIndex < d3g[0].values.length; valueIndex++) { - colName = d3g[0].values[valueIndex].x + colName = d3g[0].values[valueIndex].x; if (!colName) { - continue + continue; } - withoutAggr = colName.substring(0, colName.lastIndexOf('(')) + withoutAggr = colName.substring(0, colName.lastIndexOf('(')); if (namesWithoutAggr[withoutAggr] <= 1) { - d3g[0].values[valueIndex].x = withoutAggr + d3g[0].values[valueIndex].x = withoutAggr; } } } else { for (let d3gIndex = 0; d3gIndex < d3g.length; d3gIndex++) { - colName = d3g[d3gIndex].key - withoutAggr = colName.substring(0, colName.lastIndexOf('(')) + colName = d3g[d3gIndex].key; + withoutAggr = colName.substring(0, colName.lastIndexOf('(')); if (namesWithoutAggr[withoutAggr] <= 1) { - d3g[d3gIndex].key = withoutAggr + d3g[d3gIndex].key = withoutAggr; } } // use group name instead of group.value as a column name, if there're only one group and one value selected. if (groups.length === 1 && values.length === 1) { for (let d3gIndex = 0; d3gIndex < d3g.length; d3gIndex++) { - colName = d3g[d3gIndex].key - colName = colName.split('.').slice(0, -1).join('.') - d3g[d3gIndex].key = colName + colName = d3g[d3gIndex].key; + colName = colName.split('.').slice(0, -1).join('.'); + d3g[d3gIndex].key = colName; } } } return { xLabels: rowIndexValue, - d3g: d3g - } + d3g: d3g, + }; } /** * method will be invoked when visualization need to be destroyed. * Don't need to destroy this.targetEl. */ - destroy () { + destroy() { if (this.chart) { - d3.selectAll('#' + this.targetEl[0].id + ' svg > *').remove() - this.chart = undefined + d3.selectAll('#' + this.targetEl[0].id + ' svg > *').remove(); + this.chart = undefined; } } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-piechart.js b/zeppelin-web/src/app/visualization/builtins/visualization-piechart.js index 4f80654db1d..84479cb600d 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-piechart.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-piechart.js @@ -12,29 +12,29 @@ * limitations under the License. */ -import Nvd3ChartVisualization from './visualization-nvd3chart' -import PivotTransformation from '../../tabledata/pivot' +import Nvd3ChartVisualization from './visualization-nvd3chart'; +import PivotTransformation from '../../tabledata/pivot'; /** * Visualize data in pie chart */ export default class PiechartVisualization extends Nvd3ChartVisualization { - constructor (targetEl, config) { - super(targetEl, config) - this.pivot = new PivotTransformation(config) + constructor(targetEl, config) { + super(targetEl, config); + this.pivot = new PivotTransformation(config); } - type () { - return 'pieChart' + type() { + return 'pieChart'; } - getTransformation () { - return this.pivot + getTransformation() { + return this.pivot; } - render (pivot) { + render(pivot) { // [ZEPPELIN-2253] New chart function will be created each time inside super.render() - this.chart = null + this.chart = null; const d3Data = this.d3DataFromPivot( pivot.schema, pivot.rows, @@ -43,41 +43,45 @@ export default class PiechartVisualization extends Nvd3ChartVisualization { pivot.values, true, false, - false) - const d = d3Data.d3g + false); + const d = d3Data.d3g; - let generateLabel + let generateLabel; // data is grouped if (pivot.groups && pivot.groups.length > 0) { - generateLabel = (suffix, prefix) => `${prefix}.${suffix}` + generateLabel = (suffix, prefix) => `${prefix}.${suffix}`; } else { // data isn't grouped - generateLabel = suffix => suffix + generateLabel = (suffix) => suffix; } - let d3g = d.map(group => { - return group.values.map(row => ({ + let d3g = d.map((group) => { + return group.values.map((row) => ({ label: generateLabel(row.x, group.key), - value: row.y - })) - }) + value: row.y, + })); + }); // the map function returns d3g as a nested array // [].concat flattens it, http://stackoverflow.com/a/10865042/5154397 - d3g = [].concat.apply([], d3g) // eslint-disable-line prefer-spread - super.render({d3g: d3g}) + d3g = [].concat.apply([], d3g); // eslint-disable-line prefer-spread + super.render({d3g: d3g}); } /** * Set new config */ - setConfig (config) { - super.setConfig(config) - this.pivot.setConfig(config) + setConfig(config) { + super.setConfig(config); + this.pivot.setConfig(config); } - configureChart (chart) { - chart.x(function (d) { return d.label }) - .y(function (d) { return d.value }) - .showLabels(false) - .showTooltipPercent(true) + configureChart(chart) { + chart.x(function(d) { + return d.label; + }) + .y(function(d) { + return d.value; + }) + .showLabels(false) + .showTooltipPercent(true); } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-scatterchart.js b/zeppelin-web/src/app/visualization/builtins/visualization-scatterchart.js index d7c00dbc32b..fad7500b962 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-scatterchart.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-scatterchart.js @@ -12,25 +12,25 @@ * limitations under the License. */ -import Nvd3ChartVisualization from './visualization-nvd3chart' -import ColumnselectorTransformation from '../../tabledata/columnselector' +import Nvd3ChartVisualization from './visualization-nvd3chart'; +import ColumnselectorTransformation from '../../tabledata/columnselector'; /** * Visualize data in scatter char */ export default class ScatterchartVisualization extends Nvd3ChartVisualization { - constructor (targetEl, config) { - super(targetEl, config) + constructor(targetEl, config) { + super(targetEl, config); this.columnselectorProps = [ { - name: 'xAxis' + name: 'xAxis', }, { - name: 'yAxis' + name: 'yAxis', }, { - name: 'group' + name: 'group', }, { name: 'size', @@ -39,322 +39,330 @@ export default class ScatterchartVisualization extends Nvd3ChartVisualization { 'number of values in corresponding coordinate' will be used. Zeppelin considers values as discrete when input values contain a string or the number of distinct values is greater than 5% of the total number of values. - This field turns grey when the selected option is invalid.` - } - ] - this.columnselector = new ColumnselectorTransformation(config, this.columnselectorProps) + This field turns grey when the selected option is invalid.`, + }, + ]; + this.columnselector = new ColumnselectorTransformation(config, this.columnselectorProps); } - type () { - return 'scatterChart' + type() { + return 'scatterChart'; } - getTransformation () { - return this.columnselector + getTransformation() { + return this.columnselector; } - render (tableData) { - this.tableData = tableData - this.selectDefault() - let d3Data = this.setScatterChart(tableData, true) - this.xLabels = d3Data.xLabels - this.yLabels = d3Data.yLabels + render(tableData) { + this.tableData = tableData; + this.selectDefault(); + let d3Data = this.setScatterChart(tableData, true); + this.xLabels = d3Data.xLabels; + this.yLabels = d3Data.yLabels; - super.render(d3Data) + super.render(d3Data); } - configureChart (chart) { - let self = this + configureChart(chart) { + let self = this; - chart.xAxis.tickFormat(function (d) { // TODO remove round after bump to nvd3 > 1.8.5 - return self.xAxisTickFormat(Math.round(d * 1e3) / 1e3, self.xLabels) - }) + chart.xAxis.tickFormat(function(d) { // TODO remove round after bump to nvd3 > 1.8.5 + return self.xAxisTickFormat(Math.round(d * 1e3) / 1e3, self.xLabels); + }); - chart.yAxis.tickFormat(function (d) { // TODO remove round after bump to nvd3 > 1.8.5 - return self.yAxisTickFormat(Math.round(d * 1e3) / 1e3, self.yLabels) - }) + chart.yAxis.tickFormat(function(d) { // TODO remove round after bump to nvd3 > 1.8.5 + return self.yAxisTickFormat(Math.round(d * 1e3) / 1e3, self.yLabels); + }); - chart.showDistX(true).showDistY(true) + chart.showDistX(true).showDistY(true); // handle the problem of tooltip not showing when muliple points have same value. } - yAxisTickFormat (d, yLabels) { + yAxisTickFormat(d, yLabels) { if (yLabels[d] && (isNaN(parseFloat(yLabels[d])) || !isFinite(yLabels[d]))) { // to handle string type xlabel - return yLabels[d] + return yLabels[d]; } else { - return super.yAxisTickFormat(d) + return super.yAxisTickFormat(d); } } - selectDefault () { + selectDefault() { if (!this.config.xAxis && !this.config.yAxis) { if (this.tableData.columns.length > 1) { - this.config.xAxis = this.tableData.columns[0] - this.config.yAxis = this.tableData.columns[1] + this.config.xAxis = this.tableData.columns[0]; + this.config.yAxis = this.tableData.columns[1]; } else if (this.tableData.columns.length === 1) { - this.config.xAxis = this.tableData.columns[0] + this.config.xAxis = this.tableData.columns[0]; } } } - setScatterChart (data, refresh) { - let xAxis = this.config.xAxis - let yAxis = this.config.yAxis - let group = this.config.group - let size = this.config.size - - let xValues = [] - let yValues = [] - let rows = {} - let d3g = [] - - let rowNameIndex = {} - let colNameIndex = {} - let grpNameIndex = {} - let rowIndexValue = {} - let colIndexValue = {} - let grpIndexValue = {} - let rowIdx = 0 - let colIdx = 0 - let grpIdx = 0 - let grpName = '' - - let xValue - let yValue - let row + setScatterChart(data, refresh) { + let xAxis = this.config.xAxis; + let yAxis = this.config.yAxis; + let group = this.config.group; + let size = this.config.size; + + let xValues = []; + let yValues = []; + let rows = {}; + let d3g = []; + + let rowNameIndex = {}; + let colNameIndex = {}; + let grpNameIndex = {}; + let rowIndexValue = {}; + let colIndexValue = {}; + let grpIndexValue = {}; + let rowIdx = 0; + let colIdx = 0; + let grpIdx = 0; + let grpName = ''; + + let xValue; + let yValue; + let row; if (!xAxis && !yAxis) { return { - d3g: [] - } + d3g: [], + }; } for (let i = 0; i < data.rows.length; i++) { - row = data.rows[i] + row = data.rows[i]; if (xAxis) { - xValue = row[xAxis.index] - xValues[i] = xValue + xValue = row[xAxis.index]; + xValues[i] = xValue; } if (yAxis) { - yValue = row[yAxis.index] - yValues[i] = yValue + yValue = row[yAxis.index]; + yValues[i] = yValue; } } let isAllDiscrete = ((xAxis && yAxis && this.isDiscrete(xValues) && this.isDiscrete(yValues)) || (!xAxis && this.isDiscrete(yValues)) || - (!yAxis && this.isDiscrete(xValues))) + (!yAxis && this.isDiscrete(xValues))); if (isAllDiscrete) { - rows = this.setDiscreteScatterData(data) + rows = this.setDiscreteScatterData(data); } else { - rows = data.rows + rows = data.rows; } if (!group && isAllDiscrete) { - grpName = 'count' + grpName = 'count'; } else if (!group && !size) { if (xAxis && yAxis) { - grpName = '(' + xAxis.name + ', ' + yAxis.name + ')' + grpName = '(' + xAxis.name + ', ' + yAxis.name + ')'; } else if (xAxis && !yAxis) { - grpName = xAxis.name + grpName = xAxis.name; } else if (!xAxis && yAxis) { - grpName = yAxis.name + grpName = yAxis.name; } } else if (!group && size) { - grpName = size.name + grpName = size.name; } - let epsilon = 1e-4 // TODO remove after bump to nvd3 > 1.8.5 + let epsilon = 1e-4; // TODO remove after bump to nvd3 > 1.8.5 for (let i = 0; i < rows.length; i++) { - row = rows[i] + row = rows[i]; if (xAxis) { - xValue = row[xAxis.index] + xValue = row[xAxis.index]; } if (yAxis) { - yValue = row[yAxis.index] + yValue = row[yAxis.index]; } if (group) { - grpName = row[group.index] + grpName = row[group.index]; } - let sz = (isAllDiscrete) ? row[row.length - 1] : ((size) ? row[size.index] : 1) + let sz = (isAllDiscrete) ? row[row.length - 1] : ((size) ? row[size.index] : 1); if (grpNameIndex[grpName] === undefined) { - grpIndexValue[grpIdx] = grpName - grpNameIndex[grpName] = grpIdx++ + grpIndexValue[grpIdx] = grpName; + grpNameIndex[grpName] = grpIdx++; } if (xAxis && rowNameIndex[xValue] === undefined) { - rowIndexValue[rowIdx] = xValue - rowNameIndex[xValue] = rowIdx++ + rowIndexValue[rowIdx] = xValue; + rowNameIndex[xValue] = rowIdx++; } if (yAxis && colNameIndex[yValue] === undefined) { - colIndexValue[colIdx] = yValue - colNameIndex[yValue] = colIdx++ + colIndexValue[colIdx] = yValue; + colNameIndex[yValue] = colIdx++; } if (!d3g[grpNameIndex[grpName]]) { d3g[grpNameIndex[grpName]] = { key: grpName, - values: [] - } + values: [], + }; } // TODO remove epsilon jitter after bump to nvd3 > 1.8.5 - let xval = 0 - let yval = 0 + let xval = 0; + let yval = 0; if (xAxis) { - xval = (isNaN(xValue) ? rowNameIndex[xValue] : parseFloat(xValue)) + Math.random() * epsilon + xval = (isNaN(xValue) ? rowNameIndex[xValue] : parseFloat(xValue)) + Math.random() * epsilon; } if (yAxis) { - yval = (isNaN(yValue) ? colNameIndex[yValue] : parseFloat(yValue)) + Math.random() * epsilon + yval = (isNaN(yValue) ? colNameIndex[yValue] : parseFloat(yValue)) + Math.random() * epsilon; } d3g[grpNameIndex[grpName]].values.push({ x: xval, y: yval, - size: isNaN(parseFloat(sz)) ? 1 : parseFloat(sz) - }) + size: isNaN(parseFloat(sz)) ? 1 : parseFloat(sz), + }); } // TODO remove sort and dedup after bump to nvd3 > 1.8.5 - let d3gvalues = d3g[grpNameIndex[grpName]].values - d3gvalues.sort(function (a, b) { - return ((a['x'] - b['x']) || (a['y'] - b['y'])) - }) + let d3gvalues = d3g[grpNameIndex[grpName]].values; + d3gvalues.sort(function(a, b) { + return ((a['x'] - b['x']) || (a['y'] - b['y'])); + }); for (let i = 0; i < d3gvalues.length - 1;) { if ((Math.abs(d3gvalues[i]['x'] - d3gvalues[i + 1]['x']) < epsilon) && (Math.abs(d3gvalues[i]['y'] - d3gvalues[i + 1]['y']) < epsilon)) { - d3gvalues.splice(i + 1, 1) + d3gvalues.splice(i + 1, 1); } else { - i++ + i++; } } return { xLabels: rowIndexValue, yLabels: colIndexValue, - d3g: d3g - } + d3g: d3g, + }; } - setDiscreteScatterData (data) { - let xAxis = this.config.xAxis - let yAxis = this.config.yAxis - let group = this.config.group + setDiscreteScatterData(data) { + let xAxis = this.config.xAxis; + let yAxis = this.config.yAxis; + let group = this.config.group; - let xValue - let yValue - let grp + let xValue; + let yValue; + let grp; - let rows = {} + let rows = {}; for (let i = 0; i < data.rows.length; i++) { - let row = data.rows[i] + let row = data.rows[i]; if (xAxis) { - xValue = row[xAxis.index] + xValue = row[xAxis.index]; } if (yAxis) { - yValue = row[yAxis.index] + yValue = row[yAxis.index]; } if (group) { - grp = row[group.index] + grp = row[group.index]; } - let key = xValue + ',' + yValue + ',' + grp + let key = xValue + ',' + yValue + ',' + grp; if (!rows[key]) { rows[key] = { x: xValue, y: yValue, group: grp, - size: 1 - } + size: 1, + }; } else { - rows[key].size++ + rows[key].size++; } } // change object into array - let newRows = [] + let newRows = []; for (let r in rows) { - let newRow = [] - if (xAxis) { newRow[xAxis.index] = rows[r].x } - if (yAxis) { newRow[yAxis.index] = rows[r].y } - if (group) { newRow[group.index] = rows[r].group } - newRow[data.rows[0].length] = rows[r].size - newRows.push(newRow) + if (rows.hasOwnProperty(r)) { + let newRow = []; + if (xAxis) { + newRow[xAxis.index] = rows[r].x; + } + if (yAxis) { + newRow[yAxis.index] = rows[r].y; + } + if (group) { + newRow[group.index] = rows[r].group; + } + newRow[data.rows[0].length] = rows[r].size; + newRows.push(newRow); + } } - return newRows + return newRows; } - isDiscrete (field) { - let getUnique = function (f) { - let uniqObj = {} - let uniqArr = [] - let j = 0 + isDiscrete(field) { + let getUnique = function(f) { + let uniqObj = {}; + let uniqArr = []; + let j = 0; for (let i = 0; i < f.length; i++) { - let item = f[i] + let item = f[i]; if (uniqObj[item] !== 1) { - uniqObj[item] = 1 - uniqArr[j++] = item + uniqObj[item] = 1; + uniqArr[j++] = item; } } - return uniqArr - } + return uniqArr; + }; for (let i = 0; i < field.length; i++) { if (isNaN(parseFloat(field[i])) && (typeof field[i] === 'string' || field[i] instanceof String)) { - return true + return true; } } - let threshold = 0.05 - let unique = getUnique(field) + let threshold = 0.05; + let unique = getUnique(field); if (unique.length / field.length < threshold) { - return true + return true; } else { - return false + return false; } } - isValidSizeOption (options) { - let xValues = [] - let yValues = [] - let rows = this.tableData.rows + isValidSizeOption(options) { + let xValues = []; + let yValues = []; + let rows = this.tableData.rows; for (let i = 0; i < rows.length; i++) { - let row = rows[i] - let size = row[options.size.index] + let row = rows[i]; + let size = row[options.size.index]; // check if the field is numeric if (isNaN(parseFloat(size)) || !isFinite(size)) { - return false + return false; } if (options.xAxis) { - let x = row[options.xAxis.index] - xValues[i] = x + let x = row[options.xAxis.index]; + xValues[i] = x; } if (options.yAxis) { - let y = row[options.yAxis.index] - yValues[i] = y + let y = row[options.yAxis.index]; + yValues[i] = y; } } // check if all existing fields are discrete let isAllDiscrete = ((options.xAxis && options.yAxis && this.isDiscrete(xValues) && this.isDiscrete(yValues)) || (!options.xAxis && this.isDiscrete(yValues)) || - (!options.yAxis && this.isDiscrete(xValues))) + (!options.yAxis && this.isDiscrete(xValues))); if (isAllDiscrete) { - return false + return false; } - return true + return true; } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-table.js b/zeppelin-web/src/app/visualization/builtins/visualization-table.js index afb5394610e..d77efbc805a 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-table.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-table.js @@ -12,8 +12,8 @@ * limitations under the License. */ -import Visualization from '../visualization' -import PassthroughTransformation from '../../tabledata/passthrough' +import Visualization from '../visualization'; +import PassthroughTransformation from '../../tabledata/passthrough'; import { Widget, ValueType, @@ -22,9 +22,9 @@ import { initializeTableConfig, resetTableOptionConfig, DefaultTableColumnType, TableColumnType, updateColumnTypeState, parseTableOption, -} from './visualization-util' +} from './visualization-util'; -const SETTING_TEMPLATE = require('./visualization-table-setting.html') +const SETTING_TEMPLATE = require('./visualization-table-setting.html'); const TABLE_OPTION_SPECS = [ { @@ -48,41 +48,43 @@ const TABLE_OPTION_SPECS = [ widget: Widget.CHECKBOX, description: 'Enable a footer for displaying aggregated values', }, -] +]; /** * Visualize data in table format */ export default class TableVisualization extends Visualization { - constructor (targetEl, config) { - super(targetEl, config) - this.passthrough = new PassthroughTransformation(config) - this.emitTimeout = null - this.isRestoring = false + constructor(targetEl, config) { + super(targetEl, config); + this.passthrough = new PassthroughTransformation(config); + this.emitTimeout = null; + this.isRestoring = false; - initializeTableConfig(config, TABLE_OPTION_SPECS) + initializeTableConfig(config, TABLE_OPTION_SPECS); } getColumnMinWidth(colName) { - let width = 150 // default - const calculatedWidth = colName.length * 10 + let width = 150; // default + const calculatedWidth = colName.length * 10; // use the broad one - if (calculatedWidth > width) { width = calculatedWidth } + if (calculatedWidth > width) { + width = calculatedWidth; + } - return width + return width; } createGridOptions(tableData, onRegisterApiCallback, config) { - const rows = tableData.rows - const columnNames = tableData.columns.map(c => c.name) + const rows = tableData.rows; + const columnNames = tableData.columns.map((c) => c.name); - const gridData = rows.map(r => { + const gridData = rows.map((r) => { return columnNames.reduce((acc, colName, index) => { - acc[colName] = r[index] - return acc - }, {}) - }) + acc[colName] = r[index]; + return acc; + }, {}); + }); const gridOptions = { data: gridData, @@ -94,7 +96,7 @@ export default class TableVisualization extends Visualization { fastWatch: false, treeRowHeaderAlwaysVisible: false, - columnDefs: columnNames.map(colName => { + columnDefs: columnNames.map((colName) => { return { displayName: colName, name: colName, @@ -111,7 +113,7 @@ export default class TableVisualization extends Visualization { `, minWidth: this.getColumnMinWidth(colName), width: '*', - } + }; }), rowEditWaitInterval: -1, /** disable saveRow event */ enableRowHashing: true, @@ -126,127 +128,131 @@ export default class TableVisualization extends Visualization { saveTreeView: true, saveFilter: true, saveSelection: false, - } + }; - return gridOptions + return gridOptions; } getGridElemId() { // angular doesn't allow `-` in scope variable name - const gridElemId = `${this.targetEl[0].id}_grid`.replace('-', '_') - return gridElemId + const gridElemId = `${this.targetEl[0].id}_grid`.replace('-', '_'); + return gridElemId; } getGridApiId() { // angular doesn't allow `-` in scope variable name - const gridApiId = `${this.targetEl[0].id}_gridApi`.replace('-', '_') - return gridApiId + const gridApiId = `${this.targetEl[0].id}_gridApi`.replace('-', '_'); + return gridApiId; } refresh() { - const gridElemId = this.getGridElemId() - const gridElem = angular.element(`#${gridElemId}`) + const gridElemId = this.getGridElemId(); + const gridElem = angular.element(`#${gridElemId}`); if (gridElem) { - gridElem.css('height', this.targetEl.height() - 10) + gridElem.css('height', this.targetEl.height() - 10); } } refreshGrid() { - const gridElemId = this.getGridElemId() - const gridElem = angular.element(`#${gridElemId}`) + const gridElemId = this.getGridElemId(); + const gridElem = angular.element(`#${gridElemId}`); if (gridElem) { - const scope = this.getScope() - const gridApiId = this.getGridApiId() - scope[gridApiId].core.notifyDataChange(this._uiGridConstants.dataChange.ALL) + const scope = this.getScope(); + const gridApiId = this.getGridApiId(); + scope[gridApiId].core.notifyDataChange(this._uiGridConstants.dataChange.ALL); } } updateColDefType(colDef, type) { - if (type === colDef.type) { return } + if (type === colDef.type) { + return; + } - colDef.type = type - const colName = colDef.name - const config = this.config + colDef.type = type; + const colName = colDef.name; + const config = this.config; if (config.tableColumnTypeState.names && config.tableColumnTypeState.names[colName]) { - config.tableColumnTypeState.names[colName] = type - this.persistConfigWithGridState(this.config) + config.tableColumnTypeState.names[colName] = type; + this.persistConfigWithGridState(this.config); } } addColumnMenus(gridOptions) { - if (!gridOptions || !gridOptions.columnDefs) { return } + if (!gridOptions || !gridOptions.columnDefs) { + return; + } - const self = this // for closure + const self = this; // for closure // SHOULD use `function() { ... }` syntax for each action to get `this` - gridOptions.columnDefs.map(colDef => { + gridOptions.columnDefs.map((colDef) => { colDef.menuItems = [ { title: 'Type: String', action: function() { - self.updateColDefType(this.context.col.colDef, TableColumnType.STRING) + self.updateColDefType(this.context.col.colDef, TableColumnType.STRING); }, active: function() { - return this.context.col.colDef.type === TableColumnType.STRING + return this.context.col.colDef.type === TableColumnType.STRING; }, }, { title: 'Type: Number', action: function() { - self.updateColDefType(this.context.col.colDef, TableColumnType.NUMBER) + self.updateColDefType(this.context.col.colDef, TableColumnType.NUMBER); }, active: function() { - return this.context.col.colDef.type === TableColumnType.NUMBER + return this.context.col.colDef.type === TableColumnType.NUMBER; }, }, { title: 'Type: Date', action: function() { - self.updateColDefType(this.context.col.colDef, TableColumnType.DATE) + self.updateColDefType(this.context.col.colDef, TableColumnType.DATE); }, active: function() { - return this.context.col.colDef.type === TableColumnType.DATE + return this.context.col.colDef.type === TableColumnType.DATE; }, }, - ] - }) + ]; + }); } setDynamicGridOptions(gridOptions, config) { // parse based on their type definitions - const parsed = parseTableOption(TABLE_OPTION_SPECS, config.tableOptionValue) + const parsed = parseTableOption(TABLE_OPTION_SPECS, config.tableOptionValue); - const { showAggregationFooter, useFilter, showPagination, } = parsed + const {showAggregationFooter, useFilter, showPagination} = parsed; - gridOptions.showGridFooter = false - gridOptions.showColumnFooter = showAggregationFooter - gridOptions.enableFiltering = useFilter + gridOptions.showGridFooter = false; + gridOptions.showColumnFooter = showAggregationFooter; + gridOptions.enableFiltering = useFilter; - gridOptions.enablePagination = showPagination - gridOptions.enablePaginationControls = showPagination + gridOptions.enablePagination = showPagination; + gridOptions.enablePaginationControls = showPagination; if (showPagination) { - gridOptions.paginationPageSize = 50 - gridOptions.paginationPageSizes = [25, 50, 100, 250, 1000] + gridOptions.paginationPageSize = 50; + gridOptions.paginationPageSizes = [25, 50, 100, 250, 1000]; } // selection can't be rendered dynamically in ui-grid 4.0.4 - gridOptions.enableRowSelection = false - gridOptions.enableRowHeaderSelection = false - gridOptions.enableFullRowSelection = false - gridOptions.enableSelectAll = false - gridOptions.enableGroupHeaderSelection = false - gridOptions.enableSelectionBatchEvent = false + gridOptions.enableRowSelection = false; + gridOptions.enableRowHeaderSelection = false; + gridOptions.enableFullRowSelection = false; + gridOptions.enableSelectAll = false; + gridOptions.enableGroupHeaderSelection = false; + gridOptions.enableSelectionBatchEvent = false; } - render (tableData) { - const gridElemId = this.getGridElemId() - let gridElem = document.getElementById(gridElemId) + render(tableData) { + const gridElemId = this.getGridElemId(); + let gridElem = document.getElementById(gridElemId); - const config = this.config - const self = this // for closure + const config = this.config; + const self = this; // for closure if (!gridElem) { // create, compile and append grid elem @@ -261,125 +267,147 @@ export default class TableVisualization extends Visualization { ui-grid-move-columns ui-grid-grouping ui-grid-save-state - ui-grid-exporter>`) + ui-grid-exporter>`); - gridElem.css('height', this.targetEl.height() - 10) - const scope = this.getScope() - gridElem = this._compile(gridElem)(scope) - this.targetEl.append(gridElem) + gridElem.css('height', this.targetEl.height() - 10); + const scope = this.getScope(); + gridElem = this._compile(gridElem)(scope); + this.targetEl.append(gridElem); // set gridOptions for this elem - const gridOptions = this.createGridOptions(tableData, onRegisterApiCallback, config) - this.setDynamicGridOptions(gridOptions, config) - this.addColumnMenus(gridOptions) - scope[gridElemId] = gridOptions + const gridOptions = this.createGridOptions(tableData, onRegisterApiCallback, config); + this.setDynamicGridOptions(gridOptions, config); + this.addColumnMenus(gridOptions); + scope[gridElemId] = gridOptions; // set gridApi for this elem - const gridApiId = this.getGridApiId() + const gridApiId = this.getGridApiId(); const onRegisterApiCallback = (gridApi) => { - scope[gridApiId] = gridApi + scope[gridApiId] = gridApi; // should restore state before registering APIs // register callbacks for change evens // should persist `self.config` instead `config` (closure issue) - gridApi.core.on.columnVisibilityChanged(scope, () => { self.persistConfigWithGridState(self.config) }) - gridApi.colMovable.on.columnPositionChanged(scope, () => { self.persistConfigWithGridState(self.config) }) - gridApi.core.on.sortChanged(scope, () => { self.persistConfigWithGridState(self.config) }) - gridApi.core.on.filterChanged(scope, () => { self.persistConfigWithGridState(self.config) }) - gridApi.grouping.on.aggregationChanged(scope, () => { self.persistConfigWithGridState(self.config) }) - gridApi.grouping.on.groupingChanged(scope, () => { self.persistConfigWithGridState(self.config) }) - gridApi.treeBase.on.rowCollapsed(scope, () => { self.persistConfigWithGridState(self.config) }) - gridApi.treeBase.on.rowExpanded(scope, () => { self.persistConfigWithGridState(self.config) }) - gridApi.colResizable.on.columnSizeChanged(scope, () => { self.persistConfigWithGridState(self.config) }) + gridApi.core.on.columnVisibilityChanged(scope, () => { + self.persistConfigWithGridState(self.config); + }); + gridApi.colMovable.on.columnPositionChanged(scope, () => { + self.persistConfigWithGridState(self.config); + }); + gridApi.core.on.sortChanged(scope, () => { + self.persistConfigWithGridState(self.config); + }); + gridApi.core.on.filterChanged(scope, () => { + self.persistConfigWithGridState(self.config); + }); + gridApi.grouping.on.aggregationChanged(scope, () => { + self.persistConfigWithGridState(self.config); + }); + gridApi.grouping.on.groupingChanged(scope, () => { + self.persistConfigWithGridState(self.config); + }); + gridApi.treeBase.on.rowCollapsed(scope, () => { + self.persistConfigWithGridState(self.config); + }); + gridApi.treeBase.on.rowExpanded(scope, () => { + self.persistConfigWithGridState(self.config); + }); + gridApi.colResizable.on.columnSizeChanged(scope, () => { + self.persistConfigWithGridState(self.config); + }); // pagination doesn't follow usual life-cycle in ui-grid v4.0.4 // gridApi.pagination.on.paginationChanged(scope, () => { self.persistConfigWithGridState(self.config) }) // TBD: do we need to propagate row selection? // gridApi.selection.on.rowSelectionChanged(scope, () => { self.persistConfigWithGridState(self.config) }) // gridApi.selection.on.rowSelectionChangedBatch(scope, () => { self.persistConfigWithGridState(self.config) }) - } - gridOptions.onRegisterApi = onRegisterApiCallback + }; + gridOptions.onRegisterApi = onRegisterApiCallback; } else { // don't need to update gridOptions.data since it's synchronized by paragraph execution - const gridOptions = this.getGridOptions() - this.setDynamicGridOptions(gridOptions, config) - this.refreshGrid() + const gridOptions = this.getGridOptions(); + this.setDynamicGridOptions(gridOptions, config); + this.refreshGrid(); } - const columnDefs = this.getGridOptions().columnDefs - updateColumnTypeState(tableData.columns, config, columnDefs) + const columnDefs = this.getGridOptions().columnDefs; + updateColumnTypeState(tableData.columns, config, columnDefs); // SHOULD restore grid state after columnDefs are updated - this.restoreGridState(config.tableGridState) + this.restoreGridState(config.tableGridState); } restoreGridState(gridState) { - if (!gridState) { return } + if (!gridState) { + return; + } // should set isRestoring to avoid that changed* events are triggered while restoring - this.isRestoring = true - const gridApi = this.getGridApi() + this.isRestoring = true; + const gridApi = this.getGridApi(); // restore grid state when gridApi is available if (!gridApi) { - setTimeout(() => this.restoreGridState(gridState), 100) + setTimeout(() => this.restoreGridState(gridState), 100); } else { - gridApi.saveState.restore(this.getScope(), gridState) - this.isRestoring = false + gridApi.saveState.restore(this.getScope(), gridState); + this.isRestoring = false; } } - destroy () { + destroy() { } - getTransformation () { - return this.passthrough + getTransformation() { + return this.passthrough; } getScope() { - const scope = this.targetEl.scope() - return scope + const scope = this.targetEl.scope(); + return scope; } getGridOptions() { - const scope = this.getScope() - const gridElemId = this.getGridElemId() - return scope[gridElemId] + const scope = this.getScope(); + const gridElemId = this.getGridElemId(); + return scope[gridElemId]; } getGridApi() { - const scope = this.targetEl.scope() - const gridApiId = this.getGridApiId() - return scope[gridApiId] + const scope = this.targetEl.scope(); + const gridApiId = this.getGridApiId(); + return scope[gridApiId]; } persistConfigImmediatelyWithGridState(config) { - this.persistConfigWithGridState(config) + this.persistConfigWithGridState(config); } persistConfigWithGridState(config) { - if (this.isRestoring) { return } + if (this.isRestoring) { + return; + } - const gridApi = this.getGridApi() - config.tableGridState = gridApi.saveState.save() - this.emitConfig(config) + const gridApi = this.getGridApi(); + config.tableGridState = gridApi.saveState.save(); + this.emitConfig(config); } persistConfig(config) { - this.emitConfig(config) + this.emitConfig(config); } - getSetting (chart) { - const self = this // for closure in scope - const configObj = self.config + getSetting(chart) { + const self = this; // for closure in scope + const configObj = self.config; // emit config if it's updated in `render` if (configObj.initialized) { - configObj.initialized = false - this.persistConfig(configObj) // should persist w/o state + configObj.initialized = false; + this.persistConfig(configObj); // should persist w/o state } else if (configObj.tableColumnTypeState && configObj.tableColumnTypeState.updated) { - configObj.tableColumnTypeState.updated = false - this.persistConfig(configObj) // should persist w/o state + configObj.tableColumnTypeState.updated = false; + this.persistConfig(configObj); // should persist w/o state } return { @@ -393,27 +421,27 @@ export default class TableVisualization extends Visualization { isTextareaWidget: isTextareaWidget, isBtnGroupWidget: isBtnGroupWidget, tableOptionValueChanged: () => { - self.persistConfigWithGridState(configObj) + self.persistConfigWithGridState(configObj); }, saveTableOption: () => { - self.persistConfigWithGridState(configObj) + self.persistConfigWithGridState(configObj); }, resetTableOption: () => { - resetTableOptionConfig(configObj) - initializeTableConfig(configObj, TABLE_OPTION_SPECS) - self.persistConfigWithGridState(configObj) + resetTableOptionConfig(configObj); + initializeTableConfig(configObj, TABLE_OPTION_SPECS); + self.persistConfigWithGridState(configObj); }, tableWidgetOnKeyDown: (event, optSpec) => { - const code = event.keyCode || event.which + const code = event.keyCode || event.which; if (code === 13 && isInputWidget(optSpec)) { - self.persistConfigWithGridState(configObj) + self.persistConfigWithGridState(configObj); } else if (code === 13 && event.shiftKey && isTextareaWidget(optSpec)) { - self.persistConfigWithGridState(configObj) + self.persistConfigWithGridState(configObj); } - event.stopPropagation() /** avoid to conflict with paragraph shortcuts */ - } - } - } + event.stopPropagation(); /** avoid to conflict with paragraph shortcuts */ + }, + }, + }; } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-util.js b/zeppelin-web/src/app/visualization/builtins/visualization-util.js index cd9cd48b754..a82a18ecceb 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-util.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-util.js @@ -18,7 +18,7 @@ export const Widget = { TEXTAREA: 'textarea', OPTION: 'option', BTN_GROUP: 'btn-group', -} +}; export const ValueType = { INT: 'int', @@ -26,7 +26,7 @@ export const ValueType = { BOOLEAN: 'boolean', STRING: 'string', JSON: 'JSON', -} +}; export const TableColumnType = { STRING: 'string', @@ -35,138 +35,170 @@ export const TableColumnType = { DATE: 'date', OBJECT: 'object', NUMBER_STR: 'numberStr', -} +}; -export const DefaultTableColumnType = TableColumnType.STRING +export const DefaultTableColumnType = TableColumnType.STRING; -export function isInputWidget (spec) { return spec.widget === Widget.INPUT } -export function isOptionWidget (spec) { return spec.widget === Widget.OPTION } -export function isCheckboxWidget (spec) { return spec.widget === Widget.CHECKBOX } -export function isTextareaWidget (spec) { return spec.widget === Widget.TEXTAREA } -export function isBtnGroupWidget (spec) { return spec.widget === Widget.BTN_GROUP } +export function isInputWidget(spec) { + return spec.widget === Widget.INPUT; +} +export function isOptionWidget(spec) { + return spec.widget === Widget.OPTION; +} +export function isCheckboxWidget(spec) { + return spec.widget === Widget.CHECKBOX; +} +export function isTextareaWidget(spec) { + return spec.widget === Widget.TEXTAREA; +} +export function isBtnGroupWidget(spec) { + return spec.widget === Widget.BTN_GROUP; +} export function resetTableOptionConfig(config) { - delete config.tableOptionSpecHash - config.tableOptionSpecHash = {} - delete config.tableOptionValue - config.tableOptionValue = {} - delete config.tableColumnTypeState.names - config.tableColumnTypeState.names = {} - config.updated = false - return config + delete config.tableOptionSpecHash; + config.tableOptionSpecHash = {}; + delete config.tableOptionValue; + config.tableOptionValue = {}; + delete config.tableColumnTypeState.names; + config.tableColumnTypeState.names = {}; + config.updated = false; + return config; } export function initializeTableConfig(config, tableOptionSpecs) { - if (typeof config.tableOptionValue === 'undefined') { config.tableOptionValue = {} } - if (typeof config.tableGridState === 'undefined') { config.tableGridState = {} } - if (typeof config.tableColumnTypeState === 'undefined') { config.tableColumnTypeState = {} } + if (typeof config.tableOptionValue === 'undefined') { + config.tableOptionValue = {}; + } + if (typeof config.tableGridState === 'undefined') { + config.tableGridState = {}; + } + if (typeof config.tableColumnTypeState === 'undefined') { + config.tableColumnTypeState = {}; + } // should remove `$$hashKey` using angular.toJson - const newSpecHash = JSON.stringify(JSON.parse(angular.toJson(tableOptionSpecs))) - const previousSpecHash = config.tableOptionSpecHash + const newSpecHash = JSON.stringify(JSON.parse(angular.toJson(tableOptionSpecs))); + const previousSpecHash = config.tableOptionSpecHash; // check whether spec is updated or not if (typeof previousSpecHash === 'undefined' || (previousSpecHash !== newSpecHash)) { - resetTableOptionConfig(config) + resetTableOptionConfig(config); - config.tableOptionSpecHash = newSpecHash - config.initialized = true + config.tableOptionSpecHash = newSpecHash; + config.initialized = true; // reset all persisted option values if spec is updated for (let i = 0; i < tableOptionSpecs.length; i++) { - const option = tableOptionSpecs[i] - config.tableOptionValue[option.name] = option.defaultValue + const option = tableOptionSpecs[i]; + config.tableOptionValue[option.name] = option.defaultValue; } } - return config + return config; } export function parseTableOption(specs, persistedTableOption) { /** copy original params */ - const parsed = JSON.parse(JSON.stringify(persistedTableOption)) + const parsed = JSON.parse(JSON.stringify(persistedTableOption)); for (let i = 0; i < specs.length; i++) { - const s = specs[i] - const name = s.name + const s = specs[i]; + const name = s.name; if (s.valueType === ValueType.INT && typeof parsed[name] !== 'number') { - try { parsed[name] = parseInt(parsed[name]) } catch (error) { parsed[name] = s.defaultValue } + try { + parsed[name] = parseInt(parsed[name]); + } catch (error) { + parsed[name] = s.defaultValue; + } } else if (s.valueType === ValueType.FLOAT && typeof parsed[name] !== 'number') { - try { parsed[name] = parseFloat(parsed[name]) } catch (error) { parsed[name] = s.defaultValue } + try { + parsed[name] = parseFloat(parsed[name]); + } catch (error) { + parsed[name] = s.defaultValue; + } } else if (s.valueType === ValueType.BOOLEAN) { if (parsed[name] === 'false') { - parsed[name] = false + parsed[name] = false; } else if (parsed[name] === 'true') { - parsed[name] = true + parsed[name] = true; } else if (typeof parsed[name] !== 'boolean') { - parsed[name] = s.defaultValue + parsed[name] = s.defaultValue; } } else if (s.valueType === ValueType.JSON) { if (parsed[name] !== null && typeof parsed[name] !== 'object') { - try { parsed[name] = JSON.parse(parsed[name]) } catch (error) { parsed[name] = s.defaultValue } + try { + parsed[name] = JSON.parse(parsed[name]); + } catch (error) { + parsed[name] = s.defaultValue; + } } else if (parsed[name] === null) { - parsed[name] = s.defaultValue + parsed[name] = s.defaultValue; } } } - return parsed + return parsed; } export function isColumnNameUpdated(prevColumnNames, newColumnNames) { - if (typeof prevColumnNames === 'undefined') { return true } + if (typeof prevColumnNames === 'undefined') { + return true; + } - let columnNameUpdated = false + let columnNameUpdated = false; for (let prevColName in prevColumnNames) { if (!newColumnNames[prevColName]) { - return true + return true; } } if (!columnNameUpdated) { for (let newColName in newColumnNames) { if (!prevColumnNames[newColName]) { - return true + return true; } } } - return false + return false; } export function updateColumnTypeState(columns, config, columnDefs) { - const columnTypeState = config.tableColumnTypeState + const columnTypeState = config.tableColumnTypeState; - if (!columnTypeState) { return } + if (!columnTypeState) { + return; + } // compare objects because order might be changed - const prevColumnNames = columnTypeState.names || {} + const prevColumnNames = columnTypeState.names || {}; const newColumnNames = columns.reduce((acc, c) => { - const prevColumnType = prevColumnNames[c.name] + const prevColumnType = prevColumnNames[c.name]; // use previous column type if exists if (prevColumnType) { - acc[c.name] = prevColumnType + acc[c.name] = prevColumnType; } else { - acc[c.name] = DefaultTableColumnType + acc[c.name] = DefaultTableColumnType; } - return acc - }, {}) + return acc; + }, {}); - let columnNameUpdated = isColumnNameUpdated(prevColumnNames, newColumnNames) + let columnNameUpdated = isColumnNameUpdated(prevColumnNames, newColumnNames); if (columnNameUpdated) { - columnTypeState.names = newColumnNames - columnTypeState.updated = true + columnTypeState.names = newColumnNames; + columnTypeState.updated = true; } // update `columnDefs[n].type` for (let i = 0; i < columnDefs.length; i++) { - const colName = columnDefs[i].name - columnDefs[i].type = columnTypeState.names[colName] + const colName = columnDefs[i].name; + columnDefs[i].type = columnTypeState.names[colName]; } } diff --git a/zeppelin-web/src/app/visualization/visualization.js b/zeppelin-web/src/app/visualization/visualization.js index 6b6e36aa387..f6475cbf523 100644 --- a/zeppelin-web/src/app/visualization/visualization.js +++ b/zeppelin-web/src/app/visualization/visualization.js @@ -16,12 +16,12 @@ * Base class for visualization. */ export default class Visualization { - constructor (targetEl, config) { - this.targetEl = targetEl - this.config = config - this._dirty = false - this._active = false - this._emitter = () => {} + constructor(targetEl, config) { + this.targetEl = targetEl; + this.config = config; + this._dirty = false; + this._active = false; + this._emitter = () => {}; } /** @@ -29,33 +29,33 @@ export default class Visualization { * @abstract * @return {Transformation} */ - getTransformation () { + getTransformation() { // override this - throw new TypeError('Visualization.getTransformation() should be overrided') + throw new TypeError('Visualization.getTransformation() should be overrided'); } /** * Method will be invoked when data or configuration changed. * @abstract */ - render (tableData) { + render(tableData) { // override this - throw new TypeError('Visualization.render() should be overrided') + throw new TypeError('Visualization.render() should be overrided'); } /** * Refresh visualization. */ - refresh () { + refresh() { // override this - console.warn('A chart is missing refresh function, it might not work preperly') + console.warn('A chart is missing refresh function, it might not work preperly'); } /** * Method will be invoked when visualization need to be destroyed. * Don't need to destroy this.targetEl. */ - destroy () { + destroy() { // override this } @@ -65,113 +65,117 @@ export default class Visualization { * scope : an object to bind to template scope * } */ - getSetting () { + getSetting() { // override this } /** * Activate. Invoked when visualization is selected. */ - activate () { + activate() { if (!this._active || this._dirty) { - this.refresh() - this._dirty = false + this.refresh(); + this._dirty = false; } - this._active = true + this._active = true; } /** * Deactivate. Invoked when visualization is de selected. */ - deactivate () { - this._active = false + deactivate() { + this._active = false; } /** * Is active. */ - isActive () { - return this._active + isActive() { + return this._active; } /** * When window or paragraph is resized. */ - resize () { + resize() { if (this.isActive()) { - this.refresh() + this.refresh(); } else { - this._dirty = true + this._dirty = true; } } /** * Set new config. */ - setConfig (config) { - this.config = config + setConfig(config) { + this.config = config; if (this.isActive()) { - this.refresh() + this.refresh(); } else { - this._dirty = true + this._dirty = true; } } /** * Emit config. config will sent to server and saved. */ - emitConfig (config) { - this._emitter(config) + emitConfig(config) { + this._emitter(config); } /** * Render setting. */ - renderSetting (targetEl) { - let setting = this.getSetting() + renderSetting(targetEl) { + let setting = this.getSetting(); if (!setting) { - return + return; } // already readered if (this._scope) { - let self = this - this._scope.$apply(function () { + let self = this; + this._scope.$apply(function() { for (let k in setting.scope) { - self._scope[k] = setting.scope[k] + if (setting.scope.hasOwnProperty(k)) { + self._scope[k] = setting.scope[k]; + } } for (let k in self._prevSettingScope) { if (!setting.scope[k]) { - self._scope[k] = setting.scope[k] + self._scope[k] = setting.scope[k]; } } - }) - return + }); + return; } else { - this._prevSettingScope = setting.scope + this._prevSettingScope = setting.scope; } - let scope = this._createNewScope() + let scope = this._createNewScope(); for (let k in setting.scope) { - scope[k] = setting.scope[k] + if (setting.scope.hasOwnProperty(k)) { + scope[k] = setting.scope[k]; + } } - let template = setting.template + let template = setting.template; if (template.split('\n').length === 1 && template.endsWith('.html')) { // template is url - this._templateRequest(template).then(t => + this._templateRequest(template).then((t) => _renderSetting(this, targetEl, t, scope) - ) + ); } else { - _renderSetting(this, targetEl, template, scope) + _renderSetting(this, targetEl, template, scope); } } } -function _renderSetting (instance, targetEl, template, scope) { - instance._targetEl = targetEl - targetEl.html(template) - instance._compile(targetEl.contents())(scope) - instance._scope = scope +function _renderSetting(instance, targetEl, template, scope) { + instance._targetEl = targetEl; + targetEl.html(template); + instance._compile(targetEl.contents())(scope); + instance._scope = scope; } diff --git a/zeppelin-web/src/components/array-ordering/array-ordering.service.js b/zeppelin-web/src/components/array-ordering/array-ordering.service.js index 6fa1ad9c28a..1f275e691b9 100644 --- a/zeppelin-web/src/components/array-ordering/array-ordering.service.js +++ b/zeppelin-web/src/components/array-ordering/array-ordering.service.js @@ -12,51 +12,51 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').service('arrayOrderingSrv', ArrayOrderingService) +angular.module('zeppelinWebApp').service('arrayOrderingSrv', ArrayOrderingService); function ArrayOrderingService(TRASH_FOLDER_ID) { - 'ngInject' + 'ngInject'; - let arrayOrderingSrv = this + let arrayOrderingSrv = this; - this.noteListOrdering = function (note) { + this.noteListOrdering = function(note) { if (note.id === TRASH_FOLDER_ID) { - return '\uFFFF' + return '\uFFFF'; } - return arrayOrderingSrv.getNoteName(note) - } + return arrayOrderingSrv.getNoteName(note); + }; - this.getNoteName = function (note) { + this.getNoteName = function(note) { if (note.name === undefined || note.name.trim() === '') { - return 'Note ' + note.id + return 'Note ' + note.id; } else { - return note.name + return note.name; } - } + }; - this.noteComparator = function (v1, v2) { - let note1 = v1.value || v1 - let note2 = v2.value || v2 + this.noteComparator = function(v1, v2) { + let note1 = v1.value || v1; + let note2 = v2.value || v2; if (note1.id === TRASH_FOLDER_ID) { - return 1 + return 1; } if (note2.id === TRASH_FOLDER_ID) { - return -1 + return -1; } if (note1.children === undefined && note2.children !== undefined) { - return 1 + return 1; } if (note1.children !== undefined && note2.children === undefined) { - return -1 + return -1; } - let noteName1 = arrayOrderingSrv.getNoteName(note1) - let noteName2 = arrayOrderingSrv.getNoteName(note2) + let noteName1 = arrayOrderingSrv.getNoteName(note1); + let noteName2 = arrayOrderingSrv.getNoteName(note2); - return noteName1.localeCompare(noteName2) - } + return noteName1.localeCompare(noteName2); + }; } diff --git a/zeppelin-web/src/components/base-url/base-url.service.js b/zeppelin-web/src/components/base-url/base-url.service.js index 6ef55b95006..845293c58a0 100644 --- a/zeppelin-web/src/components/base-url/base-url.service.js +++ b/zeppelin-web/src/components/base-url/base-url.service.js @@ -12,39 +12,39 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').service('baseUrlSrv', BaseUrlService) +angular.module('zeppelinWebApp').service('baseUrlSrv', BaseUrlService); function BaseUrlService() { - this.getPort = function () { - let port = Number(location.port) + this.getPort = function() { + let port = Number(location.port); if (!port) { - port = 80 + port = 80; if (location.protocol === 'https:') { - port = 443 + port = 443; } } // Exception for when running locally via grunt if (port === process.env.WEB_PORT) { - port = process.env.SERVER_PORT + port = process.env.SERVER_PORT; } - return port - } + return port; + }; - this.getWebsocketUrl = function () { - let wsProtocol = location.protocol === 'https:' ? 'wss:' : 'ws:' + this.getWebsocketUrl = function() { + let wsProtocol = location.protocol === 'https:' ? 'wss:' : 'ws:'; return wsProtocol + '//' + location.hostname + ':' + this.getPort() + - skipTrailingSlash(location.pathname) + '/ws' - } + skipTrailingSlash(location.pathname) + '/ws'; + }; this.getBase = function() { - return location.protocol + '//' + location.hostname + ':' + this.getPort() + location.pathname - } + return location.protocol + '//' + location.hostname + ':' + this.getPort() + location.pathname; + }; - this.getRestApiBase = function () { - return skipTrailingSlash(this.getBase()) + '/api' - } + this.getRestApiBase = function() { + return skipTrailingSlash(this.getBase()) + '/api'; + }; - const skipTrailingSlash = function (path) { - return path.replace(/\/$/, '') - } + const skipTrailingSlash = function(path) { + return path.replace(/\/$/, ''); + }; } diff --git a/zeppelin-web/src/components/login/login.controller.js b/zeppelin-web/src/components/login/login.controller.js index 919095067fd..9a42d5f62bb 100644 --- a/zeppelin-web/src/components/login/login.controller.js +++ b/zeppelin-web/src/components/login/login.controller.js @@ -12,73 +12,73 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').controller('LoginCtrl', LoginCtrl) +angular.module('zeppelinWebApp').controller('LoginCtrl', LoginCtrl); -function LoginCtrl ($scope, $rootScope, $http, $httpParamSerializer, baseUrlSrv, $location, $timeout) { - 'ngInject' +function LoginCtrl($scope, $rootScope, $http, $httpParamSerializer, baseUrlSrv, $location, $timeout) { + 'ngInject'; - $scope.SigningIn = false - $scope.loginParams = {} - $scope.login = function () { - $scope.SigningIn = true + $scope.SigningIn = false; + $scope.loginParams = {}; + $scope.login = function() { + $scope.SigningIn = true; $http({ method: 'POST', url: baseUrlSrv.getRestApiBase() + '/login', headers: { - 'Content-Type': 'application/x-www-form-urlencoded' + 'Content-Type': 'application/x-www-form-urlencoded', }, data: $httpParamSerializer({ 'userName': $scope.loginParams.userName, - 'password': $scope.loginParams.password - }) - }).then(function successCallback (response) { - $rootScope.ticket = response.data.body - angular.element('#loginModal').modal('toggle') - $rootScope.$broadcast('loginSuccess', true) - $rootScope.userName = $scope.loginParams.userName - $scope.SigningIn = false + 'password': $scope.loginParams.password, + }), + }).then(function successCallback(response) { + $rootScope.ticket = response.data.body; + angular.element('#loginModal').modal('toggle'); + $rootScope.$broadcast('loginSuccess', true); + $rootScope.userName = $scope.loginParams.userName; + $scope.SigningIn = false; // redirect to the page from where the user originally was if ($location.search() && $location.search()['ref']) { - $timeout(function () { - let redirectLocation = $location.search()['ref'] - $location.$$search = {} - $location.path(redirectLocation) - }, 100) + $timeout(function() { + let redirectLocation = $location.search()['ref']; + $location.$$search = {}; + $location.path(redirectLocation); + }, 100); } - }, function errorCallback (errorResponse) { - $scope.loginParams.errorText = 'The username and password that you entered don\'t match.' - $scope.SigningIn = false - }) - } + }, function errorCallback(errorResponse) { + $scope.loginParams.errorText = 'The username and password that you entered don\'t match.'; + $scope.SigningIn = false; + }); + }; - let initValues = function () { + let initValues = function() { $scope.loginParams = { userName: '', - password: '' - } - } + password: '', + }; + }; // handle session logout message received from WebSocket - $rootScope.$on('session_logout', function (event, data) { + $rootScope.$on('session_logout', function(event, data) { if ($rootScope.userName !== '') { - $rootScope.userName = '' - $rootScope.ticket = undefined + $rootScope.userName = ''; + $rootScope.ticket = undefined; - setTimeout(function () { - $scope.loginParams = {} - $scope.loginParams.errorText = data.info - angular.element('.nav-login-btn').click() - }, 1000) - let locationPath = $location.path() - $location.path('/').search('ref', locationPath) + setTimeout(function() { + $scope.loginParams = {}; + $scope.loginParams.errorText = data.info; + angular.element('.nav-login-btn').click(); + }, 1000); + let locationPath = $location.path(); + $location.path('/').search('ref', locationPath); } - }) + }); /* ** $scope.$on functions below */ - $scope.$on('initLoginValues', function () { - initValues() - }) + $scope.$on('initLoginValues', function() { + initValues(); + }); } diff --git a/zeppelin-web/src/components/navbar/expand-collapse/expand-collapse.directive.js b/zeppelin-web/src/components/navbar/expand-collapse/expand-collapse.directive.js index e4280e865a8..58629afdec0 100644 --- a/zeppelin-web/src/components/navbar/expand-collapse/expand-collapse.directive.js +++ b/zeppelin-web/src/components/navbar/expand-collapse/expand-collapse.directive.js @@ -12,37 +12,37 @@ * limitations under the License. */ -import './expand-collapse.css' +import './expand-collapse.css'; -angular.module('zeppelinWebApp').directive('expandCollapse', expandCollapseDirective) +angular.module('zeppelinWebApp').directive('expandCollapse', expandCollapseDirective); function expandCollapseDirective() { return { restrict: 'EA', - link: function (scope, element, attrs) { - angular.element(element).click(function (event) { + link: function(scope, element, attrs) { + angular.element(element).click(function(event) { if (angular.element(element).next('.expandable:visible').length > 1) { - angular.element(element).next('.expandable:visible').slideUp('slow') - angular.element(element).find('i.fa-folder-open').toggleClass('fa-folder fa-folder-open') + angular.element(element).next('.expandable:visible').slideUp('slow'); + angular.element(element).find('i.fa-folder-open').toggleClass('fa-folder fa-folder-open'); } else { - angular.element(element).next('.expandable').first().slideToggle('200', function () { + angular.element(element).next('.expandable').first().slideToggle('200', function() { // do not toggle trash folder if (angular.element(element).find('.fa-trash-o').length === 0) { - angular.element(element).find('i').first().toggleClass('fa-folder fa-folder-open') + angular.element(element).find('i').first().toggleClass('fa-folder fa-folder-open'); } - }) + }); } - let target = event.target + let target = event.target; // add note if (target.classList !== undefined && target.classList.contains('fa-plus') && target.tagName.toLowerCase() === 'i') { - return + return; } - event.stopPropagation() - }) - } - } + event.stopPropagation(); + }); + }, + }; } diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 139328e11d7..9fb5db2aeec 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -12,237 +12,237 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').controller('NavCtrl', NavCtrl) +angular.module('zeppelinWebApp').controller('NavCtrl', NavCtrl); -function NavCtrl ($scope, $rootScope, $http, $routeParams, $location, +function NavCtrl($scope, $rootScope, $http, $routeParams, $location, noteListFactory, baseUrlSrv, websocketMsgSrv, arrayOrderingSrv, searchService, TRASH_FOLDER_ID) { - 'ngInject' - - let vm = this - vm.arrayOrderingSrv = arrayOrderingSrv - vm.connected = websocketMsgSrv.isConnected() - vm.isActive = isActive - vm.logout = logout - vm.notes = noteListFactory - vm.search = search - vm.searchForm = searchService - vm.showLoginWindow = showLoginWindow - vm.TRASH_FOLDER_ID = TRASH_FOLDER_ID - vm.isFilterNote = isFilterNote - vm.numberOfNotesDisplayed = 10 - - $scope.query = {q: ''} - - initController() - - function getZeppelinVersion () { + 'ngInject'; + + let vm = this; + vm.arrayOrderingSrv = arrayOrderingSrv; + vm.connected = websocketMsgSrv.isConnected(); + vm.isActive = isActive; + vm.logout = logout; + vm.notes = noteListFactory; + vm.search = search; + vm.searchForm = searchService; + vm.showLoginWindow = showLoginWindow; + vm.TRASH_FOLDER_ID = TRASH_FOLDER_ID; + vm.isFilterNote = isFilterNote; + vm.numberOfNotesDisplayed = 10; + + $scope.query = {q: ''}; + + initController(); + + function getZeppelinVersion() { $http.get(baseUrlSrv.getRestApiBase() + '/version').success( - function (data, status, headers, config) { - $rootScope.zeppelinVersion = data.body.version + function(data, status, headers, config) { + $rootScope.zeppelinVersion = data.body.version; }).error( - function (data, status, headers, config) { - console.log('Error %o %o', status, data.message) - }) + function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); } - function initController () { - $scope.isDrawNavbarNoteList = false - angular.element('#notebook-list').perfectScrollbar({suppressScrollX: true}) + function initController() { + $scope.isDrawNavbarNoteList = false; + angular.element('#notebook-list').perfectScrollbar({suppressScrollX: true}); - angular.element(document).click(function () { - $scope.query.q = '' - }) + angular.element(document).click(function() { + $scope.query.q = ''; + }); - getZeppelinVersion() - loadNotes() + getZeppelinVersion(); + loadNotes(); } - function isFilterNote (note) { + function isFilterNote(note) { if (!$scope.query.q) { - return true + return true; } - let noteName = note.name + let noteName = note.name; if (noteName.toLowerCase().indexOf($scope.query.q.toLowerCase()) > -1) { - return true + return true; } - return false + return false; } - function isActive (noteId) { - return ($routeParams.noteId === noteId) + function isActive(noteId) { + return ($routeParams.noteId === noteId); } - function listConfigurations () { - websocketMsgSrv.listConfigurations() + function listConfigurations() { + websocketMsgSrv.listConfigurations(); } - function loadNotes () { - websocketMsgSrv.getNoteList() + function loadNotes() { + websocketMsgSrv.getNoteList(); } - function getHomeNote () { - websocketMsgSrv.getHomeNote() + function getHomeNote() { + websocketMsgSrv.getHomeNote(); } function logout() { - let logoutURL = baseUrlSrv.getRestApiBase() + '/login/logout' + let logoutURL = baseUrlSrv.getRestApiBase() + '/login/logout'; - $http.post(logoutURL).then(function () {}, function (response) { + $http.post(logoutURL).then(function() {}, function(response) { if (response.data) { - let res = angular.fromJson(response.data).body + let res = angular.fromJson(response.data).body; if (res['redirectURL']) { if (res['isLogoutAPI'] === 'true') { - $http.get(res['redirectURL']).then(function () { - }, function () { - window.location = baseUrlSrv.getBase() - }) + $http.get(res['redirectURL']).then(function() { + }, function() { + window.location = baseUrlSrv.getBase(); + }); } else { - window.location.href = res['redirectURL'] + window.location.href + window.location.href = res['redirectURL'] + window.location.href; } - return undefined + return undefined; } } // force authcBasic (if configured) to logout if (detectIE()) { - let outcome + let outcome; try { - outcome = document.execCommand('ClearAuthenticationCache') + outcome = document.execCommand('ClearAuthenticationCache'); } catch (e) { - console.log(e) + console.log(e); } if (!outcome) { // Let's create an xmlhttp object - outcome = (function (x) { + outcome = (function(x) { if (x) { // the reason we use "random" value for password is // that browsers cache requests. changing // password effectively behaves like cache-busing. x.open('HEAD', location.href, true, 'logout', - (new Date()).getTime().toString()) - x.send('') + (new Date()).getTime().toString()); + x.send(''); // x.abort() - return 1 // this is **speculative** "We are done." + return 1; // this is **speculative** "We are done." } else { // eslint-disable-next-line no-useless-return - return + return; } })(window.XMLHttpRequest ? new window.XMLHttpRequest() // eslint-disable-next-line no-undef - : (window.ActiveXObject ? new ActiveXObject('Microsoft.XMLHTTP') : u)) + : (window.ActiveXObject ? new ActiveXObject('Microsoft.XMLHTTP') : u)); } if (!outcome) { let m = 'Your browser is too old or too weird to support log out functionality. Close all windows and ' + - 'restart the browser.' - alert(m) + 'restart the browser.'; + alert(m); } } else { // for firefox and safari - logoutURL = logoutURL.replace('//', '//false:false@') + logoutURL = logoutURL.replace('//', '//false:false@'); } - $http.post(logoutURL).error(function () { - $rootScope.userName = '' - $rootScope.ticket.principal = '' - $rootScope.ticket.screenUsername = '' - $rootScope.ticket.ticket = '' - $rootScope.ticket.roles = '' + $http.post(logoutURL).error(function() { + $rootScope.userName = ''; + $rootScope.ticket.principal = ''; + $rootScope.ticket.screenUsername = ''; + $rootScope.ticket.ticket = ''; + $rootScope.ticket.roles = ''; BootstrapDialog.show({ - message: 'Logout Success' - }) - setTimeout(function () { - window.location = baseUrlSrv.getBase() - }, 1000) - }) - }) + message: 'Logout Success', + }); + setTimeout(function() { + window.location = baseUrlSrv.getBase(); + }, 1000); + }); + }); } function detectIE() { - let ua = window.navigator.userAgent + let ua = window.navigator.userAgent; - let msie = ua.indexOf('MSIE ') + let msie = ua.indexOf('MSIE '); if (msie > 0) { // IE 10 or older => return version number - return parseInt(ua.substring(msie + 5, ua.indexOf('.', msie)), 10) + return parseInt(ua.substring(msie + 5, ua.indexOf('.', msie)), 10); } - let trident = ua.indexOf('Trident/') + let trident = ua.indexOf('Trident/'); if (trident > 0) { // IE 11 => return version number - let rv = ua.indexOf('rv:') - return parseInt(ua.substring(rv + 3, ua.indexOf('.', rv)), 10) + let rv = ua.indexOf('rv:'); + return parseInt(ua.substring(rv + 3, ua.indexOf('.', rv)), 10); } - let edge = ua.indexOf('Edge/') + let edge = ua.indexOf('Edge/'); if (edge > 0) { // Edge (IE 12+) => return version number - return parseInt(ua.substring(edge + 5, ua.indexOf('.', edge)), 10) + return parseInt(ua.substring(edge + 5, ua.indexOf('.', edge)), 10); } // other browser - return false + return false; } - function search (searchTerm) { - $location.path('/search/' + searchTerm) + function search(searchTerm) { + $location.path('/search/' + searchTerm); } - function showLoginWindow () { - setTimeout(function () { - angular.element('#userName').focus() - }, 500) + function showLoginWindow() { + setTimeout(function() { + angular.element('#userName').focus(); + }, 500); } /* ** $scope.$on functions below */ - $scope.$on('setNoteMenu', function (event, notes) { - noteListFactory.setNotes(notes) - initNotebookListEventListener() - }) + $scope.$on('setNoteMenu', function(event, notes) { + noteListFactory.setNotes(notes); + initNotebookListEventListener(); + }); - $scope.$on('setConnectedStatus', function (event, param) { - vm.connected = param - }) + $scope.$on('setConnectedStatus', function(event, param) { + vm.connected = param; + }); - $scope.$on('loginSuccess', function (event, param) { - $rootScope.ticket.screenUsername = $rootScope.ticket.principal - listConfigurations() - loadNotes() - getHomeNote() - }) + $scope.$on('loginSuccess', function(event, param) { + $rootScope.ticket.screenUsername = $rootScope.ticket.principal; + listConfigurations(); + loadNotes(); + getHomeNote(); + }); /* ** Performance optimization for Browser Render. */ - function initNotebookListEventListener () { - angular.element(document).ready(function () { - angular.element('.notebook-list-dropdown').on('show.bs.dropdown', function () { - $scope.isDrawNavbarNoteList = true - }) - - angular.element('.notebook-list-dropdown').on('hide.bs.dropdown', function () { - $scope.isDrawNavbarNoteList = false - }) - }) + function initNotebookListEventListener() { + angular.element(document).ready(function() { + angular.element('.notebook-list-dropdown').on('show.bs.dropdown', function() { + $scope.isDrawNavbarNoteList = true; + }); + + angular.element('.notebook-list-dropdown').on('hide.bs.dropdown', function() { + $scope.isDrawNavbarNoteList = false; + }); + }); } - $scope.loadMoreNotes = function () { - vm.numberOfNotesDisplayed += 10 - } + $scope.loadMoreNotes = function() { + vm.numberOfNotesDisplayed += 10; + }; - $scope.calculateTooltipPlacement = function (note) { + $scope.calculateTooltipPlacement = function(note) { if (note !== undefined && note.name !== undefined) { - let length = note.name.length + let length = note.name.length; if (length < 2) { - return 'top-left' + return 'top-left'; } else if (length > 7) { - return 'top-right' + return 'top-right'; } } - return 'top' - } + return 'top'; + }; } diff --git a/zeppelin-web/src/components/navbar/navbar.controller.test.js b/zeppelin-web/src/components/navbar/navbar.controller.test.js index bf29b842bd3..f4bb3bf05e4 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.test.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.test.js @@ -1,18 +1,18 @@ -describe('Controller: NavCtrl', function () { +describe('Controller: NavCtrl', function() { // load the controller's module - beforeEach(angular.mock.module('zeppelinWebApp')) - let NavCtrl - let scope + beforeEach(angular.mock.module('zeppelinWebApp')); + let NavCtrl; + let scope; // Initialize the controller and a mock scope - beforeEach(inject(function ($controller, $rootScope) { - scope = $rootScope.$new() + beforeEach(inject(function($controller, $rootScope) { + scope = $rootScope.$new(); NavCtrl = $controller('NavCtrl', { - $scope: scope - }) + $scope: scope, + }); - it('NavCtrl to toBeDefined', function () { - expect(NavCtrl).toBeDefined() - expect(NavCtrl.loadNotes).toBeDefined() - }) - })) -}) + it('NavCtrl to toBeDefined', function() { + expect(NavCtrl).toBeDefined(); + expect(NavCtrl.loadNotes).toBeDefined(); + }); + })); +}); diff --git a/zeppelin-web/src/components/ng-enter/ng-enter.directive.js b/zeppelin-web/src/components/ng-enter/ng-enter.directive.js index 98bc067ce15..a4d9219cd46 100644 --- a/zeppelin-web/src/components/ng-enter/ng-enter.directive.js +++ b/zeppelin-web/src/components/ng-enter/ng-enter.directive.js @@ -12,19 +12,19 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').directive('ngEnter', NgEnterDirective) +angular.module('zeppelinWebApp').directive('ngEnter', NgEnterDirective); function NgEnterDirective() { - return function (scope, element, attrs) { - element.bind('keydown keypress', function (event) { + return function(scope, element, attrs) { + element.bind('keydown keypress', function(event) { if (event.which === 13) { if (!event.shiftKey) { - scope.$apply(function () { - scope.$eval(attrs.ngEnter) - }) + scope.$apply(function() { + scope.$eval(attrs.ngEnter); + }); } - event.preventDefault() + event.preventDefault(); } - }) - } + }); + }; } diff --git a/zeppelin-web/src/components/ng-enter/ng-enter.directive.test.js b/zeppelin-web/src/components/ng-enter/ng-enter.directive.test.js index 49f97cca19a..6285b59a499 100644 --- a/zeppelin-web/src/components/ng-enter/ng-enter.directive.test.js +++ b/zeppelin-web/src/components/ng-enter/ng-enter.directive.test.js @@ -1,19 +1,19 @@ -describe('Directive: ngEnter', function () { +describe('Directive: ngEnter', function() { // load the directive's module - beforeEach(angular.mock.module('zeppelinWebApp')) + beforeEach(angular.mock.module('zeppelinWebApp')); - let element - let scope + let element; + let scope; - beforeEach(inject(function ($rootScope) { - scope = $rootScope.$new() - })) + beforeEach(inject(function($rootScope) { + scope = $rootScope.$new(); + })); - it('should be define', inject(function ($compile) { - element = angular.element('') - element = $compile(element)(scope) - expect(element.text()).toBeDefined() - })) + it('should be define', inject(function($compile) { + element = angular.element(''); + element = $compile(element)(scope); + expect(element.text()).toBeDefined(); + })); // Test the rest of function in ngEnter /* it('should make hidden element visible', inject(function ($compile) { @@ -21,4 +21,4 @@ describe('Directive: ngEnter', function () { element = $compile(element)(scope); expect(element.text()).toBe('this is the ngEnter directive'); })); */ -}) +}); diff --git a/zeppelin-web/src/components/ng-escape/ng-escape.directive.js b/zeppelin-web/src/components/ng-escape/ng-escape.directive.js index a3d35ea33f3..bdb76367952 100644 --- a/zeppelin-web/src/components/ng-escape/ng-escape.directive.js +++ b/zeppelin-web/src/components/ng-escape/ng-escape.directive.js @@ -12,17 +12,17 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').directive('ngEscape', NgEscapeDirective) +angular.module('zeppelinWebApp').directive('ngEscape', NgEscapeDirective); function NgEscapeDirective() { - return function (scope, element, attrs) { - element.bind('keydown keyup', function (event) { + return function(scope, element, attrs) { + element.bind('keydown keyup', function(event) { if (event.which === 27) { - scope.$apply(function () { - scope.$eval(attrs.ngEscape) - }) - event.preventDefault() + scope.$apply(function() { + scope.$eval(attrs.ngEscape); + }); + event.preventDefault(); } - }) - } + }); + }; } diff --git a/zeppelin-web/src/components/note-action/note-action.service.js b/zeppelin-web/src/components/note-action/note-action.service.js index 8e00c0fc447..d4bf6f01f93 100644 --- a/zeppelin-web/src/components/note-action/note-action.service.js +++ b/zeppelin-web/src/components/note-action/note-action.service.js @@ -12,172 +12,172 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').service('noteActionService', noteActionService) +angular.module('zeppelinWebApp').service('noteActionService', noteActionService); function noteActionService(websocketMsgSrv, $location, noteRenameService, noteListFactory) { - 'ngInject' + 'ngInject'; - this.moveNoteToTrash = function (noteId, redirectToHome) { + this.moveNoteToTrash = function(noteId, redirectToHome) { BootstrapDialog.confirm({ closable: true, title: 'Move this note to trash?', message: 'This note will be moved to trash.', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.moveNoteToTrash(noteId) + websocketMsgSrv.moveNoteToTrash(noteId); if (redirectToHome) { - $location.path('/') + $location.path('/'); } } - } - }) - } + }, + }); + }; - this.moveFolderToTrash = function (folderId) { + this.moveFolderToTrash = function(folderId) { BootstrapDialog.confirm({ closable: true, title: 'Move this folder to trash?', message: 'This folder will be moved to trash.', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.moveFolderToTrash(folderId) + websocketMsgSrv.moveFolderToTrash(folderId); } - } - }) - } + }, + }); + }; - this.removeNote = function (noteId, redirectToHome) { + this.removeNote = function(noteId, redirectToHome) { BootstrapDialog.confirm({ type: BootstrapDialog.TYPE_WARNING, closable: true, title: 'WARNING! This note will be removed permanently', message: 'This cannot be undone. Are you sure?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.deleteNote(noteId) + websocketMsgSrv.deleteNote(noteId); if (redirectToHome) { - $location.path('/') + $location.path('/'); } } - } - }) - } + }, + }); + }; - this.removeFolder = function (folderId) { + this.removeFolder = function(folderId) { BootstrapDialog.confirm({ type: BootstrapDialog.TYPE_WARNING, closable: true, title: 'WARNING! This folder will be removed permanently', message: 'This cannot be undone. Are you sure?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.removeFolder(folderId) + websocketMsgSrv.removeFolder(folderId); } - } - }) - } + }, + }); + }; - this.restoreAll = function () { + this.restoreAll = function() { BootstrapDialog.confirm({ closable: true, title: 'Are you sure want to restore all notes in the trash?', message: 'Folders and notes in the trash will be ' + 'merged into their original position.', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.restoreAll() + websocketMsgSrv.restoreAll(); } - } - }) - } + }, + }); + }; - this.emptyTrash = function () { + this.emptyTrash = function() { BootstrapDialog.confirm({ type: BootstrapDialog.TYPE_WARNING, closable: true, title: 'WARNING! Notes under trash will be removed permanently', message: 'This cannot be undone. Are you sure?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.emptyTrash() + websocketMsgSrv.emptyTrash(); } - } - }) - } + }, + }); + }; - this.clearAllParagraphOutput = function (noteId) { + this.clearAllParagraphOutput = function(noteId) { BootstrapDialog.confirm({ closable: true, title: '', message: 'Do you want to clear all output?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.clearAllParagraphOutput(noteId) + websocketMsgSrv.clearAllParagraphOutput(noteId); } - } - }) - } + }, + }); + }; - this.renameNote = function (noteId, notePath) { + this.renameNote = function(noteId, notePath) { noteRenameService.openRenameModal({ title: 'Rename note', oldName: notePath, - callback: function (newName) { - websocketMsgSrv.renameNote(noteId, newName) - } - }) - } + callback: function(newName) { + websocketMsgSrv.renameNote(noteId, newName); + }, + }); + }; - this.renameFolder = function (folderId) { + this.renameFolder = function(folderId) { noteRenameService.openRenameModal({ title: 'Rename folder', oldName: folderId, - callback: function (newName) { - let newFolderId = normalizeFolderId(newName) + callback: function(newName) { + let newFolderId = normalizeFolderId(newName); if (_.has(noteListFactory.flatFolderMap, newFolderId)) { BootstrapDialog.confirm({ type: BootstrapDialog.TYPE_WARNING, closable: true, title: 'WARNING! The folder will be MERGED', message: 'The folder will be merged into ' + newFolderId + '. Are you sure?', - callback: function (result) { + callback: function(result) { if (result) { - websocketMsgSrv.renameFolder(folderId, newFolderId) + websocketMsgSrv.renameFolder(folderId, newFolderId); } - } - }) + }, + }); } else { - websocketMsgSrv.renameFolder(folderId, newFolderId) + websocketMsgSrv.renameFolder(folderId, newFolderId); } - } - }) - } + }, + }); + }; - function normalizeFolderId (folderId) { - folderId = folderId.trim() + function normalizeFolderId(folderId) { + folderId = folderId.trim(); while (folderId.indexOf('\\') > -1) { - folderId = folderId.replace('\\', '/') + folderId = folderId.replace('\\', '/'); } while (folderId.indexOf('///') > -1) { - folderId = folderId.replace('///', '/') + folderId = folderId.replace('///', '/'); } - folderId = folderId.replace('//', '/') + folderId = folderId.replace('//', '/'); if (folderId === '/') { - return '/' + return '/'; } if (folderId[0] === '/') { - folderId = folderId.substring(1) + folderId = folderId.substring(1); } if (folderId.slice(-1) === '/') { - folderId = folderId.slice(0, -1) + folderId = folderId.slice(0, -1); } - return folderId + return folderId; } } diff --git a/zeppelin-web/src/components/note-create/note-create.controller.js b/zeppelin-web/src/components/note-create/note-create.controller.js index c999c20271c..a2eb5a6f6d3 100644 --- a/zeppelin-web/src/components/note-create/note-create.controller.js +++ b/zeppelin-web/src/components/note-create/note-create.controller.js @@ -12,95 +12,95 @@ * limitations under the License. */ -import './note-create.css' +import './note-create.css'; -angular.module('zeppelinWebApp').controller('NoteCreateCtrl', NoteCreateCtrl) +angular.module('zeppelinWebApp').controller('NoteCreateCtrl', NoteCreateCtrl); -function NoteCreateCtrl ($scope, noteListFactory, $routeParams, websocketMsgSrv) { - 'ngInject' +function NoteCreateCtrl($scope, noteListFactory, $routeParams, websocketMsgSrv) { + 'ngInject'; - let vm = this - vm.clone = false - vm.notes = noteListFactory - vm.websocketMsgSrv = websocketMsgSrv - $scope.note = {} - $scope.interpreterSettings = {} - $scope.note.defaultInterpreter = null + let vm = this; + vm.clone = false; + vm.notes = noteListFactory; + vm.websocketMsgSrv = websocketMsgSrv; + $scope.note = {}; + $scope.interpreterSettings = {}; + $scope.note.defaultInterpreter = null; - vm.createNote = function () { + vm.createNote = function() { if (!vm.clone) { - let defaultInterpreterId = '' + let defaultInterpreterId = ''; if ($scope.note.defaultInterpreter !== null) { - defaultInterpreterId = $scope.note.defaultInterpreter.id + defaultInterpreterId = $scope.note.defaultInterpreter.id; } - vm.websocketMsgSrv.createNotebook($scope.note.notename, defaultInterpreterId) - $scope.note.defaultInterpreter = $scope.interpreterSettings[0] + vm.websocketMsgSrv.createNotebook($scope.note.notename, defaultInterpreterId); + $scope.note.defaultInterpreter = $scope.interpreterSettings[0]; } else { - let noteId = $routeParams.noteId - vm.websocketMsgSrv.cloneNote(noteId, $scope.note.notename) + let noteId = $routeParams.noteId; + vm.websocketMsgSrv.cloneNote(noteId, $scope.note.notename); } - } + }; - vm.handleNameEnter = function () { - angular.element('#noteCreateModal').modal('toggle') - vm.createNote() - } + vm.handleNameEnter = function() { + angular.element('#noteCreateModal').modal('toggle'); + vm.createNote(); + }; vm.preVisible = function(clone, sourceNoteName, path) { - vm.clone = clone - vm.sourceNoteName = sourceNoteName - $scope.note.notename = vm.clone ? vm.cloneNoteName() : vm.newNoteName(path) - $scope.$apply() - } + vm.clone = clone; + vm.sourceNoteName = sourceNoteName; + $scope.note.notename = vm.clone ? vm.cloneNoteName() : vm.newNoteName(path); + $scope.$apply(); + }; vm.newNoteName = function(path) { - let newCount = 1 - angular.forEach(vm.notes.flatList, function (noteName) { - noteName = noteName.name + let newCount = 1; + angular.forEach(vm.notes.flatList, function(noteName) { + noteName = noteName.name; if (noteName.match(/^Untitled Note [0-9]*$/)) { - let lastCount = noteName.substr(14) * 1 + let lastCount = noteName.substr(14) * 1; if (newCount <= lastCount) { - newCount = lastCount + 1 + newCount = lastCount + 1; } } - }) - return (path ? path + '/' : '') + 'Untitled Note ' + newCount - } + }); + return (path ? path + '/' : '') + 'Untitled Note ' + newCount; + }; - vm.cloneNoteName = function () { - let copyCount = 1 - let newCloneName = '' - let lastIndex = vm.sourceNoteName.lastIndexOf(' ') - let endsWithNumber = !!vm.sourceNoteName.match('^.+?\\s\\d$') - let noteNamePrefix = endsWithNumber ? vm.sourceNoteName.substr(0, lastIndex) : vm.sourceNoteName - let regexp = new RegExp('^' + noteNamePrefix + ' .+') + vm.cloneNoteName = function() { + let copyCount = 1; + let newCloneName = ''; + let lastIndex = vm.sourceNoteName.lastIndexOf(' '); + let endsWithNumber = !!vm.sourceNoteName.match('^.+?\\s\\d$'); + let noteNamePrefix = endsWithNumber ? vm.sourceNoteName.substr(0, lastIndex) : vm.sourceNoteName; + let regexp = new RegExp('^' + noteNamePrefix + ' .+'); - angular.forEach(vm.notes.flatList, function (noteName) { - noteName = noteName.name + angular.forEach(vm.notes.flatList, function(noteName) { + noteName = noteName.name; if (noteName.match(regexp)) { - let lastCopyCount = noteName.substr(lastIndex).trim() - newCloneName = noteNamePrefix - lastCopyCount = parseInt(lastCopyCount) + let lastCopyCount = noteName.substr(lastIndex).trim(); + newCloneName = noteNamePrefix; + lastCopyCount = parseInt(lastCopyCount); if (copyCount <= lastCopyCount) { - copyCount = lastCopyCount + 1 + copyCount = lastCopyCount + 1; } } - }) + }); if (!newCloneName) { - newCloneName = vm.sourceNoteName + newCloneName = vm.sourceNoteName; } - return newCloneName + ' ' + copyCount - } + return newCloneName + ' ' + copyCount; + }; - vm.getInterpreterSettings = function () { - vm.websocketMsgSrv.getInterpreterSettings() - } + vm.getInterpreterSettings = function() { + vm.websocketMsgSrv.getInterpreterSettings(); + }; - $scope.$on('interpreterSettings', function (event, data) { - $scope.interpreterSettings = data.interpreterSettings + $scope.$on('interpreterSettings', function(event, data) { + $scope.interpreterSettings = data.interpreterSettings; // initialize default interpreter with Spark interpreter - $scope.note.defaultInterpreter = data.interpreterSettings[0] - }) + $scope.note.defaultInterpreter = data.interpreterSettings[0]; + }); } diff --git a/zeppelin-web/src/components/note-create/note-create.controller.test.js b/zeppelin-web/src/components/note-create/note-create.controller.test.js index d409a142cd4..59f01d23b33 100644 --- a/zeppelin-web/src/components/note-create/note-create.controller.test.js +++ b/zeppelin-web/src/components/note-create/note-create.controller.test.js @@ -1,39 +1,39 @@ -describe('Controller: NoteCreateCtrl', function () { - beforeEach(angular.mock.module('zeppelinWebApp')) +describe('Controller: NoteCreateCtrl', function() { + beforeEach(angular.mock.module('zeppelinWebApp')); - let scope - let ctrl - let noteList + let scope; + let ctrl; + let noteList; - beforeEach(inject(function ($injector, $rootScope, $controller) { - noteList = $injector.get('noteListFactory') - scope = $rootScope.$new() + beforeEach(inject(function($injector, $rootScope, $controller) { + noteList = $injector.get('noteListFactory'); + scope = $rootScope.$new(); ctrl = $controller('NoteCreateCtrl', { $scope: scope, - noteListFactory: noteList - }) - })) + noteListFactory: noteList, + }); + })); - it('should create a new name from current name when cloneNoteName is called', function () { + it('should create a new name from current name when cloneNoteName is called', function() { let notesList = [ {name: 'dsds 1', id: '1'}, {name: 'dsds 2', id: '2'}, {name: 'test name', id: '3'}, {name: 'aa bb cc', id: '4'}, - {name: 'Untitled Note 6', id: '4'} - ] + {name: 'Untitled Note 6', id: '4'}, + ]; - noteList.setNotes(notesList) + noteList.setNotes(notesList); - ctrl.sourceNoteName = 'test name' - expect(ctrl.cloneNoteName()).toEqual('test name 1') - ctrl.sourceNoteName = 'aa bb cc' - expect(ctrl.cloneNoteName()).toEqual('aa bb cc 1') - ctrl.sourceNoteName = 'Untitled Note 6' - expect(ctrl.cloneNoteName()).toEqual('Untitled Note 7') - ctrl.sourceNoteName = 'My_note' - expect(ctrl.cloneNoteName()).toEqual('My_note 1') - ctrl.sourceNoteName = 'dsds 2' - expect(ctrl.cloneNoteName()).toEqual('dsds 3') - }) -}) + ctrl.sourceNoteName = 'test name'; + expect(ctrl.cloneNoteName()).toEqual('test name 1'); + ctrl.sourceNoteName = 'aa bb cc'; + expect(ctrl.cloneNoteName()).toEqual('aa bb cc 1'); + ctrl.sourceNoteName = 'Untitled Note 6'; + expect(ctrl.cloneNoteName()).toEqual('Untitled Note 7'); + ctrl.sourceNoteName = 'My_note'; + expect(ctrl.cloneNoteName()).toEqual('My_note 1'); + ctrl.sourceNoteName = 'dsds 2'; + expect(ctrl.cloneNoteName()).toEqual('dsds 3'); + }); +}); diff --git a/zeppelin-web/src/components/note-create/visible.directive.js b/zeppelin-web/src/components/note-create/visible.directive.js index 48c170f41a4..7ba8db72f3d 100644 --- a/zeppelin-web/src/components/note-create/visible.directive.js +++ b/zeppelin-web/src/components/note-create/visible.directive.js @@ -12,34 +12,34 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').directive('modalvisible', modalvisible) +angular.module('zeppelinWebApp').directive('modalvisible', modalvisible); -function modalvisible () { +function modalvisible() { return { restrict: 'A', scope: { preVisibleCallback: '&previsiblecallback', postVisibleCallback: '&postvisiblecallback', - targetinput: '@targetinput' + targetinput: '@targetinput', }, - link: function (scope, element, attrs) { + link: function(scope, element, attrs) { // Add some listeners - let previsibleMethod = scope.preVisibleCallback - let postVisibleMethod = scope.postVisibleCallback - element.on('show.bs.modal', function (e) { - let relatedTarget = angular.element(e.relatedTarget) - let clone = relatedTarget.data('clone') - let sourceNoteName = relatedTarget.data('source-note-name') - let path = relatedTarget.data('path') - let cloneNote = clone ? true : false - previsibleMethod()(cloneNote, sourceNoteName, path) - }) - element.on('shown.bs.modal', function (e) { + let previsibleMethod = scope.preVisibleCallback; + let postVisibleMethod = scope.postVisibleCallback; + element.on('show.bs.modal', function(e) { + let relatedTarget = angular.element(e.relatedTarget); + let clone = relatedTarget.data('clone'); + let sourceNoteName = relatedTarget.data('source-note-name'); + let path = relatedTarget.data('path'); + let cloneNote = clone ? true : false; + previsibleMethod()(cloneNote, sourceNoteName, path); + }); + element.on('shown.bs.modal', function(e) { if (scope.targetinput) { - angular.element(e.target).find('input#' + scope.targetinput).select() + angular.element(e.target).find('input#' + scope.targetinput).select(); } - postVisibleMethod() - }) - } - } + postVisibleMethod(); + }); + }, + }; } diff --git a/zeppelin-web/src/components/note-import/note-import.controller.js b/zeppelin-web/src/components/note-import/note-import.controller.js index 8cec8908a02..fbfc5facc82 100644 --- a/zeppelin-web/src/components/note-import/note-import.controller.js +++ b/zeppelin-web/src/components/note-import/note-import.controller.js @@ -12,76 +12,76 @@ * limitations under the License. */ -import './note-import.css' - -angular.module('zeppelinWebApp').controller('NoteImportCtrl', NoteImportCtrl) - -function NoteImportCtrl ($scope, $timeout, websocketMsgSrv) { - 'ngInject' - - let vm = this - $scope.note = {} - $scope.note.step1 = true - $scope.note.step2 = false - $scope.maxLimit = '' - let limit = 0 - - websocketMsgSrv.listConfigurations() - $scope.$on('configurationsInfo', function (scope, event) { - limit = event.configurations['zeppelin.websocket.max.text.message.size'] - $scope.maxLimit = Math.round(limit / 1048576) - }) - - vm.resetFlags = function () { - $scope.note = {} - $scope.note.step1 = true - $scope.note.step2 = false - angular.element('#noteImportFile').val('') - } - - $scope.uploadFile = function () { - angular.element('#noteImportFile').click() - } - - $scope.importFile = function (element) { - $scope.note.errorText = '' - $scope.note.importFile = element.files[0] - let file = $scope.note.importFile - let reader = new FileReader() +import './note-import.css'; + +angular.module('zeppelinWebApp').controller('NoteImportCtrl', NoteImportCtrl); + +function NoteImportCtrl($scope, $timeout, websocketMsgSrv) { + 'ngInject'; + + let vm = this; + $scope.note = {}; + $scope.note.step1 = true; + $scope.note.step2 = false; + $scope.maxLimit = ''; + let limit = 0; + + websocketMsgSrv.listConfigurations(); + $scope.$on('configurationsInfo', function(scope, event) { + limit = event.configurations['zeppelin.websocket.max.text.message.size']; + $scope.maxLimit = Math.round(limit / 1048576); + }); + + vm.resetFlags = function() { + $scope.note = {}; + $scope.note.step1 = true; + $scope.note.step2 = false; + angular.element('#noteImportFile').val(''); + }; + + $scope.uploadFile = function() { + angular.element('#noteImportFile').click(); + }; + + $scope.importFile = function(element) { + $scope.note.errorText = ''; + $scope.note.importFile = element.files[0]; + let file = $scope.note.importFile; + let reader = new FileReader(); if (file.size > limit) { - $scope.note.errorText = 'File size limit Exceeded!' - $scope.$apply() - return + $scope.note.errorText = 'File size limit Exceeded!'; + $scope.$apply(); + return; } - reader.onloadend = function () { - vm.processImportJson(reader.result) - } + reader.onloadend = function() { + vm.processImportJson(reader.result); + }; if (file) { - reader.readAsText(file) + reader.readAsText(file); } - } - - $scope.uploadURL = function () { - $scope.note.errorText = '' - $scope.note.step1 = false - $timeout(function () { - $scope.note.step2 = true - }, 400) - } - - vm.importBack = function () { - $scope.note.errorText = '' - $timeout(function () { - $scope.note.step1 = true - }, 400) - $scope.note.step2 = false - } - - vm.importNote = function () { - $scope.note.errorText = '' + }; + + $scope.uploadURL = function() { + $scope.note.errorText = ''; + $scope.note.step1 = false; + $timeout(function() { + $scope.note.step2 = true; + }, 400); + }; + + vm.importBack = function() { + $scope.note.errorText = ''; + $timeout(function() { + $scope.note.step1 = true; + }, 400); + $scope.note.step2 = false; + }; + + vm.importNote = function() { + $scope.note.errorText = ''; if ($scope.note.importUrl) { jQuery.ajax({ url: $scope.note.importUrl, @@ -89,50 +89,50 @@ function NoteImportCtrl ($scope, $timeout, websocketMsgSrv) { dataType: 'json', jsonp: false, xhrFields: { - withCredentials: false + withCredentials: false, }, - error: function (xhr, ajaxOptions, thrownError) { - $scope.note.errorText = 'Unable to Fetch URL' - $scope.$apply() - }}).done(function (data) { - vm.processImportJson(data) - }) + error: function(xhr, ajaxOptions, thrownError) { + $scope.note.errorText = 'Unable to Fetch URL'; + $scope.$apply(); + }}).done(function(data) { + vm.processImportJson(data); + }); } else { - $scope.note.errorText = 'Enter URL' - $scope.$apply() + $scope.note.errorText = 'Enter URL'; + $scope.$apply(); } - } + }; - vm.processImportJson = function (result) { + vm.processImportJson = function(result) { if (typeof result !== 'object') { try { - result = JSON.parse(result) + result = JSON.parse(result); } catch (e) { - $scope.note.errorText = 'JSON parse exception' - $scope.$apply() - return + $scope.note.errorText = 'JSON parse exception'; + $scope.$apply(); + return; } } if (result.paragraphs && result.paragraphs.length > 0) { if (!$scope.note.noteImportName) { - $scope.note.noteImportName = result.name + $scope.note.noteImportName = result.name; } else { - result.name = $scope.note.noteImportName + result.name = $scope.note.noteImportName; } - websocketMsgSrv.importNote(result) + websocketMsgSrv.importNote(result); // angular.element('#noteImportModal').modal('hide'); } else { - $scope.note.errorText = 'Invalid JSON' + $scope.note.errorText = 'Invalid JSON'; } - $scope.$apply() - } + $scope.$apply(); + }; /* ** $scope.$on functions below */ - $scope.$on('setNoteMenu', function (event, notes) { - vm.resetFlags() - angular.element('#noteImportModal').modal('hide') - }) + $scope.$on('setNoteMenu', function(event, notes) { + vm.resetFlags(); + angular.element('#noteImportModal').modal('hide'); + }); } diff --git a/zeppelin-web/src/components/note-list/note-list.factory.js b/zeppelin-web/src/components/note-list/note-list.factory.js index 5e2c513821c..59662fae35d 100644 --- a/zeppelin-web/src/components/note-list/note-list.factory.js +++ b/zeppelin-web/src/components/note-list/note-list.factory.js @@ -12,71 +12,73 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').factory('noteListFactory', NoteListFactory) +angular.module('zeppelinWebApp').factory('noteListFactory', NoteListFactory); function NoteListFactory(arrayOrderingSrv, TRASH_FOLDER_ID) { - 'ngInject' + 'ngInject'; const notes = { root: {children: []}, flatList: [], flatFolderMap: {}, - setNotes: function (notesList) { + setNotes: function(notesList) { // a flat list to boost searching notes.flatList = _.map(notesList, (note) => { note.isTrash = note.name - ? note.name.split('/')[0] === TRASH_FOLDER_ID : false - return note - }) + ? note.name.split('/')[0] === TRASH_FOLDER_ID : false; + return note; + }); // construct the folder-based tree - notes.root = {children: []} - notes.flatFolderMap = {} - _.reduce(notesList, function (root, note) { - let noteName = note.name || note.id - let nodes = noteName.match(/([^\/][^\/]*)/g) + notes.root = {children: []}; + notes.flatFolderMap = {}; + _.reduce(notesList, function(root, note) { + let noteName = note.name || note.id; + let nodes = noteName.match(/([^\/][^\/]*)/g); // recursively add nodes - addNode(root, nodes, note.id) + addNode(root, nodes, note.id); - return root - }, notes.root) - notes.root.children.sort(arrayOrderingSrv.noteComparator) - } - } + return root; + }, notes.root); + notes.root.children.sort(arrayOrderingSrv.noteComparator); + }, + }; - const addNode = function (curDir, nodes, noteId) { + const addNode = function(curDir, nodes, noteId) { if (nodes.length === 1) { // the leaf curDir.children.push({ name: nodes[0], id: noteId, path: curDir.id ? curDir.id + '/' + nodes[0] : nodes[0], - isTrash: curDir.id ? curDir.id.split('/')[0] === TRASH_FOLDER_ID : false - }) + isTrash: curDir.id ? curDir.id.split('/')[0] === TRASH_FOLDER_ID : false, + }); } else { // a folder node - let node = nodes.shift() + let node = nodes.shift(); let dir = _.find(curDir.children, - function (c) { return c.name === node && c.children !== undefined }) + function(c) { + return c.name === node && c.children !== undefined; + }); if (dir !== undefined) { // found an existing dir - addNode(dir, nodes, noteId) + addNode(dir, nodes, noteId); } else { let newDir = { id: curDir.id ? curDir.id + '/' + node : node, name: node, hidden: true, children: [], - isTrash: curDir.id ? curDir.id.split('/')[0] === TRASH_FOLDER_ID : false - } + isTrash: curDir.id ? curDir.id.split('/')[0] === TRASH_FOLDER_ID : false, + }; // add the folder to flat folder map - notes.flatFolderMap[newDir.id] = newDir + notes.flatFolderMap[newDir.id] = newDir; - curDir.children.push(newDir) - addNode(newDir, nodes, noteId) + curDir.children.push(newDir); + addNode(newDir, nodes, noteId); } } - } + }; - return notes + return notes; } diff --git a/zeppelin-web/src/components/note-list/note-list.factory.test.js b/zeppelin-web/src/components/note-list/note-list.factory.test.js index c16504c8785..2a962d8447c 100644 --- a/zeppelin-web/src/components/note-list/note-list.factory.test.js +++ b/zeppelin-web/src/components/note-list/note-list.factory.test.js @@ -1,15 +1,15 @@ -describe('Factory: NoteList', function () { - let noteList +describe('Factory: NoteList', function() { + let noteList; - beforeEach(function () { - angular.mock.module('zeppelinWebApp') + beforeEach(function() { + angular.mock.module('zeppelinWebApp'); - inject(function ($injector) { - noteList = $injector.get('noteListFactory') - }) - }) + inject(function($injector) { + noteList = $injector.get('noteListFactory'); + }); + }); - it('should generate both flat list and folder-based list properly', function () { + it('should generate both flat list and folder-based list properly', function() { let notesList = [ {name: 'A', id: '000001'}, {name: 'B', id: '000002'}, @@ -19,57 +19,57 @@ describe('Factory: NoteList', function () { {name: '/C/CB/CBA', id: '000006'}, // same name with a dir {name: '/C/CB/CBA', id: '000007'}, // same name with another note {name: 'C///CB//CBB', id: '000008'}, - {name: 'D/D[A/DA]B', id: '000009'} // check if '[' and ']' considered as folder seperator - ] - noteList.setNotes(notesList) + {name: 'D/D[A/DA]B', id: '000009'}, // check if '[' and ']' considered as folder seperator + ]; + noteList.setNotes(notesList); - let flatList = noteList.flatList - expect(flatList.length).toBe(9) - expect(flatList[0].name).toBe('A') - expect(flatList[0].id).toBe('000001') - expect(flatList[1].name).toBe('B') - expect(flatList[2].name).toBeUndefined() - expect(flatList[3].name).toBe('/C/CA') - expect(flatList[4].name).toBe('/C/CB') - expect(flatList[5].name).toBe('/C/CB/CBA') - expect(flatList[6].name).toBe('/C/CB/CBA') - expect(flatList[7].name).toBe('C///CB//CBB') - expect(flatList[8].name).toBe('D/D[A/DA]B') + let flatList = noteList.flatList; + expect(flatList.length).toBe(9); + expect(flatList[0].name).toBe('A'); + expect(flatList[0].id).toBe('000001'); + expect(flatList[1].name).toBe('B'); + expect(flatList[2].name).toBeUndefined(); + expect(flatList[3].name).toBe('/C/CA'); + expect(flatList[4].name).toBe('/C/CB'); + expect(flatList[5].name).toBe('/C/CB/CBA'); + expect(flatList[6].name).toBe('/C/CB/CBA'); + expect(flatList[7].name).toBe('C///CB//CBB'); + expect(flatList[8].name).toBe('D/D[A/DA]B'); - let folderList = noteList.root.children - expect(folderList.length).toBe(5) - expect(folderList[3].name).toBe('A') - expect(folderList[3].id).toBe('000001') - expect(folderList[4].name).toBe('B') - expect(folderList[2].name).toBe('000003') - expect(folderList[0].name).toBe('C') - expect(folderList[0].id).toBe('C') - expect(folderList[0].children.length).toBe(3) - expect(folderList[0].children[0].name).toBe('CA') - expect(folderList[0].children[0].id).toBe('000004') - expect(folderList[0].children[0].children).toBeUndefined() - expect(folderList[0].children[1].name).toBe('CB') - expect(folderList[0].children[1].id).toBe('000005') - expect(folderList[0].children[1].children).toBeUndefined() - expect(folderList[0].children[2].name).toBe('CB') - expect(folderList[0].children[2].id).toBe('C/CB') - expect(folderList[0].children[2].children.length).toBe(3) - expect(folderList[0].children[2].children[0].name).toBe('CBA') - expect(folderList[0].children[2].children[0].id).toBe('000006') - expect(folderList[0].children[2].children[0].children).toBeUndefined() - expect(folderList[0].children[2].children[1].name).toBe('CBA') - expect(folderList[0].children[2].children[1].id).toBe('000007') - expect(folderList[0].children[2].children[1].children).toBeUndefined() - expect(folderList[0].children[2].children[2].name).toBe('CBB') - expect(folderList[0].children[2].children[2].id).toBe('000008') - expect(folderList[0].children[2].children[2].children).toBeUndefined() - expect(folderList[1].name).toBe('D') - expect(folderList[1].id).toBe('D') - expect(folderList[1].children.length).toBe(1) - expect(folderList[1].children[0].name).toBe('D[A') - expect(folderList[1].children[0].id).toBe('D/D[A') - expect(folderList[1].children[0].children[0].name).toBe('DA]B') - expect(folderList[1].children[0].children[0].id).toBe('000009') - expect(folderList[1].children[0].children[0].children).toBeUndefined() - }) -}) + let folderList = noteList.root.children; + expect(folderList.length).toBe(5); + expect(folderList[3].name).toBe('A'); + expect(folderList[3].id).toBe('000001'); + expect(folderList[4].name).toBe('B'); + expect(folderList[2].name).toBe('000003'); + expect(folderList[0].name).toBe('C'); + expect(folderList[0].id).toBe('C'); + expect(folderList[0].children.length).toBe(3); + expect(folderList[0].children[0].name).toBe('CA'); + expect(folderList[0].children[0].id).toBe('000004'); + expect(folderList[0].children[0].children).toBeUndefined(); + expect(folderList[0].children[1].name).toBe('CB'); + expect(folderList[0].children[1].id).toBe('000005'); + expect(folderList[0].children[1].children).toBeUndefined(); + expect(folderList[0].children[2].name).toBe('CB'); + expect(folderList[0].children[2].id).toBe('C/CB'); + expect(folderList[0].children[2].children.length).toBe(3); + expect(folderList[0].children[2].children[0].name).toBe('CBA'); + expect(folderList[0].children[2].children[0].id).toBe('000006'); + expect(folderList[0].children[2].children[0].children).toBeUndefined(); + expect(folderList[0].children[2].children[1].name).toBe('CBA'); + expect(folderList[0].children[2].children[1].id).toBe('000007'); + expect(folderList[0].children[2].children[1].children).toBeUndefined(); + expect(folderList[0].children[2].children[2].name).toBe('CBB'); + expect(folderList[0].children[2].children[2].id).toBe('000008'); + expect(folderList[0].children[2].children[2].children).toBeUndefined(); + expect(folderList[1].name).toBe('D'); + expect(folderList[1].id).toBe('D'); + expect(folderList[1].children.length).toBe(1); + expect(folderList[1].children[0].name).toBe('D[A'); + expect(folderList[1].children[0].id).toBe('D/D[A'); + expect(folderList[1].children[0].children[0].name).toBe('DA]B'); + expect(folderList[1].children[0].children[0].id).toBe('000009'); + expect(folderList[1].children[0].children[0].children).toBeUndefined(); + }); +}); diff --git a/zeppelin-web/src/components/note-rename/note-rename.controller.js b/zeppelin-web/src/components/note-rename/note-rename.controller.js index b950d2b49e8..0fa31c44ee4 100644 --- a/zeppelin-web/src/components/note-rename/note-rename.controller.js +++ b/zeppelin-web/src/components/note-rename/note-rename.controller.js @@ -12,37 +12,37 @@ * limitations under the License. */ -import './note-rename.css' +import './note-rename.css'; -angular.module('zeppelinWebApp').controller('NoteRenameCtrl', NoteRenameController) +angular.module('zeppelinWebApp').controller('NoteRenameCtrl', NoteRenameController); function NoteRenameController($scope) { - 'ngInject' + 'ngInject'; - let self = this + let self = this; - $scope.params = {newName: ''} - $scope.isValid = true + $scope.params = {newName: ''}; + $scope.isValid = true; - $scope.rename = function () { - angular.element('#noteRenameModal').modal('hide') - self.callback($scope.params.newName) - } + $scope.rename = function() { + angular.element('#noteRenameModal').modal('hide'); + self.callback($scope.params.newName); + }; - $scope.$on('openRenameModal', function (event, options) { - self.validator = options.validator || defaultValidator - self.callback = options.callback || function () {} + $scope.$on('openRenameModal', function(event, options) { + self.validator = options.validator || defaultValidator; + self.callback = options.callback || function() {}; - $scope.title = options.title || 'Rename' - $scope.params.newName = options.oldName || '' - $scope.validate = function () { - $scope.isValid = self.validator($scope.params.newName) - } + $scope.title = options.title || 'Rename'; + $scope.params.newName = options.oldName || ''; + $scope.validate = function() { + $scope.isValid = self.validator($scope.params.newName); + }; - angular.element('#noteRenameModal').modal('show') - }) + angular.element('#noteRenameModal').modal('show'); + }); - function defaultValidator (str) { - return !!str.trim() + function defaultValidator(str) { + return !!str.trim(); } } diff --git a/zeppelin-web/src/components/note-rename/note-rename.service.js b/zeppelin-web/src/components/note-rename/note-rename.service.js index 64df82ff951..fd0f3e58e5d 100644 --- a/zeppelin-web/src/components/note-rename/note-rename.service.js +++ b/zeppelin-web/src/components/note-rename/note-rename.service.js @@ -12,12 +12,12 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').service('noteRenameService', NoteRenameService) +angular.module('zeppelinWebApp').service('noteRenameService', NoteRenameService); function NoteRenameService($rootScope) { - 'ngInject' + 'ngInject'; - let self = this + let self = this; /** * @@ -26,7 +26,7 @@ function NoteRenameService($rootScope) { * callback: (newName: string)=>void - callback onButtonClick * validator: (str: string)=>boolean - input validator */ - self.openRenameModal = function (options) { - $rootScope.$broadcast('openRenameModal', options) - } + self.openRenameModal = function(options) { + $rootScope.$broadcast('openRenameModal', options); + }; } diff --git a/zeppelin-web/src/components/websocket/websocket-event.factory.js b/zeppelin-web/src/components/websocket/websocket-event.factory.js index 70d61ecd964..18c704dd6df 100644 --- a/zeppelin-web/src/components/websocket/websocket-event.factory.js +++ b/zeppelin-web/src/components/websocket/websocket-event.factory.js @@ -12,92 +12,92 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').factory('websocketEvents', WebsocketEventFactory) +angular.module('zeppelinWebApp').factory('websocketEvents', WebsocketEventFactory); -function WebsocketEventFactory ($rootScope, $websocket, $location, baseUrlSrv) { - 'ngInject' +function WebsocketEventFactory($rootScope, $websocket, $location, baseUrlSrv) { + 'ngInject'; - let websocketCalls = {} - let pingIntervalId + let websocketCalls = {}; + let pingIntervalId; - websocketCalls.ws = $websocket(baseUrlSrv.getWebsocketUrl()) - websocketCalls.ws.reconnectIfNotNormalClose = true + websocketCalls.ws = $websocket(baseUrlSrv.getWebsocketUrl()); + websocketCalls.ws.reconnectIfNotNormalClose = true; - websocketCalls.ws.onOpen(function () { - console.log('Websocket created') - $rootScope.$broadcast('setConnectedStatus', true) - pingIntervalId = setInterval(function () { - websocketCalls.sendNewEvent({op: 'PING'}) - }, 10000) - }) + websocketCalls.ws.onOpen(function() { + console.log('Websocket created'); + $rootScope.$broadcast('setConnectedStatus', true); + pingIntervalId = setInterval(function() { + websocketCalls.sendNewEvent({op: 'PING'}); + }, 10000); + }); - websocketCalls.sendNewEvent = function (data) { + websocketCalls.sendNewEvent = function(data) { if ($rootScope.ticket !== undefined) { - data.principal = $rootScope.ticket.principal - data.ticket = $rootScope.ticket.ticket - data.roles = $rootScope.ticket.roles + data.principal = $rootScope.ticket.principal; + data.ticket = $rootScope.ticket.ticket; + data.roles = $rootScope.ticket.roles; } else { - data.principal = '' - data.ticket = '' - data.roles = '' + data.principal = ''; + data.ticket = ''; + data.roles = ''; } - console.log('Send >> %o, %o, %o, %o, %o', data.op, data.principal, data.ticket, data.roles, data) - return websocketCalls.ws.send(JSON.stringify(data)) - } + console.log('Send >> %o, %o, %o, %o, %o', data.op, data.principal, data.ticket, data.roles, data); + return websocketCalls.ws.send(JSON.stringify(data)); + }; - websocketCalls.isConnected = function () { - return (websocketCalls.ws.socket.readyState === 1) - } + websocketCalls.isConnected = function() { + return (websocketCalls.ws.socket.readyState === 1); + }; - websocketCalls.ws.onMessage(function (event) { - let payload + websocketCalls.ws.onMessage(function(event) { + let payload; if (event.data) { - payload = angular.fromJson(event.data) + payload = angular.fromJson(event.data); } - console.log('Receive << %o, %o', payload.op, payload) + console.log('Receive << %o, %o', payload.op, payload); - let op = payload.op - let data = payload.data + let op = payload.op; + let data = payload.data; if (op === 'NOTE') { - $rootScope.$broadcast('setNoteContent', data.note) + $rootScope.$broadcast('setNoteContent', data.note); } else if (op === 'NEW_NOTE') { - $location.path('/notebook/' + data.note.id) + $location.path('/notebook/' + data.note.id); } else if (op === 'NOTES_INFO') { - $rootScope.$broadcast('setNoteMenu', data.notes) + $rootScope.$broadcast('setNoteMenu', data.notes); } else if (op === 'LIST_NOTE_JOBS') { - $rootScope.$emit('jobmanager:set-jobs', data.noteJobs) + $rootScope.$emit('jobmanager:set-jobs', data.noteJobs); } else if (op === 'LIST_UPDATE_NOTE_JOBS') { - $rootScope.$emit('jobmanager:update-jobs', data.noteRunningJobs) + $rootScope.$emit('jobmanager:update-jobs', data.noteRunningJobs); } else if (op === 'AUTH_INFO') { - let btn = [] + let btn = []; if ($rootScope.ticket.roles === '[]') { btn = [{ label: 'Close', - action: function (dialog) { - dialog.close() - } - }] + action: function(dialog) { + dialog.close(); + }, + }]; } else { btn = [{ label: 'Login', - action: function (dialog) { - dialog.close() + action: function(dialog) { + dialog.close(); angular.element('#loginModal').modal({ - show: 'true' - }) - } + show: 'true', + }); + }, }, { label: 'Cancel', - action: function (dialog) { - dialog.close() + action: function(dialog) { + dialog.close(); // using $rootScope.apply to trigger angular digest cycle // changing $location.path inside bootstrap modal wont trigger digest - $rootScope.$apply(function () { - $location.path('/') - }) - } - }] + $rootScope.$apply(function() { + $location.path('/'); + }); + }, + }]; } BootstrapDialog.show({ @@ -106,44 +106,44 @@ function WebsocketEventFactory ($rootScope, $websocket, $location, baseUrlSrv) { closeByKeyboard: false, title: 'Insufficient privileges', message: data.info.toString(), - buttons: btn - }) + buttons: btn, + }); } else if (op === 'PARAGRAPH') { - $rootScope.$broadcast('updateParagraph', data) + $rootScope.$broadcast('updateParagraph', data); } else if (op === 'RUN_PARAGRAPH_USING_SPELL') { - $rootScope.$broadcast('runParagraphUsingSpell', data) + $rootScope.$broadcast('runParagraphUsingSpell', data); } else if (op === 'PARAGRAPH_APPEND_OUTPUT') { - $rootScope.$broadcast('appendParagraphOutput', data) + $rootScope.$broadcast('appendParagraphOutput', data); } else if (op === 'PARAGRAPH_UPDATE_OUTPUT') { - $rootScope.$broadcast('updateParagraphOutput', data) + $rootScope.$broadcast('updateParagraphOutput', data); } else if (op === 'PROGRESS') { - $rootScope.$broadcast('updateProgress', data) + $rootScope.$broadcast('updateProgress', data); } else if (op === 'COMPLETION_LIST') { - $rootScope.$broadcast('completionList', data) + $rootScope.$broadcast('completionList', data); } else if (op === 'EDITOR_SETTING') { - $rootScope.$broadcast('editorSetting', data) + $rootScope.$broadcast('editorSetting', data); } else if (op === 'ANGULAR_OBJECT_UPDATE') { - $rootScope.$broadcast('angularObjectUpdate', data) + $rootScope.$broadcast('angularObjectUpdate', data); } else if (op === 'ANGULAR_OBJECT_REMOVE') { - $rootScope.$broadcast('angularObjectRemove', data) + $rootScope.$broadcast('angularObjectRemove', data); } else if (op === 'APP_APPEND_OUTPUT') { - $rootScope.$broadcast('appendAppOutput', data) + $rootScope.$broadcast('appendAppOutput', data); } else if (op === 'APP_UPDATE_OUTPUT') { - $rootScope.$broadcast('updateAppOutput', data) + $rootScope.$broadcast('updateAppOutput', data); } else if (op === 'APP_LOAD') { - $rootScope.$broadcast('appLoad', data) + $rootScope.$broadcast('appLoad', data); } else if (op === 'APP_STATUS_CHANGE') { - $rootScope.$broadcast('appStatusChange', data) + $rootScope.$broadcast('appStatusChange', data); } else if (op === 'LIST_REVISION_HISTORY') { - $rootScope.$broadcast('listRevisionHistory', data) + $rootScope.$broadcast('listRevisionHistory', data); } else if (op === 'NOTE_REVISION') { - $rootScope.$broadcast('noteRevision', data) + $rootScope.$broadcast('noteRevision', data); } else if (op === 'NOTE_REVISION_FOR_COMPARE') { - $rootScope.$broadcast('noteRevisionForCompare', data) + $rootScope.$broadcast('noteRevisionForCompare', data); } else if (op === 'INTERPRETER_BINDINGS') { - $rootScope.$broadcast('interpreterBindings', data) + $rootScope.$broadcast('interpreterBindings', data); } else if (op === 'SAVE_NOTE_FORMS') { - $rootScope.$broadcast('saveNoteForms', data) + $rootScope.$broadcast('saveNoteForms', data); } else if (op === 'ERROR_INFO') { BootstrapDialog.show({ closable: false, @@ -154,47 +154,47 @@ function WebsocketEventFactory ($rootScope, $websocket, $location, baseUrlSrv) { buttons: [{ // close all the dialogs when there are error on running all paragraphs label: 'Close', - action: function () { - BootstrapDialog.closeAll() - } - }] - }) + action: function() { + BootstrapDialog.closeAll(); + }, + }], + }); } else if (op === 'SESSION_LOGOUT') { - $rootScope.$broadcast('session_logout', data) + $rootScope.$broadcast('session_logout', data); } else if (op === 'CONFIGURATIONS_INFO') { - $rootScope.$broadcast('configurationsInfo', data) + $rootScope.$broadcast('configurationsInfo', data); } else if (op === 'INTERPRETER_SETTINGS') { - $rootScope.$broadcast('interpreterSettings', data) + $rootScope.$broadcast('interpreterSettings', data); } else if (op === 'PARAGRAPH_ADDED') { - $rootScope.$broadcast('addParagraph', data.paragraph, data.index) + $rootScope.$broadcast('addParagraph', data.paragraph, data.index); } else if (op === 'PARAGRAPH_REMOVED') { - $rootScope.$broadcast('removeParagraph', data.id) + $rootScope.$broadcast('removeParagraph', data.id); } else if (op === 'PARAGRAPH_MOVED') { - $rootScope.$broadcast('moveParagraph', data.id, data.index) + $rootScope.$broadcast('moveParagraph', data.id, data.index); } else if (op === 'NOTE_UPDATED') { - $rootScope.$broadcast('updateNote', data.name, data.config, data.info) + $rootScope.$broadcast('updateNote', data.name, data.config, data.info); } else if (op === 'SET_NOTE_REVISION') { - $rootScope.$broadcast('setNoteRevisionResult', data) + $rootScope.$broadcast('setNoteRevisionResult', data); } else if (op === 'PARAS_INFO') { - $rootScope.$broadcast('updateParaInfos', data) + $rootScope.$broadcast('updateParaInfos', data); } else { - console.error(`unknown websocket op: ${op}`) + console.error(`unknown websocket op: ${op}`); } - }) + }); - websocketCalls.ws.onError(function (event) { - console.log('error message: ', event) - $rootScope.$broadcast('setConnectedStatus', false) - }) + websocketCalls.ws.onError(function(event) { + console.log('error message: ', event); + $rootScope.$broadcast('setConnectedStatus', false); + }); - websocketCalls.ws.onClose(function (event) { - console.log('close message: ', event) + websocketCalls.ws.onClose(function(event) { + console.log('close message: ', event); if (pingIntervalId !== undefined) { - clearInterval(pingIntervalId) - pingIntervalId = undefined + clearInterval(pingIntervalId); + pingIntervalId = undefined; } - $rootScope.$broadcast('setConnectedStatus', false) - }) + $rootScope.$broadcast('setConnectedStatus', false); + }); - return websocketCalls + return websocketCalls; } diff --git a/zeppelin-web/src/components/websocket/websocket-message.service.js b/zeppelin-web/src/components/websocket/websocket-message.service.js index cd65e1d3194..f0cf92b3787 100644 --- a/zeppelin-web/src/components/websocket/websocket-message.service.js +++ b/zeppelin-web/src/components/websocket/websocket-message.service.js @@ -12,100 +12,100 @@ * limitations under the License. */ -angular.module('zeppelinWebApp').service('websocketMsgSrv', WebsocketMessageService) +angular.module('zeppelinWebApp').service('websocketMsgSrv', WebsocketMessageService); -function WebsocketMessageService ($rootScope, websocketEvents) { - 'ngInject' +function WebsocketMessageService($rootScope, websocketEvents) { + 'ngInject'; return { - getHomeNote: function () { - websocketEvents.sendNewEvent({op: 'GET_HOME_NOTE'}) + getHomeNote: function() { + websocketEvents.sendNewEvent({op: 'GET_HOME_NOTE'}); }, - createNotebook: function (noteName, defaultInterpreterId) { + createNotebook: function(noteName, defaultInterpreterId) { websocketEvents.sendNewEvent({ op: 'NEW_NOTE', data: { name: noteName, - defaultInterpreterId: defaultInterpreterId - } - }) + defaultInterpreterId: defaultInterpreterId, + }, + }); }, - moveNoteToTrash: function (noteId) { - websocketEvents.sendNewEvent({op: 'MOVE_NOTE_TO_TRASH', data: {id: noteId}}) + moveNoteToTrash: function(noteId) { + websocketEvents.sendNewEvent({op: 'MOVE_NOTE_TO_TRASH', data: {id: noteId}}); }, - moveFolderToTrash: function (folderId) { - websocketEvents.sendNewEvent({op: 'MOVE_FOLDER_TO_TRASH', data: {id: folderId}}) + moveFolderToTrash: function(folderId) { + websocketEvents.sendNewEvent({op: 'MOVE_FOLDER_TO_TRASH', data: {id: folderId}}); }, - restoreNote: function (noteId) { - websocketEvents.sendNewEvent({op: 'RESTORE_NOTE', data: {id: noteId}}) + restoreNote: function(noteId) { + websocketEvents.sendNewEvent({op: 'RESTORE_NOTE', data: {id: noteId}}); }, - restoreFolder: function (folderId) { - websocketEvents.sendNewEvent({op: 'RESTORE_FOLDER', data: {id: folderId}}) + restoreFolder: function(folderId) { + websocketEvents.sendNewEvent({op: 'RESTORE_FOLDER', data: {id: folderId}}); }, - restoreAll: function () { - websocketEvents.sendNewEvent({op: 'RESTORE_ALL'}) + restoreAll: function() { + websocketEvents.sendNewEvent({op: 'RESTORE_ALL'}); }, - deleteNote: function (noteId) { - websocketEvents.sendNewEvent({op: 'DEL_NOTE', data: {id: noteId}}) + deleteNote: function(noteId) { + websocketEvents.sendNewEvent({op: 'DEL_NOTE', data: {id: noteId}}); }, - removeFolder: function (folderId) { - websocketEvents.sendNewEvent({op: 'REMOVE_FOLDER', data: {id: folderId}}) + removeFolder: function(folderId) { + websocketEvents.sendNewEvent({op: 'REMOVE_FOLDER', data: {id: folderId}}); }, - emptyTrash: function () { - websocketEvents.sendNewEvent({op: 'EMPTY_TRASH'}) + emptyTrash: function() { + websocketEvents.sendNewEvent({op: 'EMPTY_TRASH'}); }, - cloneNote: function (noteIdToClone, newNoteName) { - websocketEvents.sendNewEvent({op: 'CLONE_NOTE', data: {id: noteIdToClone, name: newNoteName}}) + cloneNote: function(noteIdToClone, newNoteName) { + websocketEvents.sendNewEvent({op: 'CLONE_NOTE', data: {id: noteIdToClone, name: newNoteName}}); }, - getNoteList: function () { - websocketEvents.sendNewEvent({op: 'LIST_NOTES'}) + getNoteList: function() { + websocketEvents.sendNewEvent({op: 'LIST_NOTES'}); }, - reloadAllNotesFromRepo: function () { - websocketEvents.sendNewEvent({op: 'RELOAD_NOTES_FROM_REPO'}) + reloadAllNotesFromRepo: function() { + websocketEvents.sendNewEvent({op: 'RELOAD_NOTES_FROM_REPO'}); }, - getNote: function (noteId) { - websocketEvents.sendNewEvent({op: 'GET_NOTE', data: {id: noteId}}) + getNote: function(noteId) { + websocketEvents.sendNewEvent({op: 'GET_NOTE', data: {id: noteId}}); }, - updateNote: function (noteId, noteName, noteConfig) { - websocketEvents.sendNewEvent({op: 'NOTE_UPDATE', data: {id: noteId, name: noteName, config: noteConfig}}) + updateNote: function(noteId, noteName, noteConfig) { + websocketEvents.sendNewEvent({op: 'NOTE_UPDATE', data: {id: noteId, name: noteName, config: noteConfig}}); }, - updatePersonalizedMode: function (noteId, modeValue) { - websocketEvents.sendNewEvent({op: 'UPDATE_PERSONALIZED_MODE', data: {id: noteId, personalized: modeValue}}) + updatePersonalizedMode: function(noteId, modeValue) { + websocketEvents.sendNewEvent({op: 'UPDATE_PERSONALIZED_MODE', data: {id: noteId, personalized: modeValue}}); }, - renameNote: function (noteId, noteName) { - websocketEvents.sendNewEvent({op: 'NOTE_RENAME', data: {id: noteId, name: noteName}}) + renameNote: function(noteId, noteName) { + websocketEvents.sendNewEvent({op: 'NOTE_RENAME', data: {id: noteId, name: noteName}}); }, - renameFolder: function (folderId, folderName) { - websocketEvents.sendNewEvent({op: 'FOLDER_RENAME', data: {id: folderId, name: folderName}}) + renameFolder: function(folderId, folderName) { + websocketEvents.sendNewEvent({op: 'FOLDER_RENAME', data: {id: folderId, name: folderName}}); }, - moveParagraph: function (paragraphId, newIndex) { - websocketEvents.sendNewEvent({op: 'MOVE_PARAGRAPH', data: {id: paragraphId, index: newIndex}}) + moveParagraph: function(paragraphId, newIndex) { + websocketEvents.sendNewEvent({op: 'MOVE_PARAGRAPH', data: {id: paragraphId, index: newIndex}}); }, - insertParagraph: function (newIndex) { - websocketEvents.sendNewEvent({op: 'INSERT_PARAGRAPH', data: {index: newIndex}}) + insertParagraph: function(newIndex) { + websocketEvents.sendNewEvent({op: 'INSERT_PARAGRAPH', data: {index: newIndex}}); }, - copyParagraph: function (newIndex, paragraphTitle, paragraphData, + copyParagraph: function(newIndex, paragraphTitle, paragraphData, paragraphConfig, paragraphParams) { websocketEvents.sendNewEvent({ op: 'COPY_PARAGRAPH', @@ -114,12 +114,12 @@ function WebsocketMessageService ($rootScope, websocketEvents) { title: paragraphTitle, paragraph: paragraphData, config: paragraphConfig, - params: paragraphParams - } - }) + params: paragraphParams, + }, + }); }, - updateAngularObject: function (noteId, paragraphId, name, value, interpreterGroupId) { + updateAngularObject: function(noteId, paragraphId, name, value, interpreterGroupId) { websocketEvents.sendNewEvent({ op: 'ANGULAR_OBJECT_UPDATED', data: { @@ -127,39 +127,39 @@ function WebsocketMessageService ($rootScope, websocketEvents) { paragraphId: paragraphId, name: name, value: value, - interpreterGroupId: interpreterGroupId - } - }) + interpreterGroupId: interpreterGroupId, + }, + }); }, - clientBindAngularObject: function (noteId, name, value, paragraphId) { + clientBindAngularObject: function(noteId, name, value, paragraphId) { websocketEvents.sendNewEvent({ op: 'ANGULAR_OBJECT_CLIENT_BIND', data: { noteId: noteId, name: name, value: value, - paragraphId: paragraphId - } - }) + paragraphId: paragraphId, + }, + }); }, - clientUnbindAngularObject: function (noteId, name, paragraphId) { + clientUnbindAngularObject: function(noteId, name, paragraphId) { websocketEvents.sendNewEvent({ op: 'ANGULAR_OBJECT_CLIENT_UNBIND', data: { noteId: noteId, name: name, - paragraphId: paragraphId - } - }) + paragraphId: paragraphId, + }, + }); }, - cancelParagraphRun: function (paragraphId) { - websocketEvents.sendNewEvent({op: 'CANCEL_PARAGRAPH', data: {id: paragraphId}}) + cancelParagraphRun: function(paragraphId) { + websocketEvents.sendNewEvent({op: 'CANCEL_PARAGRAPH', data: {id: paragraphId}}); }, - paragraphExecutedBySpell: function (paragraphId, paragraphTitle, + paragraphExecutedBySpell: function(paragraphId, paragraphTitle, paragraphText, paragraphResultsMsg, paragraphStatus, paragraphErrorMessage, paragraphConfig, paragraphParams, @@ -172,10 +172,10 @@ function WebsocketMessageService ($rootScope, websocketEvents) { paragraph: paragraphText, results: { code: paragraphStatus, - msg: paragraphResultsMsg.map(dataWithType => { - let serializedData = dataWithType.data - return { type: dataWithType.type, data: serializedData, } - }) + msg: paragraphResultsMsg.map((dataWithType) => { + let serializedData = dataWithType.data; + return {type: dataWithType.type, data: serializedData}; + }), }, status: paragraphStatus, errorMessage: paragraphErrorMessage, @@ -183,11 +183,11 @@ function WebsocketMessageService ($rootScope, websocketEvents) { params: paragraphParams, dateStarted: paragraphDateStarted, dateFinished: paragraphDateFinished, - } - }) + }, + }); }, - runParagraph: function (paragraphId, paragraphTitle, paragraphData, paragraphConfig, paragraphParams) { + runParagraph: function(paragraphId, paragraphTitle, paragraphData, paragraphConfig, paragraphParams) { websocketEvents.sendNewEvent({ op: 'RUN_PARAGRAPH', data: { @@ -195,45 +195,45 @@ function WebsocketMessageService ($rootScope, websocketEvents) { title: paragraphTitle, paragraph: paragraphData, config: paragraphConfig, - params: paragraphParams - } - }) + params: paragraphParams, + }, + }); }, - runAllParagraphs: function (noteId, paragraphs) { + runAllParagraphs: function(noteId, paragraphs) { websocketEvents.sendNewEvent({ op: 'RUN_ALL_PARAGRAPHS', data: { noteId: noteId, - paragraphs: JSON.stringify(paragraphs) - } - }) + paragraphs: JSON.stringify(paragraphs), + }, + }); }, - removeParagraph: function (paragraphId) { - websocketEvents.sendNewEvent({op: 'PARAGRAPH_REMOVE', data: {id: paragraphId}}) + removeParagraph: function(paragraphId) { + websocketEvents.sendNewEvent({op: 'PARAGRAPH_REMOVE', data: {id: paragraphId}}); }, - clearParagraphOutput: function (paragraphId) { - websocketEvents.sendNewEvent({op: 'PARAGRAPH_CLEAR_OUTPUT', data: {id: paragraphId}}) + clearParagraphOutput: function(paragraphId) { + websocketEvents.sendNewEvent({op: 'PARAGRAPH_CLEAR_OUTPUT', data: {id: paragraphId}}); }, - clearAllParagraphOutput: function (noteId) { - websocketEvents.sendNewEvent({op: 'PARAGRAPH_CLEAR_ALL_OUTPUT', data: {id: noteId}}) + clearAllParagraphOutput: function(noteId) { + websocketEvents.sendNewEvent({op: 'PARAGRAPH_CLEAR_ALL_OUTPUT', data: {id: noteId}}); }, - completion: function (paragraphId, buf, cursor) { + completion: function(paragraphId, buf, cursor) { websocketEvents.sendNewEvent({ op: 'COMPLETION', data: { id: paragraphId, buf: buf, - cursor: cursor - } - }) + cursor: cursor, + }, + }); }, - commitParagraph: function (paragraphId, paragraphTitle, paragraphData, paragraphConfig, paragraphParams, noteId) { + commitParagraph: function(paragraphId, paragraphTitle, paragraphData, paragraphConfig, paragraphParams, noteId) { return websocketEvents.sendNewEvent({ op: 'COMMIT_PARAGRAPH', data: { @@ -242,132 +242,132 @@ function WebsocketMessageService ($rootScope, websocketEvents) { title: paragraphTitle, paragraph: paragraphData, config: paragraphConfig, - params: paragraphParams - } - }) + params: paragraphParams, + }, + }); }, - importNote: function (note) { + importNote: function(note) { websocketEvents.sendNewEvent({ op: 'IMPORT_NOTE', data: { - note: note - } - }) + note: note, + }, + }); }, - checkpointNote: function (noteId, commitMessage) { + checkpointNote: function(noteId, commitMessage) { websocketEvents.sendNewEvent({ op: 'CHECKPOINT_NOTE', data: { noteId: noteId, - commitMessage: commitMessage - } - }) + commitMessage: commitMessage, + }, + }); }, - setNoteRevision: function (noteId, revisionId) { + setNoteRevision: function(noteId, revisionId) { websocketEvents.sendNewEvent({ op: 'SET_NOTE_REVISION', data: { noteId: noteId, - revisionId: revisionId - } - }) + revisionId: revisionId, + }, + }); }, - listRevisionHistory: function (noteId) { + listRevisionHistory: function(noteId) { websocketEvents.sendNewEvent({ op: 'LIST_REVISION_HISTORY', data: { - noteId: noteId - } - }) + noteId: noteId, + }, + }); }, - getNoteByRevision: function (noteId, revisionId) { + getNoteByRevision: function(noteId, revisionId) { websocketEvents.sendNewEvent({ op: 'NOTE_REVISION', data: { noteId: noteId, - revisionId: revisionId - } - }) + revisionId: revisionId, + }, + }); }, - getNoteByRevisionForCompare: function (noteId, revisionId, position) { + getNoteByRevisionForCompare: function(noteId, revisionId, position) { websocketEvents.sendNewEvent({ op: 'NOTE_REVISION_FOR_COMPARE', data: { noteId: noteId, revisionId: revisionId, - position: position - } - }) + position: position, + }, + }); }, - getEditorSetting: function (paragraphId, replName) { + getEditorSetting: function(paragraphId, replName) { websocketEvents.sendNewEvent({ op: 'EDITOR_SETTING', data: { paragraphId: paragraphId, - magic: replName - } - }) + magic: replName, + }, + }); }, - isConnected: function () { - return websocketEvents.isConnected() + isConnected: function() { + return websocketEvents.isConnected(); }, - getJobs: function () { - websocketEvents.sendNewEvent({op: 'LIST_NOTE_JOBS'}) + getJobs: function() { + websocketEvents.sendNewEvent({op: 'LIST_NOTE_JOBS'}); }, - disconnectJobEvent: function () { - websocketEvents.sendNewEvent({op: 'UNSUBSCRIBE_UPDATE_NOTE_JOBS'}) + disconnectJobEvent: function() { + websocketEvents.sendNewEvent({op: 'UNSUBSCRIBE_UPDATE_NOTE_JOBS'}); }, - getUpdateNoteJobsList: function (lastUpdateServerUnixTime) { + getUpdateNoteJobsList: function(lastUpdateServerUnixTime) { websocketEvents.sendNewEvent( {op: 'LIST_UPDATE_NOTE_JOBS', data: {lastUpdateUnixTime: lastUpdateServerUnixTime * 1}} - ) + ); }, - getInterpreterBindings: function (noteId) { - websocketEvents.sendNewEvent({op: 'GET_INTERPRETER_BINDINGS', data: {noteId: noteId}}) + getInterpreterBindings: function(noteId) { + websocketEvents.sendNewEvent({op: 'GET_INTERPRETER_BINDINGS', data: {noteId: noteId}}); }, - saveInterpreterBindings: function (noteId, selectedSettingIds) { + saveInterpreterBindings: function(noteId, selectedSettingIds) { websocketEvents.sendNewEvent({op: 'SAVE_INTERPRETER_BINDINGS', - data: {noteId: noteId, selectedSettingIds: selectedSettingIds}}) + data: {noteId: noteId, selectedSettingIds: selectedSettingIds}}); }, - listConfigurations: function () { - websocketEvents.sendNewEvent({op: 'LIST_CONFIGURATIONS'}) + listConfigurations: function() { + websocketEvents.sendNewEvent({op: 'LIST_CONFIGURATIONS'}); }, - getInterpreterSettings: function () { - websocketEvents.sendNewEvent({op: 'GET_INTERPRETER_SETTINGS'}) + getInterpreterSettings: function() { + websocketEvents.sendNewEvent({op: 'GET_INTERPRETER_SETTINGS'}); }, - saveNoteForms: function (note) { + saveNoteForms: function(note) { websocketEvents.sendNewEvent({op: 'SAVE_NOTE_FORMS', data: { noteId: note.id, - noteParams: note.noteParams - } - }) + noteParams: note.noteParams, + }, + }); }, - removeNoteForms: function (note, formName) { + removeNoteForms: function(note, formName) { websocketEvents.sendNewEvent({op: 'REMOVE_NOTE_FORMS', data: { noteId: note.id, - formName: formName - } - }) - } + formName: formName, + }, + }); + }, - } + }; } diff --git a/zeppelin-web/src/index.js b/zeppelin-web/src/index.js index 4c41336a1f7..55d6155aca0 100644 --- a/zeppelin-web/src/index.js +++ b/zeppelin-web/src/index.js @@ -13,65 +13,65 @@ */ // import globally uses css here -import 'github-markdown-css/github-markdown.css' +import 'github-markdown-css/github-markdown.css'; -import './app/app.js' -import './app/app.controller.js' -import './app/home/home.controller.js' -import './app/notebook/notebook.controller.js' +import './app/app.js'; +import './app/app.controller.js'; +import './app/home/home.controller.js'; +import './app/notebook/notebook.controller.js'; -import './app/tabledata/tabledata.js' -import './app/tabledata/transformation.js' -import './app/tabledata/pivot.js' -import './app/tabledata/passthrough.js' -import './app/tabledata/columnselector.js' -import './app/tabledata/advanced-transformation.js' -import './app/visualization/visualization.js' -import './app/visualization/builtins/visualization-table.js' -import './app/visualization/builtins/visualization-nvd3chart.js' -import './app/visualization/builtins/visualization-barchart.js' -import './app/visualization/builtins/visualization-piechart.js' -import './app/visualization/builtins/visualization-areachart.js' -import './app/visualization/builtins/visualization-linechart.js' -import './app/visualization/builtins/visualization-scatterchart.js' +import './app/tabledata/tabledata.js'; +import './app/tabledata/transformation.js'; +import './app/tabledata/pivot.js'; +import './app/tabledata/passthrough.js'; +import './app/tabledata/columnselector.js'; +import './app/tabledata/advanced-transformation.js'; +import './app/visualization/visualization.js'; +import './app/visualization/builtins/visualization-table.js'; +import './app/visualization/builtins/visualization-nvd3chart.js'; +import './app/visualization/builtins/visualization-barchart.js'; +import './app/visualization/builtins/visualization-piechart.js'; +import './app/visualization/builtins/visualization-areachart.js'; +import './app/visualization/builtins/visualization-linechart.js'; +import './app/visualization/builtins/visualization-scatterchart.js'; -import './app/jobmanager/jobmanager.component.js' -import './app/interpreter/interpreter.controller.js' -import './app/interpreter/interpreter.filter.js' -import './app/interpreter/interpreter-item.directive.js' -import './app/interpreter/widget/number-widget.directive.js' -import './app/credential/credential.controller.js' -import './app/configuration/configuration.controller.js' -import './app/notebook/revisions-comparator/revisions-comparator.component.js' -import './app/notebook/paragraph/paragraph.controller.js' -import './app/notebook/paragraph/clipboard.controller.js' -import './app/notebook/paragraph/resizable.directive.js' -import './app/notebook/paragraph/result/result.controller.js' -import './app/notebook/paragraph/code-editor/code-editor.directive.js' -import './app/notebook/save-as/save-as.service.js' -import './app/notebook/save-as/browser-detect.service.js' -import './app/notebook/elastic-input/elastic-input.controller.js' -import './app/notebook/dropdown-input/dropdown-input.directive.js' -import './app/notebook/note-var-share.service.js' -import './app/notebook-repository/notebook-repository.controller.js' -import './app/search/result-list.controller.js' -import './app/search/search.service.js' -import './app/helium' -import './app/helium/helium.service.js' -import './app/notebook/dynamic-forms/dynamic-forms.directive.js' -import './components/array-ordering/array-ordering.service.js' -import './components/navbar/navbar.controller.js' -import './components/navbar/expand-collapse/expand-collapse.directive.js' -import './components/note-create/note-create.controller.js' -import './components/note-create/visible.directive.js' -import './components/note-import/note-import.controller.js' -import './components/ng-enter/ng-enter.directive.js' -import './components/ng-escape/ng-escape.directive.js' -import './components/websocket/websocket-message.service.js' -import './components/websocket/websocket-event.factory.js' -import './components/note-list/note-list.factory.js' -import './components/base-url/base-url.service.js' -import './components/login/login.controller.js' -import './components/note-action/note-action.service.js' -import './components/note-rename/note-rename.controller.js' -import './components/note-rename/note-rename.service.js' +import './app/jobmanager/jobmanager.component.js'; +import './app/interpreter/interpreter.controller.js'; +import './app/interpreter/interpreter.filter.js'; +import './app/interpreter/interpreter-item.directive.js'; +import './app/interpreter/widget/number-widget.directive.js'; +import './app/credential/credential.controller.js'; +import './app/configuration/configuration.controller.js'; +import './app/notebook/revisions-comparator/revisions-comparator.component.js'; +import './app/notebook/paragraph/paragraph.controller.js'; +import './app/notebook/paragraph/clipboard.controller.js'; +import './app/notebook/paragraph/resizable.directive.js'; +import './app/notebook/paragraph/result/result.controller.js'; +import './app/notebook/paragraph/code-editor/code-editor.directive.js'; +import './app/notebook/save-as/save-as.service.js'; +import './app/notebook/save-as/browser-detect.service.js'; +import './app/notebook/elastic-input/elastic-input.controller.js'; +import './app/notebook/dropdown-input/dropdown-input.directive.js'; +import './app/notebook/note-var-share.service.js'; +import './app/notebook-repository/notebook-repository.controller.js'; +import './app/search/result-list.controller.js'; +import './app/search/search.service.js'; +import './app/helium'; +import './app/helium/helium.service.js'; +import './app/notebook/dynamic-forms/dynamic-forms.directive.js'; +import './components/array-ordering/array-ordering.service.js'; +import './components/navbar/navbar.controller.js'; +import './components/navbar/expand-collapse/expand-collapse.directive.js'; +import './components/note-create/note-create.controller.js'; +import './components/note-create/visible.directive.js'; +import './components/note-import/note-import.controller.js'; +import './components/ng-enter/ng-enter.directive.js'; +import './components/ng-escape/ng-escape.directive.js'; +import './components/websocket/websocket-message.service.js'; +import './components/websocket/websocket-event.factory.js'; +import './components/note-list/note-list.factory.js'; +import './components/base-url/base-url.service.js'; +import './components/login/login.controller.js'; +import './components/note-action/note-action.service.js'; +import './components/note-rename/note-rename.controller.js'; +import './components/note-rename/note-rename.service.js'; From 0660164379c12aa48321d0772598f61a56904a17 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Sat, 24 Feb 2018 14:25:38 +0800 Subject: [PATCH 047/386] [HOTFIX] rename zeppelin.ipython.grpc.framesize to zeppelin.ipython.grpc.message_size ### What is this PR for? trivial change for property name renaming. Followup of #2802 ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2813 from zjffdu/HOTFIX_GRPC and squashes the following commits: d899a29 [Jeff Zhang] [HOTFIX] rename zeppelin.ipython.grpc.framesize to zeppelin.ipython.grpc.message_size --- .../java/org/apache/zeppelin/python/IPythonInterpreter.java | 4 ++-- python/src/main/resources/interpreter-setting.json | 6 +++--- .../org/apache/zeppelin/python/IPythonInterpreterTest.java | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java index 10bf530b4ae..5c5bfe39500 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java @@ -143,10 +143,10 @@ public void open() throws InterpreterException { int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); LOGGER.info("Launching IPython Kernel at port: " + ipythonPort); LOGGER.info("Launching JVM Gateway at port: " + jvmGatewayPort); - int framesize = Integer.parseInt(getProperty("zeppelin.ipython.grpc.framesize", + int message_size = Integer.parseInt(getProperty("zeppelin.ipython.grpc.message_size", 32 * 1024 * 1024 + "")); ipythonClient = new IPythonClient(ManagedChannelBuilder.forAddress("127.0.0.1", ipythonPort) - .usePlaintext(true).maxInboundMessageSize(framesize)); + .usePlaintext(true).maxInboundMessageSize(message_size)); launchIPythonKernel(ipythonPort); setupJVMGateway(jvmGatewayPort); } catch (Exception e) { diff --git a/python/src/main/resources/interpreter-setting.json b/python/src/main/resources/interpreter-setting.json index 3257e58abfb..f36add32aa3 100644 --- a/python/src/main/resources/interpreter-setting.json +++ b/python/src/main/resources/interpreter-setting.json @@ -41,10 +41,10 @@ "description": "time out for ipython launch", "type": "number" }, - "zeppelin.ipython.grpc.framesize": { - "propertyName": "zeppelin.ipython.grpc.framesize", + "zeppelin.ipython.grpc.message_size": { + "propertyName": "zeppelin.ipython.grpc.message_size", "defaultValue": "33554432", - "description": "grpc framesize, default is 32M", + "description": "grpc message size, default is 32M", "type": "number" } }, diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index dfc8c36b74a..480cae311cd 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -78,7 +78,7 @@ public void testIPython() throws IOException, InterruptedException, InterpreterE @Test public void testGrpcFrameSize() throws InterpreterException, IOException { Properties properties = new Properties(); - properties.setProperty("zeppelin.ipython.grpc.framesize", "4"); + properties.setProperty("zeppelin.ipython.grpc.message_size", "4"); startInterpreter(properties); // to make this test can run under both python2 and python3 @@ -99,7 +99,7 @@ public void testGrpcFrameSize() throws InterpreterException, IOException { close(); // increase framesize to make it work - properties.setProperty("zeppelin.ipython.grpc.framesize", "40"); + properties.setProperty("zeppelin.ipython.grpc.message_size", "40"); startInterpreter(properties); // to make this test can run under both python2 and python3 result = interpreter.interpret("from __future__ import print_function", getInterpreterContext()); From 720537aab99229309fb7b00c3f6006f0fea4a779 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Fri, 23 Feb 2018 10:43:12 +0530 Subject: [PATCH 048/386] [ZEPPELIN-3249] Add support for streaming table ### What is this PR for? Since Zeppelin support streaming from various backends, I think it will be useful if even tables (and later graphs) can also be streamed. ### What type of PR is it? [Improvement | Feature] ### Todos * [x] - At times it fails with `Uncaught TypeError: Cannot read property 'p20180220_113300_1663645286_0_table_gridApi' of undefined`, have to fix this error. ### What is the Jira issue? * [ZEPPELIN-3249](https://issues.apache.org/jira/browse/ZEPPELIN-3249) ### How should this be tested? I have done it using shell interpreter, but this should work for all other backends as well ``` %sh echo "%table" echo "Col1 Col2" echo "1 2" sleep 1 echo "3 4" echo "5 6" sleep 2 echo "7 8" sleep 3 echo "9 10" echo "11 12" sleep 4 echo "12 13" ``` ### Screenshots (if appropriate) ![zeppelin-3249](https://user-images.githubusercontent.com/674497/36419469-fa4c26e0-1657-11e8-8a8a-717b29d91771.gif) ### Questions: * Does the licenses files need update? N/A * Is there breaking changes for older versions? N/A * Does this needs documentation? N/A Author: Prabhjyot Singh Closes #2809 from prabhjyotsingh/ZEPPELIN-3249 and squashes the following commits: 463599d [Prabhjyot Singh] Merge remote-tracking branch 'origin/master' into ZEPPELIN-3249 568f334 [Prabhjyot Singh] fallback option for persistedTableOption don't commit viz if paragraph is in running or pending state e6bce80 [Prabhjyot Singh] fix undefined 6ce5d18 [Prabhjyot Singh] concat not required all time 21564fc [Prabhjyot Singh] Add support for streaming table --- .../InterpreterResultMessageOutput.java | 2 +- .../src/app/notebook/notebook.controller.js | 23 ++-- .../paragraph/result/result.controller.js | 100 +++++++++++++----- .../builtins/visualization-table.js | 40 ++++++- .../builtins/visualization-util.js | 11 +- 5 files changed, 131 insertions(+), 45 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutput.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutput.java index da31364522a..8758c98e9d0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutput.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResultMessageOutput.java @@ -232,7 +232,7 @@ public void flush() throws IOException { } public boolean isAppendSupported() { - return type == InterpreterResult.Type.TEXT; + return type == InterpreterResult.Type.TEXT || type == InterpreterResult.Type.TABLE; } private void copyStream(InputStream in, OutputStream out) throws IOException { diff --git a/zeppelin-web/src/app/notebook/notebook.controller.js b/zeppelin-web/src/app/notebook/notebook.controller.js index 05ab9fb7992..4c9de9cac70 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.js @@ -278,16 +278,19 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope, $scope.$on('listRevisionHistory', function(event, data) { console.debug('received list of revisions %o', data); $scope.noteRevisions = data.revisionList; - if ($scope.noteRevisions.length === 0 || $scope.noteRevisions[0].id !== 'Head') { - $scope.noteRevisions.splice(0, 0, { - id: 'Head', - message: 'Head', - }); - } - if ($routeParams.revisionId) { - let index = _.findIndex($scope.noteRevisions, {'id': $routeParams.revisionId}); - if (index > -1) { - $scope.currentRevision = $scope.noteRevisions[index].message; + if ($scope.noteRevisions) { + if ($scope.noteRevisions.length === 0 || $scope.noteRevisions[0].id !== 'Head') { + $scope.noteRevisions.splice(0, 0, { + id: 'Head', + message: 'Head', + }); + } + if ($routeParams.revisionId) { + let index = _.findIndex($scope.noteRevisions, + {'id': $routeParams.revisionId}); + if (index > -1) { + $scope.currentRevision = $scope.noteRevisions[index].message; + } } } }); diff --git a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js index 5bf77dcd71e..29465e5bd5c 100644 --- a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js @@ -243,10 +243,22 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location if (paragraph.id === data.paragraphId && resultIndex === data.index && (paragraph.status === ParagraphStatus.PENDING || paragraph.status === ParagraphStatus.RUNNING)) { - if (DefaultDisplayType.TEXT !== $scope.type) { + // Check if result type is eiter TEXT or TABLE, if not then treat it like TEXT + if ([DefaultDisplayType.TEXT, DefaultDisplayType.TABLE].indexOf($scope.type) < 0) { $scope.type = DefaultDisplayType.TEXT; } - appendTextOutput(data.data); + if ($scope.type === DefaultDisplayType.TEXT) { + appendTextOutput(data.data); + } else if ($scope.type === DefaultDisplayType.TABLE) { + appendTableOutput(data); + } + } + if (paragraph.id === data.paragraphId && + resultIndex === data.index && + paragraph.status === ParagraphStatus.FINISHED) { + if ($scope.type === DefaultDisplayType.TABLE) { + appendTableOutput(data); + } } }); @@ -531,6 +543,39 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location } }; + function appendTableOutput(data) { + if (!$scope.$parent.result.data) { + $scope.$parent.result.data = []; + tableData = undefined; + } + if (!$scope.$parent.result.data[data.index]) { + $scope.$parent.result.data[data.index] = ''; + } + if (!tableData) { + $scope.$parent.result.data[data.index] = $scope.$parent.result.data[data.index].concat(data.data); + $rootScope.$broadcast( + 'updateResult', + {'data': $scope.$parent.result.data[data.index], 'type': 'TABLE'}, + undefined, + paragraph, + data.index); + let elemId = `p${$scope.id}_table`; + renderGraph(elemId, 'table', true); + } else { + let textRows = data.data.split('\n'); + for (let i = 0; i < textRows.length; i++) { + if (textRows[i] !== '') { + let row = textRows[i].split('\t'); + tableData.rows.push(row); + let builtInViz = builtInVisualizations['table']; + if (builtInViz.instance !== undefined) { + builtInViz.instance.append([row], tableData.columns); + } + } + } + } + } + function appendTextOutput(data) { const elemId = getTextResultElemId($scope.id); textResultQueueForAppend.push(data); @@ -744,33 +789,32 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location }; const commitVizConfigChange = function(config, vizId) { - let newConfig = angular.copy($scope.config); - if (!newConfig.graph) { - newConfig.graph = {}; - } - - // copy setting for vizId - if (!newConfig.graph.setting) { - newConfig.graph.setting = {}; - } - newConfig.graph.setting[vizId] = angular.copy(config); - - // copy common setting - if (newConfig.graph.setting[vizId]) { - newConfig.graph.commonSetting = newConfig.graph.setting[vizId].common; - delete newConfig.graph.setting[vizId].common; - } - - // copy pivot setting - if (newConfig.graph.commonSetting && newConfig.graph.commonSetting.pivot) { - newConfig.graph.keys = newConfig.graph.commonSetting.pivot.keys; - newConfig.graph.groups = newConfig.graph.commonSetting.pivot.groups; - newConfig.graph.values = newConfig.graph.commonSetting.pivot.values; - delete newConfig.graph.commonSetting.pivot; + if ([ParagraphStatus.RUNNING, ParagraphStatus.PENDING].indexOf(paragraph.status) < 0) { + let newConfig = angular.copy($scope.config); + if (!newConfig.graph) { + newConfig.graph = {}; + } + // copy setting for vizId + if (!newConfig.graph.setting) { + newConfig.graph.setting = {}; + } + newConfig.graph.setting[vizId] = angular.copy(config); + // copy common setting + if (newConfig.graph.setting[vizId]) { + newConfig.graph.commonSetting = newConfig.graph.setting[vizId].common; + delete newConfig.graph.setting[vizId].common; + } + // copy pivot setting + if (newConfig.graph.commonSetting && newConfig.graph.commonSetting.pivot) { + newConfig.graph.keys = newConfig.graph.commonSetting.pivot.keys; + newConfig.graph.groups = newConfig.graph.commonSetting.pivot.groups; + newConfig.graph.values = newConfig.graph.commonSetting.pivot.values; + delete newConfig.graph.commonSetting.pivot; + } + console.debug('committVizConfig', newConfig); + let newParams = angular.copy(paragraph.settings.params); + commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams); } - console.debug('committVizConfig', newConfig); - let newParams = angular.copy(paragraph.settings.params); - commitParagraphResult(paragraph.title, paragraph.text, newConfig, newParams); }; $scope.$on('paragraphResized', function(event, paragraphId) { diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-table.js b/zeppelin-web/src/app/visualization/builtins/visualization-table.js index d77efbc805a..723bb3aca13 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-table.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-table.js @@ -16,12 +16,19 @@ import Visualization from '../visualization'; import PassthroughTransformation from '../../tabledata/passthrough'; import { - Widget, ValueType, - isInputWidget, isOptionWidget, isCheckboxWidget, - isTextareaWidget, isBtnGroupWidget, - initializeTableConfig, resetTableOptionConfig, - DefaultTableColumnType, TableColumnType, updateColumnTypeState, + DefaultTableColumnType, + initializeTableConfig, + isBtnGroupWidget, + isCheckboxWidget, + isInputWidget, + isOptionWidget, + isTextareaWidget, parseTableOption, + resetTableOptionConfig, + TableColumnType, + updateColumnTypeState, + ValueType, + Widget, } from './visualization-util'; const SETTING_TEMPLATE = require('./visualization-table-setting.html'); @@ -247,6 +254,29 @@ export default class TableVisualization extends Visualization { gridOptions.enableSelectionBatchEvent = false; } + append(row, columns) { + const gridOptions = this.getGridOptions(); + this.setDynamicGridOptions(gridOptions, this.config); + // this.refreshGrid() + const gridElemId = this.getGridElemId(); + const gridElem = angular.element(`#${gridElemId}`); + + if (gridElem) { + const scope = this.getScope(); + + const columnNames = columns.map((c) => c.name); + let gridData = row.map((r) => { + return columnNames.reduce((acc, colName, index) => { + acc[colName] = r[index]; + return acc; + }, {}); + }); + gridData.map((data) => { + scope[gridElemId].data.push(data); + }); + } + } + render(tableData) { const gridElemId = this.getGridElemId(); let gridElem = document.getElementById(gridElemId); diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-util.js b/zeppelin-web/src/app/visualization/builtins/visualization-util.js index a82a18ecceb..7feb129bc0e 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-util.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-util.js @@ -100,7 +100,16 @@ export function initializeTableConfig(config, tableOptionSpecs) { export function parseTableOption(specs, persistedTableOption) { /** copy original params */ - const parsed = JSON.parse(JSON.stringify(persistedTableOption)); + let parsed; + try { + parsed = JSON.parse(JSON.stringify(persistedTableOption)); + } catch (e) { + // if not able to parse fall back to default values coming from specs + parsed = {}; + for (let spec of specs) { + parsed[spec['name']] = spec['defaultValue']; + } + } for (let i = 0; i < specs.length; i++) { const s = specs[i]; From 75ada89f4d36a16713f18c6f6fc54dab4f659ef5 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 26 Feb 2018 10:18:31 +0800 Subject: [PATCH 049/386] ZEPPELIN-3246. Need option for automatically restart the livy interpreter automatically as zeppelin does not start new Livy session if yarn livy session application is killed ### What is this PR for? Add one new property `zeppelin.livy.restart_dead_session` to allow livy session to be created automatically. By default it is false, because there's many reason that session is dead, it is encouraged to ask user to check why it is dead and restart interpreter by themselves. ### What type of PR is it? [Feature] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3246 ### How should this be tested? * Manually tested. ### Screenshots (if appropriate) ![image](https://user-images.githubusercontent.com/164491/36651529-4a22ab36-1ae4-11e8-88dc-23a814e4c1b1.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2818 from zjffdu/ZPPELIN-3246 and squashes the following commits: 89d44f6 [Jeff Zhang] ZEPPELIN-3246. Need option for automatically restart the livy interpreter automatically as zeppelin does not start new Livy session if yarn livy session application is killed --- .../zeppelin/livy/BaseLivyInterpreter.java | 64 +++++++++++++------ .../apache/zeppelin/livy/LivyException.java | 4 +- .../livy/LivyPySparkBaseInterpreter.java | 4 +- .../zeppelin/livy/LivySharedInterpreter.java | 2 +- .../zeppelin/livy/LivySparkInterpreter.java | 6 +- .../zeppelin/livy/SessionDeadException.java | 22 +++++++ .../main/resources/interpreter-setting.json | 6 ++ .../interpreter/InterpreterException.java | 9 ++- 8 files changed, 90 insertions(+), 27 deletions(-) create mode 100644 livy/src/main/java/org/apache/zeppelin/livy/SessionDeadException.java diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index 724a4b36c7c..5fe7ce40c9b 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -93,6 +93,7 @@ public abstract class BaseLivyInterpreter extends Interpreter { private int sessionCreationTimeout; private int pullStatusInterval; protected boolean displayAppInfo; + private boolean restartDeadSession; protected LivyVersion livyVersion; private RestTemplate restTemplate; private Map customHeaders = new HashMap<>(); @@ -110,6 +111,8 @@ public BaseLivyInterpreter(Properties property) { this.livyURL = property.getProperty("zeppelin.livy.url"); this.displayAppInfo = Boolean.parseBoolean( property.getProperty("zeppelin.livy.displayAppInfo", "true")); + this.restartDeadSession = Boolean.parseBoolean( + property.getProperty("zeppelin.livy.restart_dead_session", "false")); this.sessionCreationTimeout = Integer.parseInt( property.getProperty("zeppelin.livy.session.create_timeout", 120 + "")); this.pullStatusInterval = Integer.parseInt( @@ -159,7 +162,7 @@ public void open() throws InterpreterException { } catch (LivyException e) { String msg = "Fail to create session, please check livy interpreter log and " + "livy server log"; - throw new RuntimeException(msg, e); + throw new InterpreterException(msg, e); } } @@ -246,7 +249,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) { } try { - return interpret(st, null, context.getParagraphId(), this.displayAppInfo, true); + return interpret(st, null, context.getParagraphId(), this.displayAppInfo, true, true); } catch (LivyException e) { LOGGER.error("Fail to interpret:" + st, e); return new InterpreterResult(InterpreterResult.Code.ERROR, @@ -359,18 +362,21 @@ private SessionInfo getSessionInfo(int sessionId) throws LivyException { public InterpreterResult interpret(String code, String paragraphId, boolean displayAppInfo, - boolean appendSessionExpired) throws LivyException { + boolean appendSessionExpired, + boolean appendSessionDead) throws LivyException { return interpret(code, sharedInterpreter.isSupported() ? getSessionKind() : null, - paragraphId, displayAppInfo, appendSessionExpired); + paragraphId, displayAppInfo, appendSessionExpired, appendSessionDead); } public InterpreterResult interpret(String code, String codeType, String paragraphId, boolean displayAppInfo, - boolean appendSessionExpired) throws LivyException { + boolean appendSessionExpired, + boolean appendSessionDead) throws LivyException { StatementInfo stmtInfo = null; boolean sessionExpired = false; + boolean sessionDead = false; try { try { stmtInfo = executeStatement(new ExecuteRequest(code, codeType)); @@ -386,6 +392,21 @@ public InterpreterResult interpret(String code, } } stmtInfo = executeStatement(new ExecuteRequest(code, codeType)); + } catch (SessionDeadException e) { + sessionDead = true; + if (restartDeadSession) { + LOGGER.warn("Livy session {} is dead, new session will be created.", sessionInfo.id); + close(); + try { + open(); + } catch (InterpreterException ie) { + throw new LivyException("Fail to restart livy session", ie); + } + stmtInfo = executeStatement(new ExecuteRequest(code, codeType)); + } else { + throw new LivyException("%html Livy session is dead somehow, " + + "please check log to see why it is dead, and then restart livy interpreter"); + } } // pull the statement status @@ -405,9 +426,9 @@ public InterpreterResult interpret(String code, paragraphId2StmtProgressMap.put(paragraphId, (int) (stmtInfo.progress * 100)); } } - if (appendSessionExpired) { - return appendSessionExpire(getResultFromStatementInfo(stmtInfo, displayAppInfo), - sessionExpired); + if (appendSessionExpired || appendSessionDead) { + return appendSessionExpireDead(getResultFromStatementInfo(stmtInfo, displayAppInfo), + sessionExpired, sessionDead); } else { return getResultFromStatementInfo(stmtInfo, displayAppInfo); } @@ -451,21 +472,27 @@ private boolean isSessionExpired() throws LivyException { } } - private InterpreterResult appendSessionExpire(InterpreterResult result, boolean sessionExpired) { + private InterpreterResult appendSessionExpireDead(InterpreterResult result, + boolean sessionExpired, + boolean sessionDead) { + InterpreterResult result2 = new InterpreterResult(result.code()); if (sessionExpired) { - InterpreterResult result2 = new InterpreterResult(result.code()); result2.add(InterpreterResult.Type.HTML, "Previous livy session is expired, new livy session is created. " + "Paragraphs that depend on this paragraph need to be re-executed!"); - for (InterpreterResultMessage message : result.message()) { - result2.add(message.getType(), message.getData()); - } - return result2; - } else { - return result; + + } + if (sessionDead) { + result2.add(InterpreterResult.Type.HTML, + "Previous livy session is dead, new livy session is created. " + + "Paragraphs that depend on this paragraph need to be re-executed!"); } - } + for (InterpreterResultMessage message : result.message()) { + result2.add(message.getType(), message.getData()); + } + return result2; + } private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo, boolean displayAppInfo) { @@ -684,8 +711,7 @@ private String callRestAPI(String targetURL, String method, String jsonData) HttpServerErrorException errorException = (HttpServerErrorException) e; String errorResponse = errorException.getResponseBodyAsString(); if (errorResponse.contains("Session is in state dead")) { - throw new LivyException("%html Livy session is dead somehow, " + - "please check log to see why it is dead, and then restart livy interpreter"); + throw new SessionDeadException(); } throw new LivyException(errorResponse, e); } diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyException.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyException.java index 5adffd4dc1a..e126a0f0bb4 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivyException.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyException.java @@ -17,10 +17,12 @@ package org.apache.zeppelin.livy; +import org.apache.zeppelin.interpreter.InterpreterException; + /** * Livy api related exception */ -public class LivyException extends Exception { +public class LivyException extends InterpreterException { public LivyException() { } diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkBaseInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkBaseInterpreter.java index 17b20e3634f..6d399814a2e 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkBaseInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkBaseInterpreter.java @@ -32,7 +32,7 @@ public LivyPySparkBaseInterpreter(Properties property) { @Override protected String extractAppId() throws LivyException { return extractStatementResult( - interpret("sc.applicationId", null, false, false).message() + interpret("sc.applicationId", null, false, false, false).message() .get(0).getData()); } @@ -40,7 +40,7 @@ protected String extractAppId() throws LivyException { protected String extractWebUIAddress() throws LivyException { return extractStatementResult( interpret( - "sc._jsc.sc().ui().get().appUIAddress()", null, false, false) + "sc._jsc.sc().ui().get().appUIAddress()", null, false, false, false) .message().get(0).getData()); } diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySharedInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySharedInterpreter.java index 77e288bfa65..cef08582a40 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySharedInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySharedInterpreter.java @@ -78,7 +78,7 @@ public InterpreterResult interpret(String st, String codeType, InterpreterContex } try { - return interpret(st, codeType, context.getParagraphId(), this.displayAppInfo, true); + return interpret(st, codeType, context.getParagraphId(), this.displayAppInfo, true, true); } catch (LivyException e) { LOGGER.error("Fail to interpret:" + st, e); return new InterpreterResult(InterpreterResult.Code.ERROR, diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java index 066d0da8cc9..ad62e9b5de5 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java @@ -36,7 +36,7 @@ public String getSessionKind() { @Override protected String extractAppId() throws LivyException { return extractStatementResult( - interpret("sc.applicationId", null, false, false).message() + interpret("sc.applicationId", null, false, false, false).message() .get(0).getData()); } @@ -45,10 +45,10 @@ protected String extractWebUIAddress() throws LivyException { interpret( "val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get", null, - null, false, false); + null, false, false, false); return extractStatementResult( interpret( - "webui.getClass.getMethod(\"appUIAddress\").invoke(webui)", null, false, false) + "webui.getClass.getMethod(\"appUIAddress\").invoke(webui)", null, false, false, false) .message().get(0).getData()); } diff --git a/livy/src/main/java/org/apache/zeppelin/livy/SessionDeadException.java b/livy/src/main/java/org/apache/zeppelin/livy/SessionDeadException.java new file mode 100644 index 00000000000..58117907628 --- /dev/null +++ b/livy/src/main/java/org/apache/zeppelin/livy/SessionDeadException.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.livy; + +public class SessionDeadException extends LivyException { + +} diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json index cecacac2e48..d096c467ea8 100644 --- a/livy/src/main/resources/interpreter-setting.json +++ b/livy/src/main/resources/interpreter-setting.json @@ -108,6 +108,12 @@ "defaultValue": "true", "description": "Whether display app info", "type": "checkbox" + }, + "zeppelin.livy.restart_dead_session": { + "propertyName": "zeppelin.livy.restart_dead_session", + "defaultValue": "false", + "description": "Whether restart a dead session", + "type": "checkbox" } }, "option": { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterException.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterException.java index 8b8a2297658..1ce63f3b78f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterException.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterException.java @@ -19,11 +19,14 @@ /** - * Runtime Exception for interpreters. + * General Exception for interpreters. * */ public class InterpreterException extends Exception { + public InterpreterException() { + } + public InterpreterException(Throwable e) { super(e); } @@ -36,4 +39,8 @@ public InterpreterException(String msg, Throwable t) { super(msg, t); } + public InterpreterException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } } From fc44693fe6589d9fbcb8a63a4664369bb7e2a2a3 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 26 Feb 2018 16:58:53 +0800 Subject: [PATCH 050/386] ZEPPELIN-3265. DevInterpreter doesn't work ### What is this PR for? This PR is trying the fix the bug that DevInterpreter doesn't work due the interpreter code refactoring. ### What type of PR is it? [Bug Fix ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3265 ### How should this be tested? * Manually verify the zeppelin clock example. ### Screenshots (if appropriate) ![jietu20180226-210416-hd](https://user-images.githubusercontent.com/164491/36671998-ae453166-1b38-11e8-92da-3c812061251a.gif) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2819 from zjffdu/ZEPPELIN-3265 and squashes the following commits: 0e85992 [Jeff Zhang] ZEPPELIN-3265. DevInterpreter doesn't work --- helium-dev/pom.xml | 2 +- .../interpreter/InterpreterSettingManager.java | 3 ++- .../interpreter/ManagedInterpreterGroup.java | 18 +++++++----------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/helium-dev/pom.xml b/helium-dev/pom.xml index 77c59791649..559b411dd50 100644 --- a/helium-dev/pom.xml +++ b/helium-dev/pom.xml @@ -34,7 +34,7 @@ Zeppelin: Helium development interpreter - helium-dev + dev diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 04d409289a3..0601c6ff5b4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -403,13 +403,14 @@ private void registerInterpreterSetting(List registeredIn .setIntepreterSettingManager(this) .create(); - LOGGER.info("Register InterpreterSettingTemplate & InterpreterSetting: {}", + LOGGER.info("Register InterpreterSettingTemplate & Create InterpreterSetting: {}", interpreterSettingTemplate.getName()); interpreterSettingTemplates.put(interpreterSettingTemplate.getName(), interpreterSettingTemplate); InterpreterSetting interpreterSetting = new InterpreterSetting(interpreterSettingTemplate); initInterpreterSetting(interpreterSetting); + interpreterSettings.put(interpreterSetting.getName(), interpreterSetting); } @VisibleForTesting diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java index d21a34d57be..e19c9caeaa0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java @@ -62,17 +62,13 @@ public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess(Strin LOGGER.info("Create InterpreterProcess for InterpreterGroup: " + getId()); remoteInterpreterProcess = interpreterSetting.createInterpreterProcess(id, userName, properties); - synchronized (remoteInterpreterProcess) { - if (!remoteInterpreterProcess.isRunning()) { - remoteInterpreterProcess.start(userName); - remoteInterpreterProcess.getRemoteInterpreterEventPoller() - .setInterpreterProcess(remoteInterpreterProcess); - remoteInterpreterProcess.getRemoteInterpreterEventPoller().setInterpreterGroup(this); - remoteInterpreterProcess.getRemoteInterpreterEventPoller().start(); - getInterpreterSetting().getRecoveryStorage() - .onInterpreterClientStart(remoteInterpreterProcess); - } - } + remoteInterpreterProcess.start(userName); + remoteInterpreterProcess.getRemoteInterpreterEventPoller() + .setInterpreterProcess(remoteInterpreterProcess); + remoteInterpreterProcess.getRemoteInterpreterEventPoller().setInterpreterGroup(this); + remoteInterpreterProcess.getRemoteInterpreterEventPoller().start(); + getInterpreterSetting().getRecoveryStorage() + .onInterpreterClientStart(remoteInterpreterProcess); } return remoteInterpreterProcess; } From 64bbba4796fe1ddfd1ca1facde7dcda33ac86ef7 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Sun, 25 Feb 2018 21:24:08 +0800 Subject: [PATCH 051/386] ZEPPELIN-3255. Can not run spark1 and spark2 in one zeppelin instance ### What is this PR for? Although #2750 enable the support of spark 2.3, it breaks the support of spark 1.6. Users have to build zeppelin against spark 1.6 to make zeppelin work with that. But previous one zeppelin instance can work with multiple versions of spark. This PR introduce spark shims module which is to resolve the api incompatible issue between different versions of spark, so that one zeppelin instance can work with multiple versions of spark. ### What type of PR is it? [ Improvement | Refactoring] ### Todos * https://issues.apache.org/jira/browse/ZEPPELIN-3254 Although zeppelin should support to run multiple versions of spark in one instance, but our travis test doesn't cover it, ZEPPELIN-3254 would do that. ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3255 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2814 from zjffdu/ZEPPELIN-3255 and squashes the following commits: 68fa437 [Jeff Zhang] Remove akka from spark-dependencies 14fef45 [Jeff Zhang] ZEPPELIN-3255. Can not run spark1 and spark2 in one zeppelin instance --- spark/interpreter/figure/null-1.png | Bin 13599 -> 0 bytes spark/interpreter/pom.xml | 12 + .../zeppelin/spark/NewSparkInterpreter.java | 67 +---- .../zeppelin/spark/OldSparkInterpreter.java | 235 +----------------- .../spark/OldSparkInterpreterTest.java | 10 +- spark/pom.xml | 41 +-- spark/spark-dependencies/pom.xml | 41 +-- spark/spark-shims/pom.xml | 70 ++++++ .../org/apache/zeppelin/spark/SparkShims.java | 110 ++++++++ spark/spark1-shims/pom.xml | 89 +++++++ .../apache/zeppelin/spark/Spark1Shims.java | 57 +++++ spark/spark2-shims/pom.xml | 88 +++++++ .../apache/zeppelin/spark/Spark2Shims.java | 36 +++ 13 files changed, 489 insertions(+), 367 deletions(-) delete mode 100644 spark/interpreter/figure/null-1.png create mode 100644 spark/spark-shims/pom.xml create mode 100644 spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java create mode 100644 spark/spark1-shims/pom.xml create mode 100644 spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java create mode 100644 spark/spark2-shims/pom.xml create mode 100644 spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java diff --git a/spark/interpreter/figure/null-1.png b/spark/interpreter/figure/null-1.png deleted file mode 100644 index 8b1ce07ea9e7d0f24bae214f3bda98a7787ee662..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13599 zcmeHuc{r5s+kUnvE&A9}7^RZLR1#T+7F4z(DcclD$i6dUNTsYvS;7noNl3PAGnC;| zLX09igE1Im7|fXMea!TEx8wce{p0=p@%d~4XmFL?=($hf z&|g1>*{G5lBRNS0UyxJ9^c&H~Nqmci*6l%jVe82t{aIW2Ri~Mn;|49@u#e-;eV_dy z5eaU0CXY_w9t&Apxj#-5OZtYT8Es=6fBzGjW9B~hc4jKiDKl_5r`(5pi;ePZzWZ?5 z^K)TW1`6_ke9NApqH&Ngex=3a?K6}SQ)I%!6J9MlHkg{3yi0%c)1i@8KRsvv{Wmn4{;{+2$n6-&eW?$7n2GPCa4wYMhxX3m`VmJp`SlOUDE##+ zy36sz#noir6MXwaV|C3|Q)EAg-zKm3U%F@LsH^#t)ex_4S&(Tse7dqe=O#z}h^f6* z-@Btbsvjw~D>H-~8kw!oa(m)Y%`4RtGTC)DopdODj(K8Lh3Wo!3^8d#sqt&G{Z{|M z^z;0^>;A#kdptMer53w0uurgZ%f6IJo1I1xsb4Y(WmE^<8{TR=jNYXQt-I5&$(Sdc ze@Bg`##A0{RDvI7wKjIZzrs7!x=PP$goGNUE_PgDCeh| zv8}DI5`CU4%U$y}(?XvSk3VqkcKq{GSJyt;Ea&cjVKHUtCT68$C0(22R^+ZhMDhut`sf)1I z6CLYan_tDOp9qOy^%93#{NE8&BdauZvN5CNfS(WpPVpw!1O#6W3TA!zCfj?{rehr! z&!4#(!bck3{$9=nuD?h?j~-ZiB$gsptYIkQxl84++?D71kK8XAkd7i&MuS!H{o2~m z+pFXXSaJP5&np$*;JQ1tvWuKzuG*S ze2B58C?`IpLx(ICQlk(jd>pOvG_v{GJ>zQX(2uNhs*7N)R|bjE~MUzAk`CPHcdK)Q00nN-aG3E3c_?+&wzQOe13kk zrFjB(rJ{mFL})?r(T==zx+EkB%g@@Q$X!E{nAjp|w{RnGMPC5xcImW+5qG%;_9=`F z9^VmmhC4bUDYp2+&pm!)K}9Oh88p>6%Tn)iM^}$1jHyiZ?%uc}b~h$6(z{J-Mu5-p z=H})~Cv2=lwI}k?qerm|+jrK4|Jc#h|9AbdRfikKuTZAN!`khnh+DUE2Og<-l?*L7 z%iUd;KvK$PN$m2V-2<#6v;)SoO;G|^^+SgaZEaa%r`#*}8Dn8?UZ6(xhrx8s%*-|b zYzs%;EAMY|7I=w7Mtra+S)r5*{8iwvn*l_I?ymIoYc)sO%;ERe{QdoZXuQnLUutr` zbLY^xxgQ!bWfsoP&M(SdC5JX1A~&yD9#7&TCcxgHDHygX5XBu){}Nkly!wp?NI7Y- zMLCecuo4%ZjR2^@lFK|yw*kOy6TZB*Wdp|sU;s=eSX~~ne1QRluNTf6nHV(vnd;91 zDt!MxITjx_GX-|)K9z?pHY)~V0G9r`exa_Irm3M}{Y^h)@n#)9QO9y96}K>4k4u!( z>0vTqr&U&722Z9%@Un`a)Rx<#G{!X*Ti^Ag4x+CzFVyG%xkNW+b zAr0z_l_W0Ilg%?Vz0E1Azo#najBo=xM$qpW^&VFUby6N zMn=ZPJ}3enHZBScd5tAUtko0eE4s|3p)g-@t)!Yf~89fS1E|MMp=w-Mp!g5FanGHpm?P{{8aSvTBR&E^%|M zeR2@k#~%&}zDaIQPI)V=_!X*5SBOzDLneE!M|%&8^mVoOMMUY1iURdPxU?h6Dg13% zJt{jp!J6|?*`x_%Gu164?HF>aZtT8jYy!D;ueYh@D9k@r`lFU#lJbZ*XzyN?2sKV- zW~QB=qwExB&VhVH9_@>7bt3Frh)+tA_Dmx$r;(~CLD*PHVzRNkn{%JkigM7KJk;?+ zo=7MBla<->YqV;r-nz`3_xxfteD*8wSn)-ogH|@C&V>+1<0Q83)xCXvujCif{ISjAk|IlFLF<`*kmI}C<>!1BY61tGUU3^JewXkbE zTo5*Bh8OyxZAzeTVcLOS>q`< zy&Y;^e68fjNL*6mr}W@K&l<^F@O5ILbm6r!;Ium>9si`G9YM@bD{v-Mg^H~taF57I z)nkh@NQ)Bhx2^KmY6~6In10hQJX<_N1wn-d6Nl9xIen|j=D3AX42CppI`^|g1p*ou zLBGt6RissU^hu$P7ZqS(B>^sS2@$Qm4@pHT1-=D=B_$<$hS1-CeV0*1 zMTO_IX}Q0&Ge}1fAdoRJR_V6J#e4JnQ=Ob_v)$$&$m@n2>*(kJW_$$9=r3Jn(cSm9 zuiJ54{+oHX>-AqJFLKfLQc7&mEGcrGTev7hM@DuYliPgN z?5_<}Mil@!#3|-LxHZz>3JPQDIquJXhZKdeOnJT)9x68jsN}Ow;T1)00LP<>yy1cT z1W?j#d-IY9Ycl}j1;z$E_V@IRL6fV&j|04F%+{buQ;&aEoIC9{etw9++e5s2;{f;* zcU|UD(Az*6D05lk&nx}^xJ&<;zzswGgRxnah1Lg?&2@t2FDx%FPkpv53pI`)3jf2e z45}kBWHh5^Dh-MRk}fjurXWkw#y)ug%~+@mU7kLvr|0m1Q@-qlVXamIdPgOocT~Ex zr%u#1POI8(4d!N)uTrx_~do?+B)n1GjWrPiUCDvc@9>SK86X&&;oB|_4>t^!eVo@=V9&JQ1 zF3Dtgvb_OO2aBn;xU~oGM_U}f7Oc^#$1bt2{jkE4m6febyry+&D(LFSwf=x9vp|>n zq`;Go9)sv%V}0k?=(bZ9l!52Y_4@0i`B!O(FSqHvg^(8`jFGj1zJ_THq5z0yu>KHA zdpPko!vo5&)h0JACwdWl5s43vP+qA6H754HybgKh(e(6mY@6QC1`35j9q2<>(^ddx z68NfxPgPIWrG(*YP=xCl-y9?sLDOpEQ7>pu*q z=>&R?F5p*`*KJp{ZPhsCikRC?ZMz27&})r`UGyXlEs2^%&~@!@Q1Vmar;UV@Rn3>l zL$&JJfj@5;UYFGYCl26eAc;%Cq=6TwyoRpJegWx#-CM8N&IdfV%G&iW8OOZU!yhIJ0o~+A-3n{$Lk5Rp=y8ZBJX6no8X2 zT^&2xb-3-!h0MYbxNW(U;nW~J-m1*)OC$#iWn!r*`t%f{3Y30ny^F)_3K)MVE#;Zz z_f@%Zz}9bM4$4^%AXW7h40_dC{c3~$5R$Z9REGt~V2yQkc~+>) zUSB);<~H-QvRJGzbpV3B@r=b-t7*(sh9GvK9{LDEJ1%3pr-51`t}Fd2&Ix^um)mAP zt@cXdXa*}Ijo~SZH23N+T8r2NTY?XWqPA)X;hdX zgUlKs=Yu0CaU_R5dOQh4aBCFOcoD>2jENKd)}l0$gdVyF6Sp_Ta2Obrxelpv!jXR3ylVM@Y(Rv>?%4|Xf^UVZGbL02RgBcz5lR8ymeQn1pX(EI;{ ziB-{~ba@yotzQ3$2S;IcA2(xaUCSZtg3)uNF*Gc8t%`%wXIX`p8MLqc&V{a!<&3sYKVC)^yPMqfj1Oc}61O&{IEnvIMX6cD zta@Nlua!9t=NW#N{QZE}qYjQnbEY*h86(sJP03MV*|g%u7OyK}wE~OvjG(6>YTqm+1~(`=1pRU$rrLQat3^~a^dQZCR{?)rUlM_o-nNNI zvHumNWsSM@`O#9T9dFv4hZ`XhmcH-`#JU?cvW!eq1A^K_3KhUbU)-y8O$@z*C<>*W z6_vi2_Iu|KT66=D=Y+)Ks8bS!ge2s|+V2CF0|)NL1R{+kQ~meG3!Ar*erA zH90bp{HnR~FpmsLh%M$n;JwUuV}QvdZN}yCr1M9p#4s z|6)U|==k=nGNRk)zY+HD?l?eQ|&p|1s0Txhb@jZ1Ad8(!z&rv*{_`r4G=?U zv7>SL!|-D|hNxRExIbk055MQemw%}$?2X3Qf0bL0GMb}>adN>!$rIICLFAF_p!xpf zX#%1BMp0jbB8Do!jhYJYFXU85qL4^)8;3ya^;sK?Ei-&3H>M|gCnUY}<3w!0+Hi1T zN2X?B$Tw2WD6-mX_+_;>@Okf-Wk7XbgQ3azj(=`+Go-sQJ{ZjyCKpt6L3A8o6fCht z+~VkwVV}?2iO)>bE)uf*&26t=KYW%|km+^F#xd08WmeYFvz+l5kojP>r;gXi?2lrr z`2+Uhq$)!@Fq~+D+aO;Pgz@q5lgExdHP;${5N~qT47U+seBM{H-W3iJb8~Y~9zT91 z9%tqC`jJvA|tfW_eoBJ9BN-ZA-0g9<=fF9#Y zB0W}8C5(+$Reyy7te$&e?@n(Dj32vK??%7hyBg8D8{n#dq}Nh;^UXY_e;U8(U5C}_RfiCGY+nzc zAuO#-_8)(%ItFf7{1nhEB>lR(yE_1%pjoiNiuzNOZL)T!12(fEOY}khG!{AKlI9RJ zdAj_wm{mhl8$XDJW+w|*&Bv5d)mP&|p_9ocb#w+#G-HRAlj0{x$Ws>O{(B03cF!6W z!IGVAvx#3{sKk-bvj%TVWAyJS`yDI*G;>hrT_T6FE|GC9N*XJ+myt=9sMI3Ubq13p)3c_kgCt)+E6o&{KPY^U}& zVk}t@$im9Xs{L+0vM`7nqi)xic{N_I^>fh%Fh5tr079=*E~;~o)zZ@9E1(Y>$^tBP z3S&p%S`zn(fBaA*u@ z()x#hWuCI)RsEHYIpTZ>2C1g7#IB}*G+0xpYFAw;Z!P?f=Kk3%c#--Dw1i;@rp zSpICS`}%e5yC$a`ZDp~XpWdvW${03j zPix$h(dRMSvI{0#rN7uDV95EcANG6P_}wET$hOE4H(pG040Eg7AQTqgQ9$Fc^6Gul z->KVKG+_Uj#p$W%)ZCr$!&qY4($#Oy;Af*;g@6xu9Yg4j6d&-xA)XzRFQhHGXabHM zg);Y^hs$Akc zqX<|v#?fw9xXOuvGT;(RSIerA16Zq!QBFj;2*uN%(nwK4hB(bhE3>OHq&|xsEgw4( zA+1$KgSyzj8N7*-6BGC?o&2jpNpJ2(m%G1a96HcN4h|6cSa8mU`E_YHcogo&T3;!_ zH1%`cuMN3Y=S0L}H0U*mo(wATdMfYf1~~ZG8n)cvU8oSO)(6DsZ?k;BBC{9BsTD(N zR%5Oa)2=mBqs=QB6S0hvhpIbiF6-$Xya8(|BIB{~)y;Tu~vqoE2_7?FtsoB0{trl-1y`i7kvS)!GL80CJcgDzWJ zl&91?wG{P!_9d%T&l%k-$C&W41MJwx@(T9KML6wJ7xfl5mPw_mu|&}N_n27y zxg{h8@29sQ0IV%r4bEOhSx{~yV`y=h3uSPU2WaiI{QJtLR8_kkzy$5lXXEvy5JBq= z1U+w}5{)GO*GKmH98MEWNua(3M%4Eu%@fcD z=f)(fcf6ZUN=G#wQJ+(blVB^b04($0Ma=s~s7^BY|mvGIRO<$h{ z1j{>s1MJhHdUoV|P6EI0Yq~39)s;9~T&7mq&94_pd=g|#+rbxj%OAi7z^2tv7ky<`(kJTn1KlLHj?orlzu>`-_mX{E z4!Yw5SURKhP9=k}eS))+w2gWEy`SB{h3?bMs+qT{`KB#ck9&X6Y>EPet zJE*I@&TIWe=0!##VttUm8-b4kjlloIwWbv5$0lD!V9OmA>?g=GkxeOp7jE(Lo4AM= z06hdCP-g($TuUP3@Gq+RgM_CIJhzW6NJvtpY z=K-@tg$1tqj9=LNTb~c8HPT1c(3&Ms41_B`L0kmqB7{fx5lPW$NI>Uw;Kzvuxc|`lKv|sVW zjZ{oG`s)zD?)!}sHjh~yRqTCD-$qG4D|TT zyZi9(wnmw6iBC*atGl+^+k4gbOOHos?V(1Jr?(0^oMi)eHdeAFZAFaquik#~D8+gF z`r>hL;&gjk3mHg8I}l`ufQA@(6yeQ{-b6TrQXcTQq! zbd46<7<6)IS9mO;m9Y*O2}@f(O2Hv^s^T-EQ6WjJnX>9E6bd{NoK)uO=pIYJYBnY$ zB&dd6hvDsDzaD|3ZCfUGM12KTed_fdUK>P*DDA1kvU83w?@0RrzQXCE%{=C{3?y#) j@$Uk>TwUA5;cq;ZUy9+JzytsIwCSRu`T5ebH}C%&jmAWl diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index e8d57a23f1e..c89cfa6ecbe 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -81,6 +81,18 @@ ${project.version} + + org.apache.zeppelin + spark1-shims + ${project.version} + + + + org.apache.zeppelin + spark2-shims + ${project.version} + + org.apache.zeppelin zeppelin-python diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java index 1d3ccd65fde..c8efa7a7d9f 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java @@ -69,6 +69,7 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter { private SparkVersion sparkVersion; private boolean enableSupportedVersionCheck; private String sparkUrl; + private SparkShims sparkShims; private static InterpreterHookRegistry hooks; @@ -117,7 +118,8 @@ public void open() throws InterpreterException { sqlContext = this.innerInterpreter.sqlContext(); sparkSession = this.innerInterpreter.sparkSession(); sparkUrl = this.innerInterpreter.sparkUrl(); - setupListeners(); + sparkShims = SparkShims.getInstance(sc.version()); + sparkShims.setupSparkListener(sparkUrl); hooks = getInterpreterGroup().getInterpreterHookRegistry(); z = new SparkZeppelinContext(sc, hooks, @@ -125,7 +127,7 @@ public void open() throws InterpreterException { this.innerInterpreter.bind("z", z.getClass().getCanonicalName(), z, Lists.newArrayList("@transient")); } catch (Exception e) { - LOGGER.error(ExceptionUtils.getStackTrace(e)); + LOGGER.error("Fail to open SparkInterpreter", ExceptionUtils.getStackTrace(e)); throw new InterpreterException("Fail to open SparkInterpreter", e); } } @@ -213,67 +215,6 @@ public int getProgress(InterpreterContext context) { return innerInterpreter.getProgress(Utils.buildJobGroupId(context), context); } - private void setupListeners() { - JobProgressListener pl = new JobProgressListener(sc.getConf()) { - @Override - public synchronized void onJobStart(SparkListenerJobStart jobStart) { - super.onJobStart(jobStart); - int jobId = jobStart.jobId(); - String jobGroupId = jobStart.properties().getProperty("spark.jobGroup.id"); - String uiEnabled = jobStart.properties().getProperty("spark.ui.enabled"); - String jobUrl = getJobUrl(jobId); - String noteId = Utils.getNoteId(jobGroupId); - String paragraphId = Utils.getParagraphId(jobGroupId); - // Button visible if Spark UI property not set, set as invalid boolean or true - java.lang.Boolean showSparkUI = - uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false"); - if (showSparkUI && jobUrl != null) { - RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient(); - Map infos = new java.util.HashMap<>(); - infos.put("jobUrl", jobUrl); - infos.put("label", "SPARK JOB"); - infos.put("tooltip", "View in Spark web UI"); - if (eventClient != null) { - eventClient.onParaInfosReceived(noteId, paragraphId, infos); - } - } - } - - private String getJobUrl(int jobId) { - String jobUrl = null; - if (sparkUrl != null) { - jobUrl = sparkUrl + "/jobs/job?id=" + jobId; - } - return jobUrl; - } - }; - try { - Object listenerBus = sc.getClass().getMethod("listenerBus").invoke(sc); - Method[] methods = listenerBus.getClass().getMethods(); - Method addListenerMethod = null; - for (Method m : methods) { - if (!m.getName().equals("addListener")) { - continue; - } - Class[] parameterTypes = m.getParameterTypes(); - if (parameterTypes.length != 1) { - continue; - } - if (!parameterTypes[0].isAssignableFrom(JobProgressListener.class)) { - continue; - } - addListenerMethod = m; - break; - } - if (addListenerMethod != null) { - addListenerMethod.invoke(listenerBus, pl); - } - } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - LOGGER.error(e.toString(), e); - } - } - public SparkZeppelinContext getZeppelinContext() { return this.z; } diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java index ff3a2caa565..1f59d18d339 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java @@ -151,6 +151,8 @@ public class OldSparkInterpreter extends AbstractSparkInterpreter { private JavaSparkContext jsc; private boolean enableSupportedVersionCheck; + private SparkShims sparkShims; + public OldSparkInterpreter(Properties property) { super(property); out = new InterpreterOutputStream(logger); @@ -158,10 +160,10 @@ public OldSparkInterpreter(Properties property) { public OldSparkInterpreter(Properties property, SparkContext sc) { this(property); - this.sc = sc; env = SparkEnv.get(); - sparkListener = setupListeners(this.sc); + sparkShims = SparkShims.getInstance(sc.version()); + sparkShims.setupSparkListener(sparkUrl); } public SparkContext getSparkContext() { @@ -169,7 +171,6 @@ public SparkContext getSparkContext() { if (sc == null) { sc = createSparkContext(); env = SparkEnv.get(); - sparkListener = setupListeners(sc); } return sc; } @@ -190,157 +191,6 @@ public boolean isSparkContextInitialized() { } } - static SparkListener setupListeners(SparkContext context) { - SparkListener pl = new SparkListener() { - @Override - public synchronized void onJobStart(SparkListenerJobStart jobStart) { - int jobId = jobStart.jobId(); - String jobGroupId = jobStart.properties().getProperty("spark.jobGroup.id"); - String uiEnabled = jobStart.properties().getProperty("spark.ui.enabled"); - String jobUrl = getJobUrl(jobId); - String noteId = Utils.getNoteId(jobGroupId); - String paragraphId = Utils.getParagraphId(jobGroupId); - // Button visible if Spark UI property not set, set as invalid boolean or true - java.lang.Boolean showSparkUI = - uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false"); - if (showSparkUI && jobUrl != null) { - RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient(); - Map infos = new java.util.HashMap<>(); - infos.put("jobUrl", jobUrl); - infos.put("label", "SPARK JOB"); - infos.put("tooltip", "View in Spark web UI"); - if (eventClient != null) { - eventClient.onParaInfosReceived(noteId, paragraphId, infos); - } - } - } - - private String getJobUrl(int jobId) { - String jobUrl = null; - if (sparkUrl != null) { - jobUrl = sparkUrl + "/jobs/job/?id=" + jobId; - } - return jobUrl; - } - - @Override - public void onBlockUpdated(SparkListenerBlockUpdated blockUpdated) { - - } - - @Override - public void onExecutorRemoved(SparkListenerExecutorRemoved executorRemoved) { - - } - - @Override - public void onExecutorAdded(SparkListenerExecutorAdded executorAdded) { - - } - - @Override - public void onExecutorMetricsUpdate( - SparkListenerExecutorMetricsUpdate executorMetricsUpdate) { - - } - - @Override - public void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) { - - } - - @Override - public void onApplicationStart(SparkListenerApplicationStart applicationStart) { - - } - - @Override - public void onUnpersistRDD(SparkListenerUnpersistRDD unpersistRDD) { - - } - - @Override - public void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) { - - } - - @Override - public void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) { - - } - - @Override - public void onEnvironmentUpdate(SparkListenerEnvironmentUpdate environmentUpdate) { - - } - - @Override - public void onJobEnd(SparkListenerJobEnd jobEnd) { - - } - - @Override - public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { - - } - - @Override - public void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) { - - } - - @Override - public void onTaskEnd(SparkListenerTaskEnd taskEnd) { - - } - - @Override - public void onTaskGettingResult(SparkListenerTaskGettingResult taskGettingResult) { - - } - - @Override - public void onTaskStart(SparkListenerTaskStart taskStart) { - - } - }; - try { - Object listenerBus = context.getClass().getMethod("listenerBus").invoke(context); - - Method[] methods = listenerBus.getClass().getMethods(); - Method addListenerMethod = null; - for (Method m : methods) { - if (!m.getName().equals("addListener")) { - continue; - } - - Class[] parameterTypes = m.getParameterTypes(); - - if (parameterTypes.length != 1) { - continue; - } - - if (!parameterTypes[0].isAssignableFrom(SparkListener.class)) { - continue; - } - - addListenerMethod = m; - break; - } - - if (addListenerMethod != null) { - addListenerMethod.invoke(listenerBus, pl); - } else { - return null; - } - } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - logger.error(e.toString(), e); - return null; - } - return pl; - } - private boolean useHiveContext() { return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.useHiveContext")); } @@ -1020,6 +870,10 @@ public void open() throws InterpreterException { } } + sparkUrl = getSparkUIUrl(); + sparkShims = SparkShims.getInstance(sc.version()); + sparkShims.setupSparkListener(sparkUrl); + numReferenceOfSparkContext.incrementAndGet(); } @@ -1373,75 +1227,6 @@ public int getProgress(InterpreterContext context) { return JobProgressUtil.progress(sc, jobGroup); } - private int[] getProgressFromStage_1_0x(SparkListener sparkListener, Object stage) - throws IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException { - int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); - int completedTasks = 0; - - int id = (int) stage.getClass().getMethod("id").invoke(stage); - - Object completedTaskInfo = null; - - completedTaskInfo = JavaConversions.mapAsJavaMap( - (HashMap) sparkListener.getClass() - .getMethod("stageIdToTasksComplete").invoke(sparkListener)).get(id); - - if (completedTaskInfo != null) { - completedTasks += (int) completedTaskInfo; - } - List parents = JavaConversions.seqAsJavaList((Seq) stage.getClass() - .getMethod("parents").invoke(stage)); - if (parents != null) { - for (Object s : parents) { - int[] p = getProgressFromStage_1_0x(sparkListener, s); - numTasks += p[0]; - completedTasks += p[1]; - } - } - - return new int[] {numTasks, completedTasks}; - } - - private int[] getProgressFromStage_1_1x(SparkListener sparkListener, Object stage) - throws IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException { - int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); - int completedTasks = 0; - int id = (int) stage.getClass().getMethod("id").invoke(stage); - - try { - Method stageIdToData = sparkListener.getClass().getMethod("stageIdToData"); - HashMap, Object> stageIdData = - (HashMap, Object>) stageIdToData.invoke(sparkListener); - Class stageUIDataClass = - this.getClass().forName("org.apache.spark.ui.jobs.UIData$StageUIData"); - - Method numCompletedTasks = stageUIDataClass.getMethod("numCompleteTasks"); - Set> keys = - JavaConverters.setAsJavaSetConverter(stageIdData.keySet()).asJava(); - for (Tuple2 k : keys) { - if (id == (int) k._1()) { - Object uiData = stageIdData.get(k).get(); - completedTasks += (int) numCompletedTasks.invoke(uiData); - } - } - } catch (Exception e) { - logger.error("Error on getting progress information", e); - } - - List parents = JavaConversions.seqAsJavaList((Seq) stage.getClass() - .getMethod("parents").invoke(stage)); - if (parents != null) { - for (Object s : parents) { - int[] p = getProgressFromStage_1_1x(sparkListener, s); - numTasks += p[0]; - completedTasks += p[1]; - } - } - return new int[] {numTasks, completedTasks}; - } - private Code getResultCode(scala.tools.nsc.interpreter.Results.Result r) { if (r instanceof scala.tools.nsc.interpreter.Results.Success$) { return Code.SUCCESS; @@ -1479,10 +1264,6 @@ public FormType getFormType() { return FormType.NATIVE; } - public SparkListener getJobProgressListener() { - return sparkListener; - } - @Override public Scheduler getScheduler() { return SchedulerFactory.singleton().createOrGetFIFOScheduler( diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java index 14214a284f2..068ff50c3d8 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java @@ -192,13 +192,7 @@ public void testNextLineCompanionObject() throws InterpreterException { public void testEndWithComment() throws InterpreterException { assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("val c=1\n//comment", context).code()); } - - @Test - public void testListener() { - SparkContext sc = repl.getSparkContext(); - assertNotNull(OldSparkInterpreter.setupListeners(sc)); - } - + @Test public void testCreateDataFrame() throws InterpreterException { if (getSparkVersionNumber(repl) >= 13) { @@ -362,7 +356,7 @@ public void testParagraphUrls() throws InterpreterException { } String sparkUIUrl = repl.getSparkUIUrl(); assertNotNull(jobUrl); - assertTrue(jobUrl.startsWith(sparkUIUrl + "/jobs/job/?id=")); + assertTrue(jobUrl.startsWith(sparkUIUrl + "/jobs/job?id=")); } } diff --git a/spark/pom.xml b/spark/pom.xml index 7a0c7c2ac66..def865e5c55 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -63,21 +63,26 @@ scala-2.10 scala-2.11 spark-dependencies + spark-shims + spark1-shims + spark2-shims - org.apache.zeppelin - zeppelin-interpreter - ${project.version} + org.slf4j + slf4j-api - org.apache.zeppelin - zeppelin-display - ${project.version} - test + org.slf4j + slf4j-log4j12 + + + + log4j + log4j @@ -92,28 +97,6 @@ junit test - - - org.datanucleus - datanucleus-core - ${datanucleus.core.version} - test - - - - org.datanucleus - datanucleus-api-jdo - ${datanucleus.apijdo.version} - test - - - - org.datanucleus - datanucleus-rdbms - ${datanucleus.rdbms.version} - test - - diff --git a/spark/spark-dependencies/pom.xml b/spark/spark-dependencies/pom.xml index 58977b4db77..4e90a9304b2 100644 --- a/spark/spark-dependencies/pom.xml +++ b/spark/spark-dependencies/pom.xml @@ -294,46 +294,7 @@ hadoop-client ${hadoop.version} - - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - - ${akka.group} - akka-actor_${scala.binary.version} - ${akka.version} - - - ${akka.group} - akka-remote_${scala.binary.version} - ${akka.version} - - - ${akka.group} - akka-slf4j_${scala.binary.version} - ${akka.version} - - - ${akka.group} - akka-testkit_${scala.binary.version} - ${akka.version} - - - ${akka.group} - akka-zeromq_${scala.binary.version} - ${akka.version} - - - ${akka.group} - akka-actor_${scala.binary.version} - - - - + org.apache.spark diff --git a/spark/spark-shims/pom.xml b/spark/spark-shims/pom.xml new file mode 100644 index 00000000000..619c7a42a86 --- /dev/null +++ b/spark/spark-shims/pom.xml @@ -0,0 +1,70 @@ + + + + + + + spark-parent + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../pom.xml + + + 4.0.0 + org.apache.zeppelin + spark-shims + 0.9.0-SNAPSHOT + jar + Zeppelin: Spark Shims + + + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + provided + + + + + + + maven-dependency-plugin + + true + + + + + maven-resources-plugin + + + copy-interpreter-setting + none + + true + + + + + + + + \ No newline at end of file diff --git a/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java b/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java new file mode 100644 index 00000000000..acf717c5ae3 --- /dev/null +++ b/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.spark; + + +import org.apache.zeppelin.interpreter.BaseZeppelinContext; +import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Constructor; +import java.util.Map; +import java.util.Properties; + +/** + * This is abstract class for anything that is api incompatible between spark1 and spark2. + * It will load the correct version of SparkShims based on the version of Spark. + */ +public abstract class SparkShims { + + private static final Logger LOGGER = LoggerFactory.getLogger(SparkShims.class); + + private static SparkShims sparkShims; + + private static SparkShims loadShims(String sparkVersion) throws ReflectiveOperationException { + Class sparkShimsClass; + if ("2".equals(sparkVersion)) { + LOGGER.info("Initializing shims for Spark 2.x"); + sparkShimsClass = Class.forName("org.apache.zeppelin.spark.Spark2Shims"); + } else { + LOGGER.info("Initializing shims for Spark 1.x"); + sparkShimsClass = Class.forName("org.apache.zeppelin.spark.Spark1Shims"); + } + + Constructor c = sparkShimsClass.getConstructor(); + return (SparkShims) c.newInstance(); + } + + public static SparkShims getInstance(String sparkVersion) { + if (sparkShims == null) { + String sparkMajorVersion = getSparkMajorVersion(sparkVersion); + try { + sparkShims = loadShims(sparkMajorVersion); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + return sparkShims; + } + + private static String getSparkMajorVersion(String sparkVersion) { + return sparkVersion.startsWith("2") ? "2" : "1"; + } + + /** + * This is due to SparkListener api change between spark1 and spark2. + * SparkListener is trait in spark1 while it is abstract class in spark2. + */ + public abstract void setupSparkListener(String sparkWebUrl); + + + protected String getNoteId(String jobgroupId) { + int indexOf = jobgroupId.indexOf("-"); + int secondIndex = jobgroupId.indexOf("-", indexOf + 1); + return jobgroupId.substring(indexOf + 1, secondIndex); + } + + protected String getParagraphId(String jobgroupId) { + int indexOf = jobgroupId.indexOf("-"); + int secondIndex = jobgroupId.indexOf("-", indexOf + 1); + return jobgroupId.substring(secondIndex + 1, jobgroupId.length()); + } + + protected void buildSparkJobUrl(String sparkWebUrl, int jobId, Properties jobProperties) { + String jobGroupId = jobProperties.getProperty("spark.jobGroup.id"); + String uiEnabled = jobProperties.getProperty("spark.ui.enabled"); + String jobUrl = sparkWebUrl + "/jobs/job?id=" + jobId; + String noteId = getNoteId(jobGroupId); + String paragraphId = getParagraphId(jobGroupId); + // Button visible if Spark UI property not set, set as invalid boolean or true + boolean showSparkUI = + uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false"); + if (showSparkUI && jobUrl != null) { + RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient(); + Map infos = new java.util.HashMap(); + infos.put("jobUrl", jobUrl); + infos.put("label", "SPARK JOB"); + infos.put("tooltip", "View in Spark web UI"); + if (eventClient != null) { + eventClient.onParaInfosReceived(noteId, paragraphId, infos); + } + } + } +} diff --git a/spark/spark1-shims/pom.xml b/spark/spark1-shims/pom.xml new file mode 100644 index 00000000000..93640c6ffe0 --- /dev/null +++ b/spark/spark1-shims/pom.xml @@ -0,0 +1,89 @@ + + + + + + + spark-parent + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../pom.xml + + + 4.0.0 + org.apache.zeppelin + spark1-shims + 0.9.0-SNAPSHOT + jar + Zeppelin: Spark1 Shims + + + 2.10 + 1.6.3 + + + + + + org.apache.zeppelin + spark-shims + ${project.version} + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + provided + + + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + provided + + + + + + + maven-dependency-plugin + + true + + + + + maven-resources-plugin + + + copy-interpreter-setting + none + + true + + + + + + + + \ No newline at end of file diff --git a/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java b/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java new file mode 100644 index 00000000000..9f233136799 --- /dev/null +++ b/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.spark; + +import org.apache.spark.SparkContext; +import org.apache.spark.scheduler.SparkListener; +import org.apache.spark.scheduler.SparkListenerApplicationEnd; +import org.apache.spark.scheduler.SparkListenerApplicationStart; +import org.apache.spark.scheduler.SparkListenerBlockManagerAdded; +import org.apache.spark.scheduler.SparkListenerBlockManagerRemoved; +import org.apache.spark.scheduler.SparkListenerBlockUpdated; +import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate; +import org.apache.spark.scheduler.SparkListenerExecutorAdded; +import org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate; +import org.apache.spark.scheduler.SparkListenerExecutorRemoved; +import org.apache.spark.scheduler.SparkListenerJobEnd; +import org.apache.spark.scheduler.SparkListenerJobStart; +import org.apache.spark.scheduler.SparkListenerStageCompleted; +import org.apache.spark.scheduler.SparkListenerStageSubmitted; +import org.apache.spark.scheduler.SparkListenerTaskEnd; +import org.apache.spark.scheduler.SparkListenerTaskGettingResult; +import org.apache.spark.scheduler.SparkListenerTaskStart; +import org.apache.spark.scheduler.SparkListenerUnpersistRDD; +import org.apache.spark.ui.jobs.JobProgressListener; +import org.apache.zeppelin.interpreter.BaseZeppelinContext; +import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper; + +import java.util.Map; + +public class Spark1Shims extends SparkShims { + + public void setupSparkListener(final String sparkWebUrl) { + SparkContext sc = SparkContext.getOrCreate(); + sc.addSparkListener(new JobProgressListener(sc.getConf()) { + @Override + public void onJobStart(SparkListenerJobStart jobStart) { + buildSparkJobUrl(sparkWebUrl, jobStart.jobId(), jobStart.properties()); + } + }); + } +} diff --git a/spark/spark2-shims/pom.xml b/spark/spark2-shims/pom.xml new file mode 100644 index 00000000000..000e3abd864 --- /dev/null +++ b/spark/spark2-shims/pom.xml @@ -0,0 +1,88 @@ + + + + + + spark-parent + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../pom.xml + + + 4.0.0 + org.apache.zeppelin + spark2-shims + 0.9.0-SNAPSHOT + jar + Zeppelin: Spark2 Shims + + + 2.11 + 2.1.2 + + + + + + org.apache.zeppelin + spark-shims + ${project.version} + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + provided + + + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + provided + + + + + + + maven-dependency-plugin + + true + + + + + maven-resources-plugin + + + copy-interpreter-setting + none + + true + + + + + + + + \ No newline at end of file diff --git a/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java b/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java new file mode 100644 index 00000000000..4b3961064c5 --- /dev/null +++ b/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.spark; + +import org.apache.spark.SparkContext; +import org.apache.spark.scheduler.SparkListener; +import org.apache.spark.scheduler.SparkListenerJobStart; + +public class Spark2Shims extends SparkShims { + + public void setupSparkListener(final String sparkWebUrl) { + SparkContext sc = SparkContext.getOrCreate(); + sc.addSparkListener(new SparkListener() { + @Override + public void onJobStart(SparkListenerJobStart jobStart) { + buildSparkJobUrl(sparkWebUrl, jobStart.jobId(), jobStart.properties()); + } + }); + } +} From 500b74b196b740c810553c43216a56e23ab9caf0 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 27 Feb 2018 20:53:54 +0800 Subject: [PATCH 052/386] ZEPPELIN-3242. Listener threw an exception java.lang.NPEat o.a.zeppelin.spark.Utils.getNoteId(Utils.java:156) ### What is this PR for? This issue also cause spark url can not be displayed in frontend. The root cause is that PySparkInterpreter/IPySparkInterpreter doesn't set JobGroup correctly. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3242 ### How should this be tested? * CI pass and also manually verified. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? NO * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2822 from zjffdu/ZEPPELIN-3242 and squashes the following commits: 8254162 [Jeff Zhang] ZEPPELIN-3242. Listener threw an exception java.lang.NPEat o.a.zeppelin.spark.Utils.getNoteId(Utils.java:156) --- .../zeppelin/spark/IPySparkInterpreter.java | 11 +++++++ .../zeppelin/spark/PySparkInterpreter.java | 13 +++++--- .../spark/IPySparkInterpreterTest.java | 33 ++++++++++++------- .../spark/NewSparkInterpreterTest.java | 17 ++++++++-- .../zeppelin/spark/SparkRInterpreterTest.java | 19 +++++++++-- .../interpreter/InterpreterContext.java | 4 +++ 6 files changed, 77 insertions(+), 20 deletions(-) diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java index a75fda8c1d9..3691156e3bf 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java @@ -23,6 +23,7 @@ import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.python.IPythonInterpreter; @@ -98,6 +99,16 @@ public BaseZeppelinContext buildZeppelinContext() { return sparkInterpreter.getZeppelinContext(); } + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + InterpreterContext.set(context); + sparkInterpreter.populateSparkWebUrl(context); + String jobGroupId = Utils.buildJobGroupId(context); + String jobDesc = "Started by: " + Utils.getUserName(context.getAuthenticationInfo()); + String setJobGroupStmt = "sc.setJobGroup('" + jobGroupId + "', '" + jobDesc + "')"; + return super.interpret(setJobGroupStmt +"\n" + st, context); + } + @Override public void cancel(InterpreterContext context) throws InterpreterException { super.cancel(context); diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index 0703ad79186..f5e4793bb1b 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -406,16 +406,16 @@ public void appendOutput(String message) throws IOException { @Override public InterpreterResult interpret(String st, InterpreterContext context) throws InterpreterException { + if (iPySparkInterpreter != null) { + return iPySparkInterpreter.interpret(st, context); + } + SparkInterpreter sparkInterpreter = getSparkInterpreter(); - sparkInterpreter.populateSparkWebUrl(context); if (sparkInterpreter.isUnsupportedSparkVersion()) { return new InterpreterResult(Code.ERROR, "Spark " + sparkInterpreter.getSparkVersion().toString() + " is not supported"); } - - if (iPySparkInterpreter != null) { - return iPySparkInterpreter.interpret(st, context); - } + sparkInterpreter.populateSparkWebUrl(context); if (!pythonscriptRunning) { return new InterpreterResult(Code.ERROR, "python process not running" @@ -467,10 +467,13 @@ public InterpreterResult interpret(String st, InterpreterContext context) } String jobGroup = Utils.buildJobGroupId(context); String jobDesc = "Started by: " + Utils.getUserName(context.getAuthenticationInfo()); + SparkZeppelinContext __zeppelin__ = sparkInterpreter.getZeppelinContext(); __zeppelin__.setInterpreterContext(context); __zeppelin__.setGui(context.getGui()); __zeppelin__.setNoteGui(context.getNoteGui()); + InterpreterContext.set(context); + pythonInterpretRequest = new PythonInterpretRequest(st, jobGroup, jobDesc); statementOutput = null; diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java index 5eaa42c4625..46a3a7277fe 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java @@ -27,6 +27,7 @@ import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.remote.RemoteEventClient; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.python.IPythonInterpreterTest; import org.apache.zeppelin.user.AuthenticationInfo; @@ -39,15 +40,21 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Properties; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class IPySparkInterpreterTest { private IPySparkInterpreter iPySparkInterpreter; private InterpreterGroup intpGroup; + private RemoteEventClient mockRemoteEventClient = mock(RemoteEventClient.class); @Before public void setup() throws InterpreterException { @@ -69,11 +76,13 @@ public void setup() throws InterpreterException { intpGroup.get("session_1").add(sparkInterpreter); sparkInterpreter.setInterpreterGroup(intpGroup); sparkInterpreter.open(); + sparkInterpreter.getZeppelinContext().setEventClient(mockRemoteEventClient); iPySparkInterpreter = new IPySparkInterpreter(p); intpGroup.get("session_1").add(iPySparkInterpreter); iPySparkInterpreter.setInterpreterGroup(intpGroup); iPySparkInterpreter.open(); + sparkInterpreter.getZeppelinContext().setEventClient(mockRemoteEventClient); } @@ -91,17 +100,21 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx // rdd InterpreterContext context = getInterpreterContext(); - InterpreterResult result = iPySparkInterpreter.interpret("sc.range(1,10).sum()", context); + InterpreterResult result = iPySparkInterpreter.interpret("sc.version", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - List interpreterResultMessages = context.out.getInterpreterResultMessages(); - assertEquals("45", interpreterResultMessages.get(0).getData()); + // spark url is sent + verify(mockRemoteEventClient).onMetaInfosReceived(any(Map.class)); context = getInterpreterContext(); - result = iPySparkInterpreter.interpret("sc.version", context); + result = iPySparkInterpreter.interpret("sc.range(1,10).sum()", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + List interpreterResultMessages = context.out.getInterpreterResultMessages(); + assertEquals("45", interpreterResultMessages.get(0).getData()); + // spark job url is sent + verify(mockRemoteEventClient).onParaInfosReceived(any(String.class), any(String.class), any(Map.class)); + // spark sql context = getInterpreterContext(); if (interpreterResultMessages.get(0).getData().startsWith("'1.") || @@ -146,7 +159,6 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx "1 a\n" + "2 b\n", interpreterResultMessages.get(0).getData()); } - // cancel final InterpreterContext context2 = getInterpreterContext(); @@ -166,6 +178,7 @@ public void run() { }; thread.start(); + // sleep 1 second to wait for the spark job starts Thread.sleep(1000); iPySparkInterpreter.cancel(context); @@ -177,10 +190,6 @@ public void run() { assertEquals("range", completions.get(0).getValue()); // pyspark streaming - - Class klass = py4j.GatewayServer.class; - URL location = klass.getResource('/' + klass.getName().replace('.', '/') + ".class"); - System.out.println("py4j location: " + location); context = getInterpreterContext(); result = iPySparkInterpreter.interpret( "from pyspark.streaming import StreamingContext\n" + @@ -204,7 +213,7 @@ public void run() { } private InterpreterContext getInterpreterContext() { - return new InterpreterContext( + InterpreterContext context = new InterpreterContext( "noteId", "paragraphId", "replName", @@ -218,5 +227,7 @@ private InterpreterContext getInterpreterContext() { null, null, new InterpreterOutput(null)); + context.setClient(mockRemoteEventClient); + return context; } } diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java index cfcf2a54aaa..3d22af31963 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java @@ -29,6 +29,8 @@ import org.apache.zeppelin.interpreter.InterpreterOutputListener; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput; +import org.apache.zeppelin.interpreter.remote.RemoteEventClient; +import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.After; @@ -42,12 +44,14 @@ import java.nio.channels.ReadableByteChannel; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class NewSparkInterpreterTest { @@ -59,6 +63,8 @@ public class NewSparkInterpreterTest { // catch the interpreter output in onUpdate private InterpreterResultMessageOutput messageOutput; + private RemoteEventClient mockRemoteEventClient = mock(RemoteEventClient.class); + @Test public void testSparkInterpreter() throws IOException, InterruptedException, InterpreterException { Properties properties = new Properties(); @@ -72,9 +78,12 @@ public void testSparkInterpreter() throws IOException, InterruptedException, Int interpreter.setInterpreterGroup(mock(InterpreterGroup.class)); interpreter.open(); + interpreter.getZeppelinContext().setEventClient(mockRemoteEventClient); InterpreterResult result = interpreter.interpret("val a=\"hello world\"", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals("a: String = hello world\n", output); + // spark web url is sent + verify(mockRemoteEventClient).onMetaInfosReceived(any(Map.class)); result = interpreter.interpret("print(a)", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -124,6 +133,8 @@ public void testSparkInterpreter() throws IOException, InterruptedException, Int result = interpreter.interpret("sc.range(1, 10).sum", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(output.contains("45")); + // spark job url is sent + verify(mockRemoteEventClient).onParaInfosReceived(any(String.class), any(String.class), any(Map.class)); // case class result = interpreter.interpret("val bankText = sc.textFile(\"bank.csv\")", getInterpreterContext()); @@ -349,7 +360,7 @@ public void tearDown() throws InterpreterException { private InterpreterContext getInterpreterContext() { output = ""; - return new InterpreterContext( + InterpreterContext context = new InterpreterContext( "noteId", "paragraphId", "replName", @@ -385,5 +396,7 @@ public void onUpdate(int index, InterpreterResultMessageOutput out) { } }) ); + context.setClient(mockRemoteEventClient); + return context; } } diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java index 2d585f5387c..0bd88d44d77 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java @@ -24,21 +24,27 @@ import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.apache.zeppelin.interpreter.remote.RemoteEventClient; import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.Test; import java.io.IOException; import java.util.HashMap; +import java.util.Map; import java.util.Properties; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class SparkRInterpreterTest { private SparkRInterpreter sparkRInterpreter; private SparkInterpreter sparkInterpreter; - + private RemoteEventClient mockRemoteEventClient = mock(RemoteEventClient.class); @Test public void testSparkRInterpreter() throws IOException, InterruptedException, InterpreterException { @@ -60,10 +66,13 @@ public void testSparkRInterpreter() throws IOException, InterruptedException, In sparkInterpreter.setInterpreterGroup(interpreterGroup); sparkRInterpreter.open(); + sparkInterpreter.getZeppelinContext().setEventClient(mockRemoteEventClient); InterpreterResult result = sparkRInterpreter.interpret("1+1", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains("2")); + // spark web url is sent + verify(mockRemoteEventClient).onMetaInfosReceived(any(Map.class)); result = sparkRInterpreter.interpret("sparkR.version()", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -72,16 +81,20 @@ public void testSparkRInterpreter() throws IOException, InterruptedException, In result = sparkRInterpreter.interpret("df <- as.DataFrame(faithful)\nhead(df)", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains("eruptions waiting")); + // spark job url is sent + verify(mockRemoteEventClient, atLeastOnce()).onParaInfosReceived(any(String.class), any(String.class), any(Map.class)); } else { // spark 1.x result = sparkRInterpreter.interpret("df <- createDataFrame(sqlContext, faithful)\nhead(df)", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains("eruptions waiting")); + // spark job url is sent + verify(mockRemoteEventClient, atLeastOnce()).onParaInfosReceived(any(String.class), any(String.class), any(Map.class)); } } private InterpreterContext getInterpreterContext() { - return new InterpreterContext( + InterpreterContext context = new InterpreterContext( "noteId", "paragraphId", "replName", @@ -95,5 +108,7 @@ private InterpreterContext getInterpreterContext() { null, null, null); + context.setClient(mockRemoteEventClient); + return context; } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java index 293f9bfa2bf..8fa09049773 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java @@ -226,6 +226,10 @@ public RemoteEventClientWrapper getClient() { return client; } + public void setClient(RemoteEventClientWrapper client) { + this.client = client; + } + public RemoteWorksController getRemoteWorksController() { return remoteWorksController; } From 65b797c22eadebb9a18575e91fdcfe33722762c8 Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Tue, 13 Feb 2018 12:21:21 +0100 Subject: [PATCH 053/386] ZEPPELIN-3154. Fixed Checkstyle errors and warnings in flink module ### What is this PR for? Fixed the Checkstyle errors and warnings in the flink module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3154 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2795 from HorizonNet/ZEPPELIN-3154 and squashes the following commits: f70e93d [Jan Hentschel] ZEPPELIN-3154. Fixed Checkstyle errors and warnings in flink module --- flink/pom.xml | 253 +++++++++--------- .../zeppelin/flink/FlinkInterpreter.java | 37 +-- .../zeppelin/flink/FlinkInterpreterTest.java | 22 +- 3 files changed, 164 insertions(+), 148 deletions(-) diff --git a/flink/pom.xml b/flink/pom.xml index 455092d8a15..70c076d41f5 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -154,142 +154,149 @@ - net.alchim31.maven - scala-maven-plugin - ${plugin.scalamaven.version} - - - - scala-compile-first - process-resources - - compile - - - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - -Xms128m - -Xmx512m - - - - org.scalamacros - paradise_${scala.version} - ${scala.macros.version} - - - + net.alchim31.maven + scala-maven-plugin + ${plugin.scalamaven.version} + + + + scala-compile-first + process-resources + + compile + + + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + -Xms128m + -Xmx512m + + + + org.scalamacros + paradise_${scala.version} + ${scala.macros.version} + + + - org.apache.maven.plugins - maven-eclipse-plugin - ${plugin.eclipse.version} - - true - - org.scala-ide.sdt.core.scalanature - org.eclipse.jdt.core.javanature - - - org.scala-ide.sdt.core.scalabuilder - - - org.scala-ide.sdt.launching.SCALA_CONTAINER - org.eclipse.jdt.launching.JRE_CONTAINER - - - - **/*.scala - **/*.java - - + org.apache.maven.plugins + maven-eclipse-plugin + ${plugin.eclipse.version} + + true + + org.scala-ide.sdt.core.scalanature + org.eclipse.jdt.core.javanature + + + org.scala-ide.sdt.core.scalabuilder + + + org.scala-ide.sdt.launching.SCALA_CONTAINER + org.eclipse.jdt.launching.JRE_CONTAINER + + + + **/*.scala + **/*.java + + - org.codehaus.mojo - build-helper-maven-plugin - ${plugin.buildhelper.version} - - - - add-source - generate-sources - - add-source - - - - src/main/scala - - - - - - add-test-source - generate-test-sources - - add-test-source - - - - src/test/scala - - - - + org.codehaus.mojo + build-helper-maven-plugin + ${plugin.buildhelper.version} + + + + add-source + generate-sources + + add-source + + + + src/main/scala + + + + + + add-test-source + generate-test-sources + + add-test-source + + + + src/test/scala + + + + - org.scalastyle - scalastyle-maven-plugin - ${plugin.scalastyle.version} - - - - check - - - - - false - true - true - false - ${basedir}/src/main/scala - ${basedir}/src/test/scala - ${project.basedir}/../_tools/scalastyle.xml - ${project.basedir}/target/scalastyle-output.xml - UTF-8 - + org.scalastyle + scalastyle-maven-plugin + ${plugin.scalastyle.version} + + + + check + + + + + false + true + true + false + ${basedir}/src/main/scala + ${basedir}/src/test/scala + ${project.basedir}/../_tools/scalastyle.xml + ${project.basedir}/target/scalastyle-output.xml + UTF-8 + - - maven-enforcer-plugin - - - maven-dependency-plugin - - - maven-resources-plugin - + + maven-enforcer-plugin + + + maven-dependency-plugin + + + maven-resources-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java index 19c77de914a..9d664378d60 100644 --- a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java +++ b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java @@ -17,15 +17,6 @@ */ package org.apache.zeppelin.flink; -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; -import java.io.PrintWriter; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.*; - import org.apache.flink.api.common.JobID; import org.apache.flink.api.scala.FlinkILoop; import org.apache.flink.configuration.Configuration; @@ -33,15 +24,21 @@ import org.apache.flink.runtime.instance.ActorGateway; import org.apache.flink.runtime.messages.JobManagerMessages; import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.InterpreterUtils; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + import scala.Console; import scala.Some; import scala.collection.JavaConversions; @@ -54,8 +51,15 @@ import scala.tools.nsc.settings.MutableSettings.BooleanSetting; import scala.tools.nsc.settings.MutableSettings.PathSetting; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterUtils; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; + /** - * Interpreter for Apache Flink (http://flink.apache.org) + * Interpreter for Apache Flink (http://flink.apache.org). */ public class FlinkInterpreter extends Interpreter { Logger logger = LoggerFactory.getLogger(FlinkInterpreter.class); @@ -116,7 +120,6 @@ public void open() { org.apache.flink.api.scala.ExecutionEnvironment benv = flinkIloop.scalaBenv(); - //new ExecutionEnvironment(remoteBenv) org.apache.flink.streaming.api.scala.StreamExecutionEnvironment senv = flinkIloop.scalaSenv(); diff --git a/flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java b/flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java index c9cb1f63508..a95db3967d3 100644 --- a/flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java +++ b/flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java @@ -20,15 +20,16 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + import java.util.Arrays; import java.util.Properties; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; public class FlinkInterpreterTest { @@ -40,7 +41,8 @@ public static void setUp() { Properties p = new Properties(); flink = new FlinkInterpreter(p); flink.open(); - context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null, null, null); + context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null, + null, null, null); } @AfterClass @@ -50,17 +52,20 @@ public static void tearDown() { @Test public void testNextLineInvocation() { - assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret("\"123\"\n.toInt", context).code()); + assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret("\"123\"\n.toInt", context) + .code()); } @Test public void testNextLineComments() { - assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret("\"123\"\n/*comment here\n*/.toInt", context).code()); + assertEquals(InterpreterResult.Code.SUCCESS, + flink.interpret("\"123\"\n/*comment here\n*/.toInt", context).code()); } @Test public void testNextLineCompanionObject() { - String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter {\n def apply(x: Long) = new Counter()\n}"; + String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter " + + "{\n def apply(x: Long) = new Counter()\n}"; assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret(code, context).code()); } @@ -81,7 +86,8 @@ public void testSimpleStatementWithSystemOutput() { @Test public void testWordCount() { flink.interpret("val text = benv.fromElements(\"To be or not to be\")", context); - flink.interpret("val counts = text.flatMap { _.toLowerCase.split(\" \") }.map { (_, 1) }.groupBy(0).sum(1)", context); + flink.interpret("val counts = text.flatMap { _.toLowerCase.split(\" \") }" + + ".map { (_, 1) }.groupBy(0).sum(1)", context); InterpreterResult result = flink.interpret("counts.print()", context); assertEquals(Code.SUCCESS, result.code()); From 2c322d72b662e703710a30abc4460f9aee2a29bb Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Thu, 1 Mar 2018 15:28:08 +0530 Subject: [PATCH 054/386] [ZEPPELIN-3271] Option for disabling scheduler ### What is this PR for? Zeppelin server should have an option to enable/disable cron scheduler from Zeppelin config. ### What type of PR is it? [Bug Fix] ### What is the Jira issue? * [ZEPPELIN-3271](https://issues.apache.org/jira/browse/ZEPPELIN-3271) ### How should this be tested? * On adding below mentioned property in `zeppelin-site.xml`, this feature should get disabled from both web-socket and REST-APIs. ``` zeppelin.notebook.cron.enable false Notebook enable cron scheduler feature ``` ### Questions: * Does the licenses files need update? N/A * Is there breaking changes for older versions? Yes * Does this needs documentation? Yes Author: Prabhjyot Singh Closes #2821 from prabhjyotsingh/ZEPPELIN-3271 and squashes the following commits: 93e9dc6 [Prabhjyot Singh] cron disable by default 359e687 [Prabhjyot Singh] add test-case for Cron disabled eaec944 [Prabhjyot Singh] refactor isCronSupported 6421439 [Prabhjyot Singh] setCronSupported property for import, clone and create new 48488cf [Prabhjyot Singh] option for 'zeppelin.notebook.cron.folders' 9491d3d [Prabhjyot Singh] log error before throwing ForbiddenException d25ebcc [Prabhjyot Singh] ZEPPELIN-3271: Option for disabling scheduler --- conf/zeppelin-site.xml.template | 11 ++ docs/usage/other_features/cron_scheduler.md | 8 ++ .../zeppelin/conf/ZeppelinConfiguration.java | 12 +- .../apache/zeppelin/rest/NotebookRestApi.java | 38 ++++-- .../zeppelin/socket/NotebookServer.java | 9 +- .../zeppelin/rest/ZeppelinRestApiTest.java | 63 +++++++-- .../src/app/notebook/notebook-actionBar.html | 2 +- .../org/apache/zeppelin/notebook/Note.java | 33 ++++- .../apache/zeppelin/notebook/Notebook.java | 53 ++++++-- .../zeppelin/notebook/NotebookTest.java | 121 +++++++++++++++--- 10 files changed, 285 insertions(+), 65 deletions(-) diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 9774f0d7c5c..d39b19c2669 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -540,5 +540,16 @@ origin Git repository remote + + + zeppelin.notebook.cron.enable + false + Notebook enable cron scheduler feature + + + zeppelin.notebook.cron.folders + + Notebook cron folders + --> diff --git a/docs/usage/other_features/cron_scheduler.md b/docs/usage/other_features/cron_scheduler.md index e8a9975a0ed..c7fc2844574 100644 --- a/docs/usage/other_features/cron_scheduler.md +++ b/docs/usage/other_features/cron_scheduler.md @@ -50,3 +50,11 @@ You can set the cron executing user by filling in this form and press the enter When this checkbox is set to "on", the interpreters which are binded to the notebook are stopped automatically after the cron execution. This feature is useful if you want to release the interpreter resources after the cron execution. > **Note**: A cron execution is skipped if one of the paragraphs is in a state of `RUNNING` or `PENDING` no matter whether it is executed automatically (i.e. by the cron scheduler) or manually by a user opening this notebook. + +### Enable cron + +Set property **zeppelin.notebook.cron.enable** to **true** in `$ZEPPELIN_HOME/conf/zeppelin-site.xml` to enable Cron feature. + +### Run cron selectively on folders + +In `$ZEPPELIN_HOME/conf/zeppelin-site.xml` make sure the property **zeppelin.notebook.cron.enable** is set to **true**, and then set property **zeppelin.notebook.cron.folders** to the desired folder as comma-separated values, e.g. `*yst*, Sys?em, System`. This property accepts wildcard and joker. diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 5beb2c70e8e..e0ebfa2c643 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -581,6 +581,14 @@ public String getZeppelinNotebookGitRemoteOrigin() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ORIGIN); } + public Boolean isZeppelinNotebookCronEnable() { + return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_CRON_ENABLE); + } + + public String getZeppelinNotebookCronFolders() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_CRON_FOLDERS); + } + public Map dumpConfigurations(ZeppelinConfiguration conf, ConfigurationKeyPredicate predicate) { Map configurations = new HashMap<>(); @@ -771,7 +779,9 @@ public enum ConfVars { ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL("zeppelin.notebook.git.remote.url", ""), ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME("zeppelin.notebook.git.remote.username", "token"), ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN("zeppelin.notebook.git.remote.access-token", ""), - ZEPPELIN_NOTEBOOK_GIT_REMOTE_ORIGIN("zeppelin.notebook.git.remote.origin", "origin"); + ZEPPELIN_NOTEBOOK_GIT_REMOTE_ORIGIN("zeppelin.notebook.git.remote.origin", "origin"), + ZEPPELIN_NOTEBOOK_CRON_ENABLE("zeppelin.notebook.cron.enable", false), + ZEPPELIN_NOTEBOOK_CRON_FOLDERS("zeppelin.notebook.cron.folders", null); private String varName; @SuppressWarnings("rawtypes") diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 2042c4c2024..8bfaef57184 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -33,6 +33,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.annotation.ZeppelinApi; +import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Notebook; @@ -139,7 +140,7 @@ private void checkIfUserIsOwner(String noteId, String errorMsg) { throw new ForbiddenException(errorMsg); } } - + /** * Check if the current user is either Owner or Writer for the given note. */ @@ -151,7 +152,7 @@ private void checkIfUserCanWrite(String noteId, String errorMsg) { throw new ForbiddenException(errorMsg); } } - + /** * Check if the current user can access (at least he have to be reader) the given note. */ @@ -175,19 +176,26 @@ private void checkIfUserCanRun(String noteId, String errorMsg) { throw new ForbiddenException(errorMsg); } } - + private void checkIfNoteIsNotNull(Note note) { if (note == null) { throw new NotFoundException("note not found"); } } - + + private void checkIfNoteSupportsCron(Note note) { + if (!note.isCronSupported(notebook.getConf())) { + LOG.error("Cron is not enabled from Zeppelin server"); + throw new ForbiddenException("Cron is not enabled from Zeppelin server"); + } + } + private void checkIfParagraphIsNotNull(Paragraph paragraph) { if (paragraph == null) { throw new NotFoundException("paragraph not found"); } } - + /** * set note authorization information */ @@ -205,7 +213,7 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re checkIfUserIsAnon(getBlockNotAuthenticatedUserErrorMsg()); checkIfUserIsOwner(noteId, ownerPermissionError(userAndRoles, notebookAuthorization.getOwners(noteId))); - + HashMap> permMap = gson.fromJson(req, new TypeToken>>() {}.getType()); Note note = notebook.getNote(noteId); @@ -380,6 +388,7 @@ public Response createNote(String message) throws IOException { note.setName(noteName); note.persist(subject); + note.setCronSupported(notebook.getConf()); notebookServer.broadcastNote(note); notebookServer.broadcastNoteList(subject, SecurityUtils.getRoles()); return new JsonResponse<>(Status.OK, "", note.getId()).build(); @@ -715,7 +724,7 @@ public Response getNoteJobStatus(@PathParam("noteId") String noteId) @GET @Path("job/{noteId}/{paragraphId}") @ZeppelinApi - public Response getNoteParagraphJobStatus(@PathParam("noteId") String noteId, + public Response getNoteParagraphJobStatus(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException { LOG.info("get note paragraph job status."); @@ -853,6 +862,7 @@ public Response registerCronJob(@PathParam("noteId") String noteId, String messa Note note = notebook.getNote(noteId); checkIfNoteIsNotNull(note); checkIfUserCanRun(noteId, "Insufficient privileges you cannot set a cron job for this note"); + checkIfNoteSupportsCron(note); if (!CronExpression.isValidExpression(request.getCronString())) { return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build(); @@ -879,11 +889,12 @@ public Response registerCronJob(@PathParam("noteId") String noteId, String messa public Response removeCronJob(@PathParam("noteId") String noteId) throws IOException, IllegalArgumentException { LOG.info("Remove cron job note {}", noteId); - + Note note = notebook.getNote(noteId); checkIfNoteIsNotNull(note); checkIfUserIsOwner(noteId, "Insufficient privileges you cannot remove this cron job from this note"); + checkIfNoteSupportsCron(note); Map config = note.getConfig(); config.put("cron", null); @@ -910,6 +921,7 @@ public Response getCronJob(@PathParam("noteId") String noteId) Note note = notebook.getNote(noteId); checkIfNoteIsNotNull(note); checkIfUserCanRead(noteId, "Insufficient privileges you cannot get cron information"); + checkIfNoteSupportsCron(note); return new JsonResponse<>(Status.OK, note.getConfig().get("cron")).build(); } @@ -1015,22 +1027,22 @@ private void initParagraph(Paragraph p, NewParagraphRequest request, String user checkIfParagraphIsNotNull(p); p.setTitle(request.getTitle()); p.setText(request.getText()); - Map< String, Object > config = request.getConfig(); - if ( config != null && !config.isEmpty()) { + Map config = request.getConfig(); + if (config != null && !config.isEmpty()) { configureParagraph(p, config, user); } } - private void configureParagraph(Paragraph p, Map< String, Object> newConfig, String user) + private void configureParagraph(Paragraph p, Map newConfig, String user) throws IOException { LOG.info("Configure Paragraph for user {}", user); if (newConfig == null || newConfig.isEmpty()) { LOG.warn("{} is trying to update paragraph {} of note {} with empty config", - user, p.getId(), p.getNote().getId()); + user, p.getId(), p.getNote().getId()); throw new BadRequestException("paragraph config cannot be empty"); } Map origConfig = p.getConfig(); - for ( final Map.Entry entry : newConfig.entrySet()){ + for (final Map.Entry entry : newConfig.entrySet()) { origConfig.put(entry.getKey(), entry.getValue()); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index d1cf9e5c081..113dfd60edd 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -869,7 +869,7 @@ private void sendHomeNote(NotebookSocket conn, HashSet userAndRoles, Not } private void updateNote(NotebookSocket conn, HashSet userAndRoles, Notebook notebook, - Message fromMessage) throws SchedulerException, IOException { + Message fromMessage) throws IOException { String noteId = (String) fromMessage.get("id"); String name = (String) fromMessage.get("name"); Map config = (Map) fromMessage.get("config"); @@ -887,6 +887,11 @@ private void updateNote(NotebookSocket conn, HashSet userAndRoles, Noteb Note note = notebook.getNote(noteId); if (note != null) { + if (!(Boolean) note.getConfig().get("isZeppelinNotebookCronEnable")) { + if (config.get("cron") != null) { + config.remove("cron"); + } + } boolean cronUpdated = isCronUpdated(config, note.getConfig()); note.setName(name); note.setConfig(config); @@ -950,6 +955,7 @@ private void renameNote(NotebookSocket conn, HashSet userAndRoles, Note note = notebook.getNote(noteId); if (note != null) { note.setName(name); + note.setCronSupported(notebook.getConf()); AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal); note.persist(subject); @@ -1040,6 +1046,7 @@ private void createNote(NotebookSocket conn, HashSet userAndRoles, Noteb noteName = "Note " + note.getId(); } note.setName(noteName); + note.setCronSupported(notebook.getConf()); } note.persist(subject); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 7e1a28a3855..da68087c970 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -29,6 +29,7 @@ import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.methods.PutMethod; import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.server.ZeppelinServer; @@ -375,11 +376,11 @@ public void testNoteJobs() throws IOException, InterruptedException { assertNotNull("can't create new note", note); note.setName("note for run test"); Paragraph paragraph = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - + Map config = paragraph.getConfig(); config.put("enabled", true); paragraph.setConfig(config); - + paragraph.setText("%md This is test paragraph."); note.persist(anonymous); String noteId = note.getId(); @@ -394,7 +395,7 @@ public void testNoteJobs() throws IOException, InterruptedException { break; } } - + // Call Run note jobs REST API PostMethod postNoteJobs = httpPost("/notebook/job/" + noteId, ""); assertThat("test note jobs run:", postNoteJobs, isAllowed()); @@ -403,21 +404,21 @@ public void testNoteJobs() throws IOException, InterruptedException { // Call Stop note jobs REST API DeleteMethod deleteNoteJobs = httpDelete("/notebook/job/" + noteId); assertThat("test note stop:", deleteNoteJobs, isAllowed()); - deleteNoteJobs.releaseConnection(); + deleteNoteJobs.releaseConnection(); Thread.sleep(1000); - + // Call Run paragraph REST API PostMethod postParagraph = httpPost("/notebook/job/" + noteId + "/" + paragraph.getId(), ""); assertThat("test paragraph run:", postParagraph, isAllowed()); - postParagraph.releaseConnection(); + postParagraph.releaseConnection(); Thread.sleep(1000); - + // Call Stop paragraph REST API DeleteMethod deleteParagraph = httpDelete("/notebook/job/" + noteId + "/" + paragraph.getId()); assertThat("test paragraph stop:", deleteParagraph, isAllowed()); - deleteParagraph.releaseConnection(); + deleteParagraph.releaseConnection(); Thread.sleep(1000); - + //cleanup ZeppelinServer.notebook.removeNote(note.getId(), anonymous); } @@ -514,7 +515,7 @@ public void testJobs() throws InterruptedException, IOException{ note.setName("note for run test"); Paragraph paragraph = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); paragraph.setText("%md This is test paragraph."); - + Map config = paragraph.getConfig(); config.put("enabled", true); paragraph.setConfig(config); @@ -526,20 +527,20 @@ public void testJobs() throws InterruptedException, IOException{ PostMethod postCron = httpPost("/notebook/cron/notexistnote", jsonRequest); assertThat("", postCron, isNotFound()); postCron.releaseConnection(); - + // right cron expression. postCron = httpPost("/notebook/cron/" + note.getId(), jsonRequest); assertThat("", postCron, isAllowed()); postCron.releaseConnection(); Thread.sleep(1000); - + // wrong cron expression. jsonRequest = "{\"cron\":\"a * * * * ?\" }"; postCron = httpPost("/notebook/cron/" + note.getId(), jsonRequest); assertThat("", postCron, isBadRequest()); postCron.releaseConnection(); Thread.sleep(1000); - + // remove cron job. DeleteMethod deleteCron = httpDelete("/notebook/cron/" + note.getId()); assertThat("", deleteCron, isAllowed()); @@ -547,6 +548,42 @@ public void testJobs() throws InterruptedException, IOException{ ZeppelinServer.notebook.removeNote(note.getId(), anonymous); } + @Test + public void testCronDisable() throws InterruptedException, IOException{ + // create a note and a paragraph + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_CRON_ENABLE.getVarName(), "false"); + Note note = ZeppelinServer.notebook.createNote(anonymous); + + note.setName("note for run test"); + Paragraph paragraph = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + paragraph.setText("%md This is test paragraph."); + + Map config = paragraph.getConfig(); + config.put("enabled", true); + paragraph.setConfig(config); + + note.runAll(AuthenticationInfo.ANONYMOUS, false); + + String jsonRequest = "{\"cron\":\"* * * * * ?\" }"; + // right cron expression. + PostMethod postCron = httpPost("/notebook/cron/" + note.getId(), jsonRequest); + assertThat("", postCron, isForbidden()); + postCron.releaseConnection(); + + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_CRON_ENABLE.getVarName(), "true"); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_CRON_FOLDERS.getVarName(), "System/*"); + + note.setName("System/test2"); + note.runAll(AuthenticationInfo.ANONYMOUS, false); + postCron = httpPost("/notebook/cron/" + note.getId(), jsonRequest); + assertThat("", postCron, isAllowed()); + postCron.releaseConnection(); + Thread.sleep(1000); + + System.clearProperty(ConfVars.ZEPPELIN_NOTEBOOK_CRON_FOLDERS.getVarName()); + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + @Test public void testRegressionZEPPELIN_527() throws IOException { Note note = ZeppelinServer.notebook.createNote(anonymous); diff --git a/zeppelin-web/src/app/notebook/notebook-actionBar.html b/zeppelin-web/src/app/notebook/notebook-actionBar.html index f8ff830846c..b4add9375a5 100644 --- a/zeppelin-web/src/app/notebook/notebook-actionBar.html +++ b/zeppelin-web/src/app/notebook/notebook-actionBar.html @@ -249,7 +249,7 @@

    -
    +
    paragraphs = oldNote.getParagraphs(); for (Paragraph p : paragraphs) { newNote.addCloneParagraph(p); @@ -222,6 +245,7 @@ public Note cloneNote(String sourceNoteId, String newNoteName, AuthenticationInf } else { newNote.setName("Note " + newNote.getId()); } + newNote.setCronSupported(getConf()); // Copy the interpreter bindings List boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId()); bindInterpretersToNote(subject.getUser(), newNote.getId(), boundInterpreterSettingsIds); @@ -498,6 +522,7 @@ public Note loadNoteFromRepo(String id, AuthenticationInfo subject) { note.setJobListenerFactory(jobListenerFactory); note.setNotebookRepo(notebookRepo); note.setRevisionSupported(notebookRepo); + note.setCronSupported(getConf()); Map angularObjectSnapshot = new HashMap<>(); @@ -899,6 +924,11 @@ public void execute(JobExecutionContext context) throws JobExecutionException { return; } + if (!note.isCronSupported(notebook.getConf())) { + logger.warn("execution of the cron job is skipped cron is not enabled from Zeppelin server"); + return; + } + note.runAll(); boolean releaseResource = false; @@ -941,6 +971,11 @@ public void refreshCron(String id) { return; } + if (!note.isCronSupported(getConf())) { + logger.warn("execution of the cron job is skipped cron is not enabled from Zeppelin server"); + return; + } + String cronExpr = (String) note.getConfig().get("cron"); if (cronExpr == null || cronExpr.trim().length() == 0) { return; diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 83c09327c35..02490ea685c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -86,6 +86,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener public void setUp() throws Exception { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true"); System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2"); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_CRON_ENABLE.getVarName(), "true"); super.setUp(); schedulerFactory = SchedulerFactory.singleton(); @@ -226,7 +227,7 @@ public void testLoadAllNotes() { fail("Subject is non-emtpy anonymous, shouldn't fail"); } } - + @Test public void testPersist() throws IOException, SchedulerException, RepositoryException { Note note = notebook.createNote(anonymous); @@ -434,13 +435,93 @@ private void executeNewParagraphByCron(Note note, String cron) { notebook.refreshCron(note.getId()); } + @Test + public void testScheduleDisabled() throws InterruptedException, IOException { + + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_CRON_ENABLE.getVarName(), "false"); + try { + final int timeout = 10; + final String everySecondCron = "* * * * * ?"; + final CountDownLatch jobsToExecuteCount = new CountDownLatch(5); + final Note note = notebook.createNote(anonymous); + + executeNewParagraphByCron(note, everySecondCron); + afterStatusChangedListener = new StatusChangedListener() { + @Override + public void onStatusChanged(Job job, Status before, Status after) { + if (after == Status.FINISHED) { + jobsToExecuteCount.countDown(); + } + } + }; + + //This job should not run because "ZEPPELIN_NOTEBOOK_CRON_ENABLE" is set to false + assertFalse(jobsToExecuteCount.await(timeout, TimeUnit.SECONDS)); + + terminateScheduledNote(note); + afterStatusChangedListener = null; + } finally { + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_CRON_ENABLE.getVarName(), "true"); + } + } + + @Test + public void testScheduleDisabledWithName() throws InterruptedException, IOException { + + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_CRON_FOLDERS.getVarName(), "System/*"); + try { + final int timeout = 10; + final String everySecondCron = "* * * * * ?"; + final CountDownLatch jobsToExecuteCount = new CountDownLatch(5); + final Note note = notebook.createNote(anonymous); + + executeNewParagraphByCron(note, everySecondCron); + afterStatusChangedListener = new StatusChangedListener() { + @Override + public void onStatusChanged(Job job, Status before, Status after) { + if (after == Status.FINISHED) { + jobsToExecuteCount.countDown(); + } + } + }; + + //This job should not run because it's name does not matches "ZEPPELIN_NOTEBOOK_CRON_FOLDERS" + assertFalse(jobsToExecuteCount.await(timeout, TimeUnit.SECONDS)); + + terminateScheduledNote(note); + afterStatusChangedListener = null; + + final Note noteNameSystem = notebook.createNote(anonymous); + noteNameSystem.setName("System/test1"); + final CountDownLatch jobsToExecuteCountNameSystem = new CountDownLatch(5); + + executeNewParagraphByCron(noteNameSystem, everySecondCron); + afterStatusChangedListener = new StatusChangedListener() { + @Override + public void onStatusChanged(Job job, Status before, Status after) { + if (after == Status.FINISHED) { + jobsToExecuteCountNameSystem.countDown(); + } + } + }; + + //This job should run because it's name contains "System/" + assertTrue(jobsToExecuteCountNameSystem.await(timeout, TimeUnit.SECONDS)); + + terminateScheduledNote(noteNameSystem); + afterStatusChangedListener = null; + } finally { + System.clearProperty(ConfVars.ZEPPELIN_NOTEBOOK_CRON_FOLDERS.getVarName()); + } + } + private void terminateScheduledNote(Note note) { note.getConfig().remove("cron"); notebook.refreshCron(note.getId()); notebook.removeNote(note.getId(), anonymous); } - + @Test public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{ // create a note and a paragraph @@ -859,9 +940,9 @@ public void testAuthorizationRoles() throws IOException { // set admin roles for both user1 and user2 notebookAuthorization.setRoles(user1, roles); notebookAuthorization.setRoles(user2, roles); - + Note note = notebook.createNote(new AuthenticationInfo(user1)); - + // check that user1 is owner, reader, runner and writer assertEquals(notebookAuthorization.isOwner(note.getId(), Sets.newHashSet(user1)), true); @@ -871,7 +952,7 @@ public void testAuthorizationRoles() throws IOException { Sets.newHashSet(user2)), true); assertEquals(notebookAuthorization.isWriter(note.getId(), Sets.newHashSet(user1)), true); - + // since user1 and user2 both have admin role, user2 will be reader and writer as well assertEquals(notebookAuthorization.isOwner(note.getId(), Sets.newHashSet(user2)), false); @@ -881,14 +962,14 @@ public void testAuthorizationRoles() throws IOException { Sets.newHashSet(user2)), true); assertEquals(notebookAuthorization.isWriter(note.getId(), Sets.newHashSet(user2)), true); - + // check that user1 has note listed in his workbench Set user1AndRoles = notebookAuthorization.getRoles(user1); user1AndRoles.add(user1); List user1Notes = notebook.getAllNotes(user1AndRoles); assertEquals(user1Notes.size(), 1); assertEquals(user1Notes.get(0).getId(), note.getId()); - + // check that user2 has note listed in his workbench because of admin role Set user2AndRoles = notebookAuthorization.getRoles(user2); user2AndRoles.add(user2); @@ -896,7 +977,7 @@ public void testAuthorizationRoles() throws IOException { assertEquals(user2Notes.size(), 1); assertEquals(user2Notes.get(0).getId(), note.getId()); } - + @Test public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException, IOException, InterpreterException { @@ -1225,7 +1306,7 @@ public void testGetAllNotesWithDifferentPermissions() throws IOException { notes2 = notebook.getAllNotes(user2); assertEquals(notes1.size(), 1); assertEquals(notes2.size(), 1); - + notebook.getNotebookAuthorization().setReaders(note.getId(), Sets.newHashSet("user1")); //note is public since writers empty notes1 = notebook.getAllNotes(user1); @@ -1238,7 +1319,7 @@ public void testGetAllNotesWithDifferentPermissions() throws IOException { notes2 = notebook.getAllNotes(user2); assertEquals(notes1.size(), 1); assertEquals(notes2.size(), 1); - + notebook.getNotebookAuthorization().setWriters(note.getId(), Sets.newHashSet("user1")); notes1 = notebook.getAllNotes(user1); notes2 = notebook.getAllNotes(user2); @@ -1250,19 +1331,19 @@ public void testGetAllNotesWithDifferentPermissions() throws IOException { public void testPublicPrivateNewNote() throws IOException, SchedulerException { HashSet user1 = Sets.newHashSet("user1"); HashSet user2 = Sets.newHashSet("user2"); - + // case of public note assertTrue(conf.isNotebookPublic()); assertTrue(notebookAuthorization.isPublic()); - + List notes1 = notebook.getAllNotes(user1); List notes2 = notebook.getAllNotes(user2); assertEquals(notes1.size(), 0); assertEquals(notes2.size(), 0); - + // user1 creates note Note notePublic = notebook.createNote(new AuthenticationInfo("user1")); - + // both users have note notes1 = notebook.getAllNotes(user1); notes2 = notebook.getAllNotes(user2); @@ -1270,7 +1351,7 @@ public void testPublicPrivateNewNote() throws IOException, SchedulerException { assertEquals(notes2.size(), 1); assertEquals(notes1.get(0).getId(), notePublic.getId()); assertEquals(notes2.get(0).getId(), notePublic.getId()); - + // user1 is only owner assertEquals(notebookAuthorization.getOwners(notePublic.getId()).size(), 1); assertEquals(notebookAuthorization.getReaders(notePublic.getId()).size(), 0); @@ -1283,13 +1364,13 @@ public void testPublicPrivateNewNote() throws IOException, SchedulerException { assertFalse(conf2.isNotebookPublic()); // notebook authorization reads from conf, so no need to re-initilize assertFalse(notebookAuthorization.isPublic()); - + // check that still 1 note per user notes1 = notebook.getAllNotes(user1); notes2 = notebook.getAllNotes(user2); assertEquals(notes1.size(), 1); assertEquals(notes2.size(), 1); - + // create private note Note notePrivate = notebook.createNote(new AuthenticationInfo("user1")); @@ -1299,18 +1380,18 @@ public void testPublicPrivateNewNote() throws IOException, SchedulerException { assertEquals(notes1.size(), 2); assertEquals(notes2.size(), 1); assertEquals(true, notes1.contains(notePrivate)); - + // user1 have all rights assertEquals(notebookAuthorization.getOwners(notePrivate.getId()).size(), 1); assertEquals(notebookAuthorization.getReaders(notePrivate.getId()).size(), 1); assertEquals(notebookAuthorization.getRunners(notePrivate.getId()).size(), 1); assertEquals(notebookAuthorization.getWriters(notePrivate.getId()).size(), 1); - + //set back public to true System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true"); ZeppelinConfiguration.create(); } - + private void delete(File file){ if(file.isFile()) file.delete(); else if(file.isDirectory()){ From fa9f88ba4fbd8f8faa0ad7cf762cfaf795eb0e51 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 28 Feb 2018 16:55:52 +0800 Subject: [PATCH 055/386] ZEPPELIN-3273. SparkRInterpreter doesn't work in yarn mode ### What is this PR for? The root cause is the sparkr package is not distributed correctly. This PR correct the distributed file name and also add unit test for this. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3273 ### How should this be tested? * Unit test added ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2823 from zjffdu/ZEPPELIN-3273 and squashes the following commits: 7f6c3db [Jeff Zhang] ZEPPELIN-3273. SparkRInterpreter doesn't work in yarn mode --- .travis.yml | 4 +-- .../spark/IPySparkInterpreterTest.java | 28 +++++++++++-------- .../launcher/SparkInterpreterLauncher.java | 3 +- .../interpreter/SparkInterpreterModeTest.java | 16 +++++++++++ 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index c31694a4dbc..bc568fce2e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,9 +66,7 @@ matrix: - sudo: required jdk: "oraclejdk8" dist: trusty - addons: - firefox: "31.0" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk8" diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java index 46a3a7277fe..17c2af8c370 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java @@ -103,6 +103,7 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx InterpreterResult result = iPySparkInterpreter.interpret("sc.version", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + String sparkVersion = context.out.getInterpreterResultMessages().get(0).getData(); // spark url is sent verify(mockRemoteEventClient).onMetaInfosReceived(any(Map.class)); @@ -117,18 +118,17 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx // spark sql context = getInterpreterContext(); - if (interpreterResultMessages.get(0).getData().startsWith("'1.") || - interpreterResultMessages.get(0).getData().startsWith("u'1.")) { + if (!isSpark2(sparkVersion)) { result = iPySparkInterpreter.interpret("df = sqlContext.createDataFrame([(1,'a'),(2,'b')])\ndf.show()", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.getInterpreterResultMessages(); assertEquals( "+---+---+\n" + - "| _1| _2|\n" + - "+---+---+\n" + - "| 1| a|\n" + - "| 2| b|\n" + - "+---+---+\n\n", interpreterResultMessages.get(0).getData()); + "| _1| _2|\n" + + "+---+---+\n" + + "| 1| a|\n" + + "| 2| b|\n" + + "+---+---+\n\n", interpreterResultMessages.get(0).getData()); context = getInterpreterContext(); result = iPySparkInterpreter.interpret("z.show(df)", context); @@ -144,11 +144,11 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx interpreterResultMessages = context.out.getInterpreterResultMessages(); assertEquals( "+---+---+\n" + - "| _1| _2|\n" + - "+---+---+\n" + - "| 1| a|\n" + - "| 2| b|\n" + - "+---+---+\n\n", interpreterResultMessages.get(0).getData()); + "| _1| _2|\n" + + "+---+---+\n" + + "| 1| a|\n" + + "| 2| b|\n" + + "+---+---+\n\n", interpreterResultMessages.get(0).getData()); context = getInterpreterContext(); result = iPySparkInterpreter.interpret("z.show(df)", context); @@ -212,6 +212,10 @@ public void run() { assertTrue(interpreterResultMessages.get(0).getData().contains("(0, 100)")); } + private boolean isSpark2(String sparkVersion) { + return sparkVersion.startsWith("'2.") || sparkVersion.startsWith("u'2."); + } + private InterpreterContext getInterpreterContext() { InterpreterContext context = new InterpreterContext( "noteId", diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java index 3c5326f9635..688d95fa919 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java @@ -161,7 +161,8 @@ private void setupPropertiesForSparkR(Properties sparkProperties) { File sparkRPath = new File(sparkRBasePath, "sparkr.zip"); if (sparkRPath.exists() && sparkRPath.isFile()) { - mergeSparkProperty(sparkProperties, "spark.yarn.dist.archives", sparkRPath.getAbsolutePath()); + mergeSparkProperty(sparkProperties, "spark.yarn.dist.archives", + sparkRPath.getAbsolutePath() + "#sparkr"); } else { LOGGER.warn("sparkr.zip is not found, SparkR may not work."); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java index 930a26d9b62..22bb17e440f 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java @@ -52,6 +52,7 @@ private void testInterpreterBasics() throws IOException, InterpreterException { InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").getContext(); InterpreterResult interpreterResult = sparkInterpreter.interpret("sc.version", context); assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code); + String sparkVersion = interpreterResult.message().get(0).getData(); interpreterResult = sparkInterpreter.interpret("sc.range(1,10).sum()", context); assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code); assertTrue(interpreterResult.msg.get(0).getData().contains("45")); @@ -72,6 +73,17 @@ private void testInterpreterBasics() throws IOException, InterpreterException { assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code); assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType()); assertEquals("count(1)\n2\n", interpreterResult.message().get(0).getData()); + + // test SparkRInterpreter + Interpreter sparkrInterpreter = interpreterFactory.getInterpreter("user1", "note1", "spark.r"); + if (isSpark2(sparkVersion)) { + interpreterResult = sparkrInterpreter.interpret("df <- as.DataFrame(faithful)\nhead(df)", context); + } else { + interpreterResult = sparkrInterpreter.interpret("df <- createDataFrame(sqlContext, faithful)\nhead(df)", context); + } + assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code); + assertEquals(InterpreterResult.Type.TEXT, interpreterResult.message().get(0).getType()); + assertTrue(interpreterResult.message().get(0).getData().contains("eruptions waiting")); } @Test @@ -137,6 +149,10 @@ public void testYarnClusterMode() throws IOException, YarnException, Interrupted interpreterSettingManager.close(); } + private boolean isSpark2(String sparkVersion) { + return sparkVersion.startsWith("2."); + } + private String getPythonExec() throws IOException, InterruptedException { Process process = Runtime.getRuntime().exec(new String[]{"which", "python"}); if (process.waitFor() != 0) { From 21dc20d8864915475f58705d8bc2fc8678c22dbe Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Fri, 2 Mar 2018 09:47:56 +0800 Subject: [PATCH 056/386] ZEPPELIN-3278. Avoid duplicated interpreter setting ### What is this PR for? Straightforward bugfix for ZEPPELIN-3278. Just don't create interpreter setting instance when registering from template. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3278 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2824 from zjffdu/ZEPPELIN-3278 and squashes the following commits: 6ece9ca [Jeff Zhang] [ZEPPELIN-3278] Avoid duplicated interpreter setting --- .../interpreter/InterpreterSettingManager.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 0601c6ff5b4..711812e07f4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -192,13 +192,14 @@ private void initInterpreterSetting(InterpreterSetting interpreterSetting) { } /** - * Load interpreter setting from interpreter-setting.json + * Load interpreter setting from interpreter.json */ private void loadFromFile() throws IOException { InterpreterInfoSaving infoSaving = configStorage.loadInterpreterSettings(); if (infoSaving == null) { - // nothing to read + // it is fresh zeppelin instance if there's no interpreter.json, just create interpreter + // setting from interpreterSettingTemplates for (InterpreterSetting interpreterSettingTemplate : interpreterSettingTemplates.values()) { InterpreterSetting interpreterSetting = new InterpreterSetting(interpreterSettingTemplate); initInterpreterSetting(interpreterSetting); @@ -403,14 +404,10 @@ private void registerInterpreterSetting(List registeredIn .setIntepreterSettingManager(this) .create(); - LOGGER.info("Register InterpreterSettingTemplate & Create InterpreterSetting: {}", + LOGGER.info("Register InterpreterSettingTemplate: {}", interpreterSettingTemplate.getName()); interpreterSettingTemplates.put(interpreterSettingTemplate.getName(), interpreterSettingTemplate); - - InterpreterSetting interpreterSetting = new InterpreterSetting(interpreterSettingTemplate); - initInterpreterSetting(interpreterSetting); - interpreterSettings.put(interpreterSetting.getName(), interpreterSetting); } @VisibleForTesting From bfc93dc030f94be0ac624fa8d58279d553e3ff07 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Fri, 2 Mar 2018 18:57:10 +0800 Subject: [PATCH 057/386] ZEPPELIN-3281. Apply getRelativePath when it is LocalConfigStorage ### What is this PR for? When it is LocalConfigStorage, we should use getRelativePath to get the config path. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3281 ### How should this be tested? * Unit test added ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2831 from zjffdu/ZEPPELIN-3281 and squashes the following commits: 3438577 [Jeff Zhang] ZEPPELIN-3281. Apply getRelativePath when it is LocalConfigStorage --- .../apache/zeppelin/conf/ZeppelinConfiguration.java | 8 +++++++- .../apache/zeppelin/storage/LocalConfigStorage.java | 1 + .../zeppelin/conf/ZeppelinConfigurationTest.java | 12 ++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index e0ebfa2c643..81f9341e36e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -524,7 +524,13 @@ public String getConfigFSDir() { "conf directory " + ConfVars.ZEPPELIN_CONF_DIR.varName); return getConfDir(); } - return fsConfigDir; + if (getString(ConfVars.ZEPPELIN_CONFIG_STORAGE_CLASS) + .equals("org.apache.zeppelin.storage.LocalConfigStorage")) { + // only apply getRelativeDir when it is LocalConfigStorage + return getRelativeDir(fsConfigDir); + } else { + return fsConfigDir; + } } public List getAllowedOrigins() diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java index c1edbb51dd4..464d6ce212d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java @@ -50,6 +50,7 @@ public LocalConfigStorage(ZeppelinConfiguration zConf) { @Override public void save(InterpreterInfoSaving settingInfos) throws IOException { + LOGGER.info("Save Interpreter Setting to " + interpreterSettingPath.getAbsolutePath()); writeToFile(settingInfos.toJson(), interpreterSettingPath); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java index 2b50427f21f..1771fd39d40 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java @@ -22,6 +22,7 @@ import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.net.MalformedURLException; @@ -100,4 +101,15 @@ public void getPathTest() throws ConfigurationException { Assert.assertEquals("/usr/lib/zeppelin", conf.getZeppelinHome()); Assert.assertEquals("/usr/lib/zeppelin/conf", conf.getConfDir()); } + + @Test + public void getConfigFSPath() throws ConfigurationException { + System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), "/usr/lib/zeppelin"); + System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), "conf"); + ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml")); + assertEquals("/usr/lib/zeppelin/conf", conf.getConfigFSDir()); + + System.setProperty(ConfVars.ZEPPELIN_CONFIG_STORAGE_CLASS.getVarName(), "org.apache.zeppelin.storage.FileSystemConfigStorage"); + assertEquals("conf", conf.getConfigFSDir()); + } } From 63c53fcc5979b8c2cbd8bae13de2561078780637 Mon Sep 17 00:00:00 2001 From: Maksim Reshetov Date: Thu, 1 Mar 2018 23:34:20 +0500 Subject: [PATCH 058/386] [ZEPPELIN-3194][NEW-INTERPRETER] SAP Universe interpreter ### What is this PR for? New interpreter, SAP BusinessObjects (Universes). See [documentation](https://github.com/masyan/zeppelin/blob/9178d1cc80253b9ae58b988443b619127f42d945/docs/interpreter/sap.md) ### What type of PR is it? [Feature] ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-3194 ### How should this be tested? see screenshots ### Screenshots (if appropriate) Autocomplete + get results ![comletion_and_results](https://user-images.githubusercontent.com/705100/35770106-1fb3e9e6-0937-11e8-9ee4-df0c06a7b6c3.gif) Errors ![errors](https://user-images.githubusercontent.com/705100/35770108-23de9674-0937-11e8-89e1-a0d3380e88bb.gif) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Maksim Reshetov Closes #2763 from masyan/ZEPPELIN-3194 and squashes the following commits: 696b7dd54 [Maksim Reshetov] Merge remote-tracking branch 'upstream/master' into ZEPPELIN-3194 1e25fd1ee [Maksim Reshetov] [ZEPPELIN-3194] refactor parser os query. add tests 24336a855 [Maksim Reshetov] [ZEPPELIN-3194] fix parser for result objects and conditions 9178d1cc8 [Maksim Reshetov] [ZEPPELIN-3194] docs ce97551b9 [Maksim Reshetov] Merge remote-tracking branch 'upstream/master' into ZEPPELIN-3194 0784ba030 [Maksim Reshetov] [ZEPPELIN-3194] docs e00ace2e3 [Maksim Reshetov] [ZEPPELIN-3194] completer tests 1aeb7ad0f [Maksim Reshetov] [ZEPPELIN-3194] pom license c1c09af5c [Maksim Reshetov] [ZEPPELIN-3194] docs fix + unit tests for util f6b71f9c2 [Maksim Reshetov] [ZEPPELIN-3194] is null fix --- docs/index.md | 1 + docs/interpreter/sap.md | 123 +++ pom.xml | 1 + sap/pom.xml | 86 ++ .../zeppelin/sap/UniverseInterpreter.java | 215 +++++ .../zeppelin/sap/universe/UniverseClient.java | 733 ++++++++++++++++++ .../sap/universe/UniverseCompleter.java | 344 ++++++++ .../sap/universe/UniverseException.java | 38 + .../zeppelin/sap/universe/UniverseInfo.java | 60 ++ .../sap/universe/UniverseNodeInfo.java | 85 ++ .../universe/UniverseNodeInfoCompleter.java | 183 +++++ .../zeppelin/sap/universe/UniverseQuery.java | 53 ++ .../sap/universe/UniverseQueryPrompt.java | 110 +++ .../zeppelin/sap/universe/UniverseUtil.java | 643 +++++++++++++++ .../main/resources/interpreter-setting.json | 42 + sap/src/main/resources/universe.keywords | 1 + .../sap/universe/UniverseCompleterTest.java | 134 ++++ .../sap/universe/UniverseUtilTest.java | 371 +++++++++ .../zeppelin/completer/CompletionType.java | 3 +- .../zeppelin/conf/ZeppelinConfiguration.java | 3 +- .../paragraph/paragraph.controller.js | 2 +- 21 files changed, 3228 insertions(+), 3 deletions(-) create mode 100644 docs/interpreter/sap.md create mode 100644 sap/pom.xml create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseClient.java create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseCompleter.java create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseException.java create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseInfo.java create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseNodeInfo.java create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseNodeInfoCompleter.java create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseQuery.java create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseQueryPrompt.java create mode 100644 sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseUtil.java create mode 100644 sap/src/main/resources/interpreter-setting.json create mode 100644 sap/src/main/resources/universe.keywords create mode 100644 sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseCompleterTest.java create mode 100644 sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseUtilTest.java diff --git a/docs/index.md b/docs/index.md index 3d42735d092..f00571e5bed 100644 --- a/docs/index.md +++ b/docs/index.md @@ -150,6 +150,7 @@ limitations under the License. * [Postgresql, HAWQ](./interpreter/postgresql.html) * [Python](./interpreter/python.html) * [R](./interpreter/r.html) + * [SAP](./interpreter/sap.html) * [Scalding](./interpreter/scalding.html) * [Scio](./interpreter/scio.html) * [Shell](./interpreter/Shell.html) diff --git a/docs/interpreter/sap.md b/docs/interpreter/sap.md new file mode 100644 index 00000000000..be05aee8f26 --- /dev/null +++ b/docs/interpreter/sap.md @@ -0,0 +1,123 @@ +--- + +layout: page + +title: "SAP BusinessObjects Interpreter for Apache Zeppelin" + +description: "SAP BusinessObjects BI platform can simplify the lives of business users and IT staff. SAP BusinessObjects is based on universes. The universe contains dual-semantic layer model. The users make queries upon universes. This interpreter is new interface for universes." + +group: interpreter + +--- + + + +{% include JB/setup %} + +# SAP BusinessObjects (Universe) Interpreter for Apache Zeppelin + +
    + +## Overview + +[SAP BusinessObjects BI platform (universes)](https://help.sap.com/viewer/p/SAP_BUSINESSOBJECTS_BUSINESS_INTELLIGENCE_PLATFORM) can simplify the lives of business users and IT staff. SAP BusinessObjects is based on universes. The universe contains dual-semantic layer model. The users make queries upon universes. This interpreter is new interface for universes. + +*Disclaimer* SAP interpreter is not official interpreter for SAP BusinessObjects BI platform. It uses [BI Semantic Layer REST API](https://help.sap.com/viewer/5431204882b44fc98d56bd752e69f132/4.2.5/en-US/ec54808e6fdb101497906a7cb0e91070.html) + +This interpreter is not directly supported by SAP AG. + +Tested with versions 4.2SP3 (14.2.3.2220) and 4.2SP5. There is no support for filters in UNX-universes converted from old UNV format. + +The universe name must be unique. + +## Configuring SAP Universe Interpreter + +At the "Interpreters" menu, you can edit SAP interpreter or create new one. Zeppelin provides these properties for SAP. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Property NameValueDescription
    universe.api.urlhttp://localhost:6405/biprwsThe base url for the SAP BusinessObjects BI platform. You have to edit "localhost" that you may use (ex. http://0.0.0.0:6405/biprws)
    universe.authTypesecEnterpriseThe type of authentication for API of Universe. Available values: secEnterprise, secLDAP, secWinAD, secSAPR3
    universe.passwordThe BI platform user password
    universe.userAdministratorThe BI platform user login
    + +![SAP Interpreter Setting]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/sap-interpreter-setting.png) + +### How to use + +
  • Choose the universe +
  • Choose dimensions and measures in `select` statement +
  • Define conditions in `where` statement +You can compare two dimensions/measures or use Filter (without value). +Dimesions/Measures can be compared with static values, may be `is null` or `is not null`, contains or not in list. +Available the nested conditions (using braces "()"). "and" operator have more priority than "or". + + +If generated query contains promtps, then promtps will appear as dynamic form after paragraph submitting. + +Example query + +``` +%sap + +universe [Universe Name]; + +select + + [Folder1].[Dimension2], + + [Folder2].[Dimension3], + + [Measure1] + +where + + [Filter1] + + and [Date] > '2018-01-01 00:00:00' + + and [Folder1].[Dimension4] is not null + + and [Folder1].[Dimension5] in ('Value1', 'Value2'); +``` \ No newline at end of file diff --git a/pom.xml b/pom.xml index 6ce20aa6beb..dc6d64b89a3 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,7 @@ zeppelin-server zeppelin-jupyter zeppelin-distribution + sap diff --git a/sap/pom.xml b/sap/pom.xml new file mode 100644 index 00000000000..f0bf3178488 --- /dev/null +++ b/sap/pom.xml @@ -0,0 +1,86 @@ + + + + + 4.0.0 + + + interpreter-parent + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../interpreter-parent + + + org.apache.zeppelin + sap + jar + 0.9.0-SNAPSHOT + Zeppelin: Sap + Zeppelin SAP support + + + sap + + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + junit + junit + test + + + + org.mockito + mockito-all + test + + + + + + + maven-enforcer-plugin + + + maven-dependency-plugin + + + maven-resources-plugin + + + + + \ No newline at end of file diff --git a/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java b/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java new file mode 100644 index 00000000000..1fdb6d6392d --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap; + +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.sap.universe.*; + + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * SAP Universe interpreter for Zeppelin. + */ +public class UniverseInterpreter extends Interpreter { + + public UniverseInterpreter(Properties properties) { + super(properties); + } + + private UniverseClient client; + private UniverseUtil universeUtil; + private UniverseCompleter universeCompleter; + + private static final String EMPTY_COLUMN_VALUE = StringUtils.EMPTY; + private static final char WHITESPACE = ' '; + private static final char NEWLINE = '\n'; + private static final char TAB = '\t'; + private static final String TABLE_MAGIC_TAG = "%table "; + + @Override + public void open() throws InterpreterException { + String user = getProperty("universe.user"); + String password = getProperty("universe.password"); + String apiUrl = getProperty("universe.api.url"); + String authType = getProperty("universe.authType"); + this.client = new UniverseClient(user, password, apiUrl, authType); + this.universeUtil = new UniverseUtil(); + } + + @Override + public void close() throws InterpreterException { + try { + client.close(); + } catch (Exception e) { + throw new InterpreterException(e.getCause()); + } + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { + try { + InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); + String paragraphId = context.getParagraphId(); + String token = client.getToken(paragraphId); + client.loadUniverses(token); + UniverseQuery universeQuery = universeUtil.convertQuery(st, client, token); + String queryId = client.createQuery(token, universeQuery); + // process parameters + List parameters = client.getParameters(token, queryId); + + for (UniverseQueryPrompt parameter : parameters) { + Object value = context.getGui().getParams().get(parameter.getName()); + if (value != null) { + parameter.setValue(value.toString()); + } + context.getGui().textbox(parameter.getName(), StringUtils.EMPTY); + } + + if (!parameters.isEmpty() && parameters.size() != context.getGui().getParams().size()) { + client.deleteQuery(token, queryId); + interpreterResult.add("Set parameters"); + return interpreterResult; + } + + if (!parameters.isEmpty()) { + client.setParametersValues(token, queryId, parameters); + } + + // get results + List> results = client.getResults(token, queryId); + String table = formatResults(results); + // remove query + client.deleteQuery(token, queryId); + interpreterResult.add(table); + return interpreterResult; + } catch (Exception e) { + throw new InterpreterException(e.getMessage(), e); + } finally { + try { + client.closeSession(context.getParagraphId()); + } catch (Exception e) { + logger.error("Error close SAP session", e ); + } + } + } + + @Override + public void cancel(InterpreterContext context) throws InterpreterException { + try { + client.closeSession(context.getParagraphId()); + } catch (Exception e) { + logger.error("Error close SAP session", e ); + } + } + + @Override + public FormType getFormType() throws InterpreterException { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) throws InterpreterException { + return 0; + } + + @Override + public List completion(String buf, int cursor, + InterpreterContext interpreterContext) + throws InterpreterException { + List candidates = new ArrayList<>(); + + try { + universeCompleter = createOrUpdateUniverseCompleter(interpreterContext, buf, cursor); + universeCompleter.complete(buf, cursor, candidates); + } catch (UniverseException e) { + logger.error("Error update completer", e ); + } + + return candidates; + } + + private String formatResults(List> results) { + StringBuilder msg = new StringBuilder(); + if (results != null) { + msg.append(TABLE_MAGIC_TAG); + for (int i = 0; i < results.size(); i++) { + List items = results.get(i); + for (int j = 0; j < items.size(); j++) { + if (j > 0) { + msg.append(TAB); + } + msg.append(replaceReservedChars(items.get(j))); + } + msg.append(NEWLINE); + } + } + + return msg.toString(); + } + + private String replaceReservedChars(String str) { + if (str == null) { + return EMPTY_COLUMN_VALUE; + } + return str.replace(TAB, WHITESPACE).replace(NEWLINE, WHITESPACE); + } + + private UniverseCompleter createOrUpdateUniverseCompleter(InterpreterContext interpreterContext, + final String buf, final int cursor) + throws UniverseException { + final UniverseCompleter completer; + if (universeCompleter == null) { + completer = new UniverseCompleter(3600); + } else { + completer = universeCompleter; + } + try { + final String token = client.getToken(interpreterContext.getParagraphId()); + ExecutorService executorService = Executors.newFixedThreadPool(1); + executorService.execute(new Runnable() { + @Override + public void run() { + completer.createOrUpdate(client, token, buf, cursor); + } + }); + + executorService.shutdown(); + + executorService.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + logger.warn("Completion timeout", e); + } finally { + try { + client.closeSession(interpreterContext.getParagraphId()); + } catch (Exception e) { + logger.error("Error close SAP session", e ); + } + } + return completer; + } +} diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseClient.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseClient.java new file mode 100644 index 00000000000..b60eac47704 --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseClient.java @@ -0,0 +1,733 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap.universe; + +import com.sun.org.apache.xpath.internal.NodeSet; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.*; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * Client for API SAP Universe + */ +public class UniverseClient { + + private static Logger logger = LoggerFactory.getLogger(UniverseClient.class); + private static final String TOKEN_HEADER = "X-SAP-LogonToken"; + private static final String EL_FOLDER = "folder"; + private static final String EL_ITEM = "item"; + private static final String EL_NAME = "name"; + private static final String EL_PATH = "path"; + private static final String EL_ID = "id"; + private static final String EL_TECH_NAME = "technicalName"; + private static final String EL_ANSWER = "answer"; + private static final String EL_INFO = "info"; + private Map tokens = new HashMap(); + private static final long DAY = 1000 * 60 * 60 * 24; + private CloseableHttpClient httpClient; + private String user; + private String password; + private String apiUrl; + private String authType; + private Header[] commonHeaders = { + new BasicHeader("Accept", "application/xml"), + new BasicHeader("Content-Type", "application/xml") + }; + // + private final Map universesMap = new ConcurrentHashMap(); + private final Map> universeInfosMap = + new ConcurrentHashMap(); + // for update the data (which was not updated a long time) + private long universesUpdated = 0; + private Map universesInfoUpdatedMap = new HashMap<>(); + + private final String loginRequestTemplate = "\n" + + "%s\n" + + "%s\n" + + "%s\n" + ""; + private final String createQueryRequestTemplate = + "\n" + + "\n" + + " \n%s\n" + + " %s\n" + + "\n" + + "\n" + + "\n"; + private final String filterPartTemplate = "%s\n"; + private final String errorMessageTemplate = "%s\n\n%s"; + private final String parameterTemplate = "\n" + + "%s\n" + + "%s\n" + + "%s\n" + + "%s\n" + + "\n"; + private final String parameterAnswerTemplate = "\n" + + " \n" + + " \n" + " " + + " %s\n" + + " \n" + + " \n"; + + public UniverseClient(String user, String password, String apiUrl, String authType) { + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(20 * 60 * 1000) + .setSocketTimeout(20 * 60 * 1000) + .build(); + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + cm.setMaxTotal(100); + cm.setDefaultMaxPerRoute(100); + cm.closeIdleConnections(10, TimeUnit.MINUTES); + httpClient = HttpClientBuilder.create() + .setConnectionManager(cm) + .setDefaultRequestConfig(requestConfig) + .build(); + + this.user = user; + this.password = password; + this.authType = authType; + if (StringUtils.isNotBlank(apiUrl)) { + this.apiUrl = apiUrl.replaceAll("/$", ""); + } + } + + public void close() throws UniverseException { + for (String s : tokens.keySet()) { + closeSession(s); + } + try { + httpClient.close(); + } catch (Exception e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(close all): Error close HTTP client", ExceptionUtils.getStackTrace(e))); + } + + } + + public String createQuery(String token, UniverseQuery query) throws UniverseException { + try { + HttpPost httpPost = new HttpPost(String.format("%s%s", apiUrl, "/sl/v1/queries")); + setHeaders(httpPost, token); + String where = StringUtils.isNotBlank(query.getWhere()) ? + String.format(filterPartTemplate, query.getWhere()) : StringUtils.EMPTY; + httpPost.setEntity(new StringEntity( + String.format(createQueryRequestTemplate, query.getUniverseInfo().getType(), + query.getUniverseInfo().getId(), query.getSelect(), where), "UTF-8")); + HttpResponse response = httpClient.execute(httpPost); + + if (response.getStatusLine().getStatusCode() == 200) { + return getValue(EntityUtils.toString(response.getEntity()), "//success/id"); + } + + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(create query): Request failed\n", EntityUtils.toString(response.getEntity()))); + } catch (IOException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(create query): Request failed", ExceptionUtils.getStackTrace(e))); + } catch (ParserConfigurationException | SAXException | XPathExpressionException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(create query): Response processing failed", ExceptionUtils.getStackTrace(e))); + } + } + + public void deleteQuery(String token, String queryId) throws UniverseException { + try { + if (StringUtils.isNotBlank(queryId)) { + HttpDelete httpDelete = new HttpDelete(String.format("%s%s%s", apiUrl, "/sl/v1/queries/", + queryId)); + setHeaders(httpDelete, token); + httpClient.execute(httpDelete); + } + } catch (Exception e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(delete query): Request failed", ExceptionUtils.getStackTrace(e))); + } + } + + public List> getResults(String token, String queryId) throws UniverseException { + HttpGet httpGet = new HttpGet(String.format("%s%s%s%s", apiUrl, "/sl/v1/queries/", + queryId, "/data.svc/Flows0")); + setHeaders(httpGet, token); + HttpResponse response = null; + try { + response = httpClient.execute(httpGet); + if (response.getStatusLine().getStatusCode() != 200) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get results): Request failed\n", EntityUtils.toString(response.getEntity()))); + } + } catch (IOException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get results): Request failed", ExceptionUtils.getStackTrace(e))); + } + + try (InputStream xmlStream = response.getEntity().getContent()) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(xmlStream); + XPathFactory xPathfactory = XPathFactory.newInstance(); + XPath xpath = xPathfactory.newXPath(); + XPathExpression expr = xpath.compile("//feed/entry/content/properties"); + NodeList resultsNodes = (NodeList) expr.evaluate(doc, XPathConstants.NODESET); + if (resultsNodes != null) { + return parseResults(resultsNodes); + } else { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get results): Response processing failed")); + } + } catch (IOException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get results): Request failed", ExceptionUtils.getStackTrace(e))); + } catch (ParserConfigurationException | SAXException | XPathExpressionException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get results): Response processing failed", ExceptionUtils.getStackTrace(e))); + } + } + + public String getToken(String paragraphId) throws UniverseException { + try { + if (tokens.containsKey(paragraphId)) { + return tokens.get(paragraphId); + } + HttpPost httpPost = new HttpPost(String.format("%s%s", apiUrl, "/logon/long")); + setHeaders(httpPost); + + httpPost.setEntity(new StringEntity( + String.format(loginRequestTemplate, user, password, authType), "UTF-8")); + HttpResponse response = httpClient.execute(httpPost); + String result = null; + if (response.getStatusLine().getStatusCode() == 200) { + result = getValue(EntityUtils.toString(response.getEntity()), + "//content/attrs/attr[@name=\"logonToken\"]"); + tokens.put(paragraphId, result); + } + + return result; + } catch (IOException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get token): Request failed", ExceptionUtils.getStackTrace(e))); + } catch (ParserConfigurationException | SAXException | XPathExpressionException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get token): Response processing failed", ExceptionUtils.getStackTrace(e))); + } + } + + public boolean closeSession(String paragraphId) throws UniverseException { + try { + if (tokens.containsKey(paragraphId)) { + HttpPost httpPost = new HttpPost(String.format("%s%s", apiUrl, "/logoff")); + setHeaders(httpPost, tokens.get(paragraphId)); + HttpResponse response = httpClient.execute(httpPost); + if (response.getStatusLine().getStatusCode() == 200) { + return true; + } + } + + return false; + } catch (Exception e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(close session): Request failed", ExceptionUtils.getStackTrace(e))); + } finally { + tokens.remove(paragraphId); + } + } + + public UniverseInfo getUniverseInfo(String universeName) { + return universesMap.get(universeName); + } + + public Map getUniverseNodesInfo(String token, String universeName) + throws UniverseException { + UniverseInfo universeInfo = universesMap.get(universeName); + if (universeInfo != null && StringUtils.isNotBlank(universeInfo.getId())) { + Map universeNodeInfoMap = universeInfosMap.get(universeName); + if (universeNodeInfoMap != null && universesInfoUpdatedMap.containsKey(universeName) && + !isExpired(universesInfoUpdatedMap.get(universeName))) { + return universeNodeInfoMap; + } else { + universeNodeInfoMap = new HashMap<>(); + } + try { + HttpGet httpGet = + new HttpGet(String.format("%s%s%s", apiUrl, "/sl/v1/universes/", universeInfo.getId())); + setHeaders(httpGet, token); + HttpResponse response = httpClient.execute(httpGet); + + if (response.getStatusLine().getStatusCode() == 200) { + try (InputStream xmlStream = response.getEntity().getContent()) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(xmlStream); + XPathFactory xPathfactory = XPathFactory.newInstance(); + XPath xpath = xPathfactory.newXPath(); + XPathExpression expr = xpath.compile("//outline/folder"); + XPathExpression exprRootItems = xpath.compile("//outline/item"); + NodeList universeInfoNodes = (NodeList) expr.evaluate(doc, XPathConstants.NODESET); + NodeList universeRootInfoNodes = + (NodeList) exprRootItems.evaluate(doc, XPathConstants.NODESET); + if (universeInfoNodes != null) { + parseUniverseInfo(universeInfoNodes, universeNodeInfoMap); + } + if (universeRootInfoNodes != null) { + parseUniverseInfo(universeRootInfoNodes, universeNodeInfoMap); + } + } catch (Exception e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get universe nodes info): Response processing failed", + ExceptionUtils.getStackTrace(e))); + } + } + } catch (IOException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get universe nodes info): Request failed", ExceptionUtils.getStackTrace(e))); + } + universeInfosMap.put(universeName, universeNodeInfoMap); + universesInfoUpdatedMap.put(universeName, System.currentTimeMillis()); + + return universeNodeInfoMap; + } + return Collections.emptyMap(); + + } + + public void loadUniverses(String token) throws UniverseException { + if (universesMap.isEmpty() || universesUpdated == 0 || isExpired(universesUpdated)) { + Map universes = new ConcurrentHashMap(); + loadUniverses(token, 0, universes); + universesMap.clear(); + universesMap.putAll(universes); + universesUpdated = System.currentTimeMillis(); + } + } + + public void cleanUniverses() { + universesMap.clear(); + } + + public void removeUniverseInfo(String universe) { + universeInfosMap.remove(universe); + } + + public Map getUniversesMap() { + return universesMap; + } + + public List getParameters(String token, String queryId) + throws UniverseException { + HttpGet httpGet = new HttpGet(String.format("%s%s%s%s", apiUrl, "/sl/v1/queries/", + queryId, "/parameters")); + setHeaders(httpGet, token); + HttpResponse response = null; + try { + response = httpClient.execute(httpGet); + if (response.getStatusLine().getStatusCode() != 200) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get parameters): Request failed\n", EntityUtils.toString(response.getEntity()))); + } + } catch (IOException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get parameters): Request failed", ExceptionUtils.getStackTrace(e))); + } + + try (InputStream xmlStream = response.getEntity().getContent()) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(xmlStream); + XPathFactory xPathfactory = XPathFactory.newInstance(); + XPath xpath = xPathfactory.newXPath(); + XPathExpression expr = xpath.compile("//parameters/parameter"); + NodeList parametersNodes = (NodeList) expr.evaluate(doc, XPathConstants.NODESET); + if (parametersNodes != null) { + return parseParameters(parametersNodes); + } else { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get parameters): Response processing failed")); + } + } catch (IOException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get parameters): Response processing failed", ExceptionUtils.getStackTrace(e))); + } catch (ParserConfigurationException | SAXException | XPathExpressionException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get parameters): Response processing failed", ExceptionUtils.getStackTrace(e))); + } + } + + public void setParametersValues(String token, String queryId, + List parameters) throws UniverseException { + HttpPut httpPut = new HttpPut(String.format("%s%s%s%s", apiUrl, "/sl/v1/queries/", + queryId, "/parameters")); + setHeaders(httpPut, token); + HttpResponse response = null; + try { + StringBuilder request = new StringBuilder(); + request.append("\n"); + for (UniverseQueryPrompt parameter : parameters) { + String answer = String.format(parameterAnswerTemplate, parameter.getConstrained(), + parameter.getType(), parameter.getCardinality(), parameter.getKeepLastValues(), + parameter.getValue()); + String id = parameter.getId() != null ? String.format("%s\n", parameter.getId()) : + StringUtils.EMPTY; + String technicalName = parameter.getTechnicalName() != null ? + String.format("%s\n", parameter.getTechnicalName()) : + StringUtils.EMPTY; + String name = parameter.getTechnicalName() != null ? + String.format("%s\n", parameter.getName()) : + StringUtils.EMPTY; + request.append(String.format(parameterTemplate, id, technicalName, name, answer)); + } + request.append("\n"); + + httpPut.setEntity(new StringEntity(request.toString(), "UTF-8")); + + response = httpClient.execute(httpPut); + if (response.getStatusLine().getStatusCode() != 200) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(set parameters): Request failed\n", EntityUtils.toString(response.getEntity()))); + } + } catch (IOException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(set parameters): Request failed", ExceptionUtils.getStackTrace(e))); + } + } + + private void loadUniverses(String token, int offset, Map universesMap) + throws UniverseException { + int limit = 50; + HttpGet httpGet = new HttpGet(String.format("%s%s?offset=%s&limit=%s", apiUrl, + "/sl/v1/universes", + offset, limit)); + setHeaders(httpGet, token); + HttpResponse response = null; + try { + response = httpClient.execute(httpGet); + } catch (Exception e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get universes): Request failed", ExceptionUtils.getStackTrace(e))); + } + if (response != null && response.getStatusLine().getStatusCode() == 200) { + try (InputStream xmlStream = response.getEntity().getContent()) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(xmlStream); + XPathFactory xPathfactory = XPathFactory.newInstance(); + XPath xpath = xPathfactory.newXPath(); + XPathExpression expr = xpath.compile("//universe"); + NodeList universesNodes = (NodeList) expr.evaluate(doc, XPathConstants.NODESET); + if (universesNodes != null) { + int count = universesNodes.getLength(); + for (int i = 0; i < count; i++) { + Node universe = universesNodes.item(i); + if (universe.hasChildNodes()) { + NodeList universeParameters = universe.getChildNodes(); + int parapetersCount = universeParameters.getLength(); + String id = null; + String name = null; + String type = null; + for (int j = 0; j < parapetersCount; j++) { + Node parameterNode = universeParameters.item(j); + parameterNode.getNodeName(); + if (parameterNode.getNodeType() == Node.ELEMENT_NODE) { + if (parameterNode.getNodeName().equalsIgnoreCase("id")) { + id = parameterNode.getTextContent(); + continue; + } + if (parameterNode.getNodeName().equalsIgnoreCase("name")) { + name = parameterNode.getTextContent(); + continue; + } + if (parameterNode.getNodeName().equalsIgnoreCase("type")) { + type = parameterNode.getTextContent(); + continue; + } + } + } + if (StringUtils.isNotBlank(type)) { + name = name.replaceAll(String.format("\\.%s$", type), StringUtils.EMPTY); + } + universesMap.put(name, new UniverseInfo(id, name, type)); + } + } + if (count == limit) { + offset += limit; + loadUniverses(token, offset, universesMap); + } + } + } catch (IOException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get universes): Response processing failed", ExceptionUtils.getStackTrace(e))); + } catch (ParserConfigurationException | SAXException | XPathExpressionException e) { + throw new UniverseException(String.format(errorMessageTemplate, "UniverseClient " + + "(get universes): Response processing failed", ExceptionUtils.getStackTrace(e))); + } + } + } + + private boolean isExpired(Long lastUpdated) { + if (lastUpdated == null || System.currentTimeMillis() - lastUpdated > DAY) { + return true; + } + + return false; + } + + private void setHeaders(HttpRequestBase request) { + setHeaders(request, null); + } + + private void setHeaders(HttpRequestBase request, String token) { + request.setHeaders(commonHeaders); + if (StringUtils.isNotBlank(token)) { + request.addHeader(TOKEN_HEADER, token); + } + } + + private String getValue(String response, String xPathString) throws ParserConfigurationException, + IOException, SAXException, XPathExpressionException { + try (InputStream xmlStream = new ByteArrayInputStream(response.getBytes())) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(xmlStream); + XPathFactory xPathfactory = XPathFactory.newInstance(); + XPath xpath = xPathfactory.newXPath(); + XPathExpression expr = xpath.compile(xPathString); + Node tokenNode = (Node) expr.evaluate(doc, XPathConstants.NODE); + if (tokenNode != null) { + return tokenNode.getTextContent(); + } + } + return null; + } + + private List parseParameters(NodeList parametersNodeList) { + List parameters = new ArrayList<>(); + if (parametersNodeList != null) { + int count = parametersNodeList.getLength(); + for (int i = 0; i < count; i++) { + Node parameterNode = parametersNodeList.item(i); + Node type = parameterNode.getAttributes().getNamedItem("type"); + if (type != null && type.getTextContent().equalsIgnoreCase("prompt") && + parameterNode.hasChildNodes()) { + NodeList parameterInfoNodes = parameterNode.getChildNodes(); + int childNodesCount = parameterInfoNodes.getLength(); + String name = null; + Integer id = null; + String cardinality = null; + String constrained = null; + String valueType = null; + String technicalName = null; + String keepLastValues = null; + for (int j = 0; j < childNodesCount; j++) { + Node childNode = parameterInfoNodes.item(j); + String childNodeName = childNode.getNodeName(); + if (childNodeName.equalsIgnoreCase(EL_NAME)) { + name = childNode.getTextContent(); + continue; + } + if (childNodeName.equalsIgnoreCase(EL_ID)) { + id = Integer.parseInt(childNode.getTextContent()); + continue; + } + if (childNodeName.equalsIgnoreCase(EL_TECH_NAME)) { + technicalName = childNode.getTextContent(); + continue; + } + if (childNodeName.equalsIgnoreCase(EL_ANSWER)) { + NamedNodeMap answerAttributes = childNode.getAttributes(); + if (answerAttributes.getNamedItem("constrained") != null) { + constrained = answerAttributes.getNamedItem("constrained").getTextContent(); + } + if (answerAttributes.getNamedItem("type") != null) { + valueType = answerAttributes.getNamedItem("type").getTextContent(); + } + NodeList answerNodes = childNode.getChildNodes(); + int answerCount = answerNodes.getLength(); + for (int k = 0; k < answerCount; k++) { + Node answerChildNode = answerNodes.item(k); + String answerChildNodeName = answerChildNode.getNodeName(); + if (answerChildNodeName.equalsIgnoreCase(EL_INFO)) { + NamedNodeMap infoAttributes = answerChildNode.getAttributes(); + if (infoAttributes.getNamedItem("cardinality") != null) { + cardinality = infoAttributes.getNamedItem("cardinality").getTextContent(); + } + if (infoAttributes.getNamedItem("keepLastValues") != null) { + keepLastValues = infoAttributes.getNamedItem("keepLastValues").getTextContent(); + } + break; + } + } + continue; + } + } + if (name != null && id != null && cardinality != null) { + parameters.add(new UniverseQueryPrompt(id, name, cardinality, constrained, valueType, + technicalName, keepLastValues)); + break; + } + } + } + } + + return parameters; + } + + private List> parseResults(NodeList resultsNodeList) { + List> results = new ArrayList<>(); + if (resultsNodeList != null) { + int count = resultsNodeList.getLength(); + for (int i = 0; i < count; i++) { + Node node = resultsNodeList.item(i); + if (node.getNodeType() == Node.ELEMENT_NODE && node.hasChildNodes()) { + NodeList properties = node.getChildNodes(); + if (properties != null) { + int countProperties = properties.getLength(); + List headers = new ArrayList<>(); + List row = new ArrayList<>(); + // first property is id + for (int j = 1; j < countProperties; j++) { + Node propertyNode = properties.item(j); + if (i == 0) { + headers.add(propertyNode.getNodeName().replaceAll("^\\w*:", StringUtils.EMPTY)); + } + row.add(propertyNode.getTextContent()); + } + if (i == 0) { + results.add(headers); + } + results.add(row); + } + } + } + } + + return results; + } + + private void parseUniverseInfo(NodeList universeInfoNodes, Map nodes) { + if (universeInfoNodes != null) { + int count = universeInfoNodes.getLength(); + for (int i = 0; i < count; i++) { + Node node = universeInfoNodes.item(i); + if (node.getNodeType() == Node.ELEMENT_NODE && node.hasChildNodes()) { + String name = node.getNodeName(); + NodeList childNodes = node.getChildNodes(); + int childNodesCount = childNodes.getLength(); + if (name.equalsIgnoreCase(EL_FOLDER)) { + NodeSet list = new NodeSet(); + for (int j = 0; j < childNodesCount; j++) { + Node childNode = childNodes.item(j); + if (childNode.getNodeType() == Node.ELEMENT_NODE && childNode.hasChildNodes()) { + String childNodeName = childNode.getNodeName(); + if (childNodeName.equalsIgnoreCase(EL_FOLDER) + || childNodeName.equalsIgnoreCase(EL_ITEM)) { + list.addNode(childNode); + } + } + } + if (list.getLength() > 0) { + parseUniverseInfo(list, nodes); + } + } else if (name.equalsIgnoreCase(EL_ITEM)) { + String nodeId = null; + String nodeName = null; + String nodePath = null; + for (int j = 0; j < childNodesCount; j++) { + Node childNode = childNodes.item(j); + if (childNode.getNodeType() == Node.ELEMENT_NODE) { + String childNodeName = childNode.getNodeName(); + if (childNodeName.equalsIgnoreCase(EL_NAME)) { + nodeName = childNode.getTextContent(); + continue; + } + if (childNodeName.equalsIgnoreCase(EL_ID)) { + nodeId = childNode.getTextContent(); + continue; + } + if (childNodeName.equalsIgnoreCase(EL_PATH)) { + nodePath = childNode.getTextContent(); + continue; + } + } + } + String folder = null; + StringBuilder key = new StringBuilder(); + if (StringUtils.isNotBlank(nodeName)) { + String nodeType = null; + if (StringUtils.isNotBlank(nodePath)) { + String[] parts = nodePath.split("\\\\"); + List path = new ArrayList(); + for (String part : parts) { + String[] p = part.split("\\|"); + if (p.length == 2) { + if (p[1].equalsIgnoreCase("folder")) { + path.add(p[0]); + } else { + nodeName = p[0]; + nodeType = p[1]; + } + } + } + folder = StringUtils.join(path, "\\"); + if (path.isEmpty()) { + key.append(String.format("[%s]", nodeName)); + } else { + key.append("["); + key.append(StringUtils.join(path, "].[")); + key.append(String.format("].[%s]", nodeName)); + } + } + nodes.put(key.toString(), + new UniverseNodeInfo(nodeId, nodeName, nodeType, folder, nodePath)); + } + } + } + } + } + } +} diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseCompleter.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseCompleter.java new file mode 100644 index 00000000000..e67011b5dc4 --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseCompleter.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap.universe; + +import jline.console.completer.ArgumentCompleter.ArgumentList; +import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.completer.CachedCompleter; +import org.apache.zeppelin.completer.CompletionType; +import org.apache.zeppelin.completer.StringsCompleter; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.*; +import java.util.regex.Pattern; + +/** + * SAP Universe auto complete functionality. + */ +public class UniverseCompleter { + + private static Logger logger = LoggerFactory.getLogger(UniverseCompleter.class); + + private static final String KEYWORD_SPLITERATOR = ","; + public static final String CLEAN_NAME_REGEX = "\\[|\\]"; + public static final Character START_NAME = '['; + public static final Character END_NAME = ']'; + public static final String KW_UNIVERSE = "universe"; + public static final String TYPE_FOLDER = "folder"; + + private static final Comparator nodeInfoComparator = new Comparator() { + @Override + public int compare(UniverseNodeInfo o1, UniverseNodeInfo o2) { + if (o1.getType().equalsIgnoreCase(TYPE_FOLDER) + && o2.getType().equalsIgnoreCase(TYPE_FOLDER)) { + return o1.getName().compareToIgnoreCase(o2.getName()); + } + if (o1.getType().equalsIgnoreCase(TYPE_FOLDER)) { + return -1; + } + if (o2.getType().equalsIgnoreCase(TYPE_FOLDER)) { + return 1; + } + if (!o1.getType().equalsIgnoreCase(o2.getType())) { + return o1.getType().compareToIgnoreCase(o2.getType()); + } else { + + return o1.getName().compareToIgnoreCase(o2.getName()); + } + } + }; + + /** + * Delimiter that can split keyword list + */ + private WhitespaceArgumentDelimiter sqlDelimiter = new WhitespaceArgumentDelimiter() { + + private Pattern pattern = Pattern.compile(",|;"); + + @Override + public boolean isDelimiterChar(CharSequence buffer, int pos) { + char c = buffer.charAt(pos); + boolean endName = false; + for (int i = pos; i > 0; i--) { + char ch = buffer.charAt(i); + if (ch == '\n') { + break; + } + if (ch == START_NAME && !endName) { + return false; + } + if (ch == END_NAME) { + break; + } + } + return pattern.matcher(StringUtils.EMPTY + buffer.charAt(pos)).matches() + || super.isDelimiterChar(buffer, pos); + } + }; + + /** + * Universe completer + */ + private CachedCompleter universeCompleter; + + /** + * Keywords completer + */ + private CachedCompleter keywordCompleter; + + /** + * UniverseInfo completers + */ + private Map universeInfoCompletersMap = new HashMap<>(); + + private int ttlInSeconds; + + public UniverseCompleter(int ttlInSeconds) { + this.ttlInSeconds = ttlInSeconds; + } + + public int complete(String buffer, int cursor, List candidates) { + CursorArgument cursorArgument = parseCursorArgument(buffer, cursor); + + String argument = cursorArgument.getCursorArgumentPartForComplete(); + if (cursorArgument.isUniverseNamePosition()) { + List universeCandidates = new ArrayList<>(); + universeCompleter.getCompleter().complete(argument, argument.length(), + universeCandidates); + addCompletions(candidates, universeCandidates, CompletionType.universe.name()); + return universeCandidates.size(); + } + + if (cursorArgument.isUniverseNodePosition()) { + List universeNodeCandidates = new ArrayList(); + CachedCompleter completer = universeInfoCompletersMap.get(cursorArgument.getUniverse()); + if (completer != null) { + completer.getCompleter().complete(argument, argument.length(), universeNodeCandidates); + } + Collections.sort(universeNodeCandidates, nodeInfoComparator); + addCompletions(candidates, universeNodeCandidates); + return universeNodeCandidates.size(); + } + + List keywordCandidates = new ArrayList<>(); + keywordCompleter.getCompleter().complete(argument, + argument.length() > 0 ? argument.length() : 0, keywordCandidates); + addCompletions(candidates, keywordCandidates, CompletionType.keyword.name()); + + return keywordCandidates.size(); + } + + public void createOrUpdate(UniverseClient client, String token, String buffer, int cursor) { + try { + CursorArgument cursorArgument = parseCursorArgument(buffer, cursor); + if (keywordCompleter == null || keywordCompleter.getCompleter() == null + || keywordCompleter.isExpired()) { + Set keywords = getKeywordsCompletions(); + if (keywords != null && !keywords.isEmpty()) { + keywordCompleter = new CachedCompleter(new StringsCompleter(keywords), 0); + } + } + if (cursorArgument.needLoadUniverses() || (universeCompleter == null + || universeCompleter.getCompleter() == null || universeCompleter.isExpired())) { + client.cleanUniverses(); + client.loadUniverses(token); + if (client.getUniversesMap().size() > 0) { + universeCompleter = new CachedCompleter( + new StringsCompleter(client.getUniversesMap().keySet()), ttlInSeconds); + } + } + if (cursorArgument.needLoadUniverseInfo() && + (!universeInfoCompletersMap.containsKey(cursorArgument.getUniverse()) || + universeInfoCompletersMap.get(cursorArgument.getUniverse()).getCompleter() == null || + universeInfoCompletersMap.get(cursorArgument.getUniverse()).isExpired())) { + if (StringUtils.isNotBlank(cursorArgument.getUniverse())) { + client.removeUniverseInfo(cursorArgument.getUniverse()); + Map info = client.getUniverseNodesInfo(token, cursorArgument + .getUniverse()); + CachedCompleter completer = new CachedCompleter( + new UniverseNodeInfoCompleter(info.values()), ttlInSeconds); + universeInfoCompletersMap.put(cursorArgument.getUniverse(), completer); + } + } + } catch (Exception e) { + logger.error("Failed to update completions", e); + } + } + + private Set getKeywordsCompletions() throws IOException { + String keywords = + new BufferedReader(new InputStreamReader( + UniverseCompleter.class.getResourceAsStream("/universe.keywords"))).readLine(); + + Set completions = new TreeSet<>(); + + if (StringUtils.isNotBlank(keywords)) { + String[] words = keywords.split(KEYWORD_SPLITERATOR); + for (String word : words) { + completions.add(word); + } + } + + return completions; + } + + private CursorArgument parseCursorArgument(String buffer, int cursor) { + CursorArgument result = new CursorArgument(); + if (buffer != null && buffer.length() >= cursor) { + String buf = buffer.substring(0, cursor); + if (StringUtils.isNotBlank(buf)) { + ArgumentList argList = sqlDelimiter.delimit(buf, cursor); + int argIndex = argList.getCursorArgumentIndex(); + if (argIndex == 0) { + result.setCursorArgumentPartForComplete(argList.getCursorArgument()); + return result; + } + + if (argIndex > 0 && argList.getArguments()[argIndex - 1].equalsIgnoreCase(KW_UNIVERSE)) { + result.setUniverseNamePosition(true); + result.setCursorArgumentPartForComplete(cleanName(argList.getCursorArgument() + .substring(0, argList.getArgumentPosition()))); + return result; + } + if (argIndex > 1) { + for (int i = argIndex - 2; i >= 0; i--) { + if (argList.getArguments()[i].equalsIgnoreCase(KW_UNIVERSE)) { + result.setUniverse(cleanName(argList.getArguments()[i + 1])); + break; + } + } + + if (StringUtils.isNotBlank(result.getUniverse()) + && argList.getCursorArgument().startsWith(START_NAME.toString())) { + result.setCursorArgumentPartForComplete( + argList.getCursorArgument().substring(0, argList.getArgumentPosition())); + result.setUniverseNodePosition(true); + return result; + } else { + result.setCursorArgumentPartForComplete(argList.getCursorArgument() + .substring(0, argList.getArgumentPosition())); + } + } + } + } + + if (result.getCursorArgumentPartForComplete() == null) { + result.setCursorArgumentPartForComplete(StringUtils.EMPTY); + } + + return result; + } + + private String cleanName(String name) { + return name.replaceAll(CLEAN_NAME_REGEX, StringUtils.EMPTY); + } + + private void addCompletions(List interpreterCompletions, + List candidates, String meta) { + for (CharSequence candidate : candidates) { + String value; + if (meta.equalsIgnoreCase(CompletionType.universe.name())) { + value = String.format("%s%s;\n", candidate.toString(), END_NAME); + } else { + value = candidate.toString(); + } + interpreterCompletions.add(new InterpreterCompletion(candidate.toString(), value, meta)); + } + } + + private void addCompletions(List interpreterCompletions, + List candidates) { + for (UniverseNodeInfo candidate : candidates) { + String value; + if (candidate.getType().equalsIgnoreCase(TYPE_FOLDER)) { + value = String.format("%s%s.%s", candidate.getName(), END_NAME, START_NAME); + } else { + value = String.format("%s%s", candidate.getName(), END_NAME); + } + interpreterCompletions.add(new InterpreterCompletion(candidate.getName(), value, + candidate.getType())); + } + } + + public CachedCompleter getUniverseCompleter() { + return universeCompleter; + } + + public Map getUniverseInfoCompletersMap() { + return universeInfoCompletersMap; + } + + private class CursorArgument { + private boolean universeNamePosition = false; + private boolean universeNodePosition = false; + private String universe; + private String cursorArgumentPartForComplete; + + public boolean isUniverseNamePosition() { + return universeNamePosition; + } + + public void setUniverseNamePosition(boolean universeNamePosition) { + this.universeNamePosition = universeNamePosition; + } + + public boolean isUniverseNodePosition() { + return universeNodePosition; + } + + public void setUniverseNodePosition(boolean universeNodePosition) { + this.universeNodePosition = universeNodePosition; + } + + public String getCursorArgumentPartForComplete() { + return cursorArgumentPartForComplete; + } + + public void setCursorArgumentPartForComplete(String cursorArgumentPartForComplete) { + this.cursorArgumentPartForComplete = cursorArgumentPartForComplete; + } + + public String getUniverse() { + return universe; + } + + public void setUniverse(String universe) { + this.universe = universe; + } + + public boolean needLoadUniverses() { + if (universe == null) { + return true; + } + return false; + } + + public boolean needLoadUniverseInfo() { + if (universe != null && universeNodePosition) { + return true; + } + return false; + } + } +} diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseException.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseException.java new file mode 100644 index 00000000000..8086f94419a --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap.universe; + + +/** + * Runtime Exception for SAP universe + */ +public class UniverseException extends Exception { + + public UniverseException(Throwable e) { + super(e); + } + + public UniverseException(String m) { + super(m); + } + + public UniverseException(String msg, Throwable t) { + super(msg, t); + } + +} diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseInfo.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseInfo.java new file mode 100644 index 00000000000..4f40dced2bc --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseInfo.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap.universe; + +/** + * Info about of universe node + */ +public class UniverseInfo { + private String id; + private String name; + private String type; + + public UniverseInfo() { + } + + public UniverseInfo(String id, String name, String type) { + this.id = id; + this.name = name; + this.type = type; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseNodeInfo.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseNodeInfo.java new file mode 100644 index 00000000000..fe0c97e0ab7 --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseNodeInfo.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap.universe; + +/** + * Info about of universe item + */ +public class UniverseNodeInfo { + private String id; + private String name; + private String type; + private String folder; + private String nodePath; + + public UniverseNodeInfo() { + } + + public UniverseNodeInfo(String id, String name, String type, String folder, String nodePath) { + this.id = id; + this.name = name; + this.type = type; + this.folder = folder; + this.nodePath = nodePath; + } + + public UniverseNodeInfo(String name, String type) { + this.name = name; + this.type = type; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getFolder() { + return folder; + } + + public void setFolder(String folder) { + this.folder = folder; + } + + public String getNodePath() { + return nodePath; + } + + public void setNodePath(String nodePath) { + this.nodePath = nodePath; + } +} diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseNodeInfoCompleter.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseNodeInfoCompleter.java new file mode 100644 index 00000000000..0704e1b77fb --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseNodeInfoCompleter.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.sap.universe; + +import jline.console.completer.Completer; +import jline.internal.Preconditions; +import org.apache.commons.lang.StringUtils; + +import java.util.*; + +/** + * Case-insensitive completer. + */ +public class UniverseNodeInfoCompleter implements Completer { + private final UniverseInfoTreeNode tree = new UniverseInfoTreeNode(); + + public UniverseNodeInfoCompleter() { + } + + public UniverseNodeInfoCompleter(final Collection nodes) { + Preconditions.checkNotNull(nodes); + for (UniverseNodeInfo node : nodes) { + String folder = node.getFolder(); + if (StringUtils.isBlank(folder)) { + tree.putInfo(node); + } else { + String[] path = folder.split("\\\\"); + UniverseInfoTreeNode universeInfoTreeNode = tree; + for (String s : path) { + if (!universeInfoTreeNode.contains(s)) { + universeInfoTreeNode = universeInfoTreeNode.putFolder(s); + } else { + universeInfoTreeNode = universeInfoTreeNode.getFolder(s); + } + } + universeInfoTreeNode.putInfo(node); + } + } + } + + public int complete(final String buffer, final int cursor, final List candidates) { + return completeCollection(buffer, cursor, candidates); + } + + private int completeCollection(final String buffer, final int cursor, + final Collection candidates) { + Preconditions.checkNotNull(candidates); + if (buffer == null) { + candidates.addAll(tree.getNodesInfo()); + } else { + String part = buffer.substring(0, cursor); + List path = new ArrayList<>(); + path.addAll(Arrays.asList(part.split("\\]\\.\\["))); + if (part.endsWith(UniverseCompleter.START_NAME.toString())) { + path.add(StringUtils.EMPTY); + } + + UniverseInfoTreeNode treeNode = tree; + for (int i = 0; i < path.size() - 1; i++) { + String folder = cleanName(path.get(i)); + if (treeNode.contains(folder)) { + treeNode = treeNode.getFolder(folder); + if (treeNode == null) { + break; + } + } + } + String p = cleanName(path.get(path.size() - 1)).toUpperCase(); + if (treeNode != null && treeNode.getChildren() != null) { + if (p.isEmpty()) { + candidates.addAll(treeNode.getNodesInfo()); + } else { + for (UniverseNodeInfo universeNodeInfo : treeNode.getNodesInfo()) { + if (universeNodeInfo.getName().toUpperCase().startsWith(p)) { + candidates.add(universeNodeInfo); + } + } + } + } + } + + return candidates.isEmpty() ? -1 : 0; + } + + private String cleanName(String name) { + return name.replaceAll(UniverseCompleter.CLEAN_NAME_REGEX, StringUtils.EMPTY); + } + + private class UniverseInfoTreeNode { + private String name; + private boolean isFolder; + private Map children; + + public UniverseInfoTreeNode() { + this.name = "/"; + this.isFolder = true; + this.children = new HashMap<>(); + } + + public UniverseInfoTreeNode(String name) { + this.name = name; + this.isFolder = true; + this.children = new HashMap<>(); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public boolean isFolder() { + return isFolder; + } + + public void setFolder(boolean folder) { + isFolder = folder; + } + + public Map getChildren() { + return children; + } + + public void setChildren(Map children) { + this.children = children; + } + + public boolean contains(String name) { + return children.containsKey(name); + } + + public UniverseInfoTreeNode getFolder(String name) { + Object child = children.get(name); + if (child instanceof UniverseInfoTreeNode) { + return (UniverseInfoTreeNode) children.get(name); + } + + return null; + } + + public UniverseInfoTreeNode putFolder(String name) { + UniverseInfoTreeNode newNode = new UniverseInfoTreeNode(name); + children.put(name, newNode); + return newNode; + } + + public void putInfo(UniverseNodeInfo info) { + children.put(info.getName(), info); + } + + public List getNodesInfo() { + List list = new ArrayList<>(); + if (children != null) { + for (Object o : children.values()) { + if (o instanceof UniverseNodeInfo) { + list.add((UniverseNodeInfo) o); + } else { + UniverseInfoTreeNode treeNode = (UniverseInfoTreeNode) o; + list.add(new UniverseNodeInfo(treeNode.getName(), UniverseCompleter.TYPE_FOLDER)); + } + } + } + + return list; + } + } +} diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseQuery.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseQuery.java new file mode 100644 index 00000000000..824d1bea66a --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseQuery.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap.universe; + +import org.apache.commons.lang.StringUtils; + +/** + * Data of universe query + */ +public class UniverseQuery { + private String select; + private String where; + private UniverseInfo universeInfo; + + public UniverseQuery(String select, String where, UniverseInfo universeInfo) { + this.select = select; + this.where = where; + this.universeInfo = universeInfo; + } + + public boolean isCorrect() { + return StringUtils.isNotBlank(select) && universeInfo != null && + StringUtils.isNotBlank(universeInfo.getId()) + && StringUtils.isNotBlank(universeInfo.getName()); + } + + public String getSelect() { + return select; + } + + public String getWhere() { + return where; + } + + public UniverseInfo getUniverseInfo() { + return universeInfo; + } +} diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseQueryPrompt.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseQueryPrompt.java new file mode 100644 index 00000000000..04b2b49b63a --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseQueryPrompt.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap.universe; + +/** + * Info about parameter of universe query + */ +public class UniverseQueryPrompt { + private Integer id; + private String name; + private String cardinality; + private String constrained; + private String type; + private String value; + private String technicalName; + private String keepLastValues; + + public UniverseQueryPrompt() { + } + + public UniverseQueryPrompt(Integer id, String name, String cardinality, String constrained, + String type, String technicalName, String keepLastValues) { + this.id = id; + this.name = name; + this.cardinality = cardinality; + this.constrained = constrained; + this.type = type; + this.technicalName = technicalName; + this.keepLastValues = keepLastValues; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getCardinality() { + return cardinality; + } + + public void setCardinality(String cardinality) { + this.cardinality = cardinality; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getConstrained() { + return constrained; + } + + public void setConstrained(String constrained) { + this.constrained = constrained; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getTechnicalName() { + return technicalName; + } + + public void setTechnicalName(String technicalName) { + this.technicalName = technicalName; + } + + public String getKeepLastValues() { + return keepLastValues; + } + + public void setKeepLastValues(String keepLastValues) { + this.keepLastValues = keepLastValues; + } +} diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseUtil.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseUtil.java new file mode 100644 index 00000000000..dc9099d8cb8 --- /dev/null +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseUtil.java @@ -0,0 +1,643 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap.universe; + +import org.apache.commons.lang.StringUtils; + +import java.util.*; + +/** + * Util class for convert request from Zeppelin to SAP + */ +public class UniverseUtil { + + private static final String COMPRASION_START_TEMPLATE = "\n"; + private static final String COMPRASION_END_TEMPLATE = "\n"; + private static final String COMPARISON_FILTER = "\n"; + private static final String CONST_OPERAND_START_TEMPLATE = "\n"; + private static final String CONST_OPERAND_END_TEMPLATE = "\n"; + private static final String CONST_OPERAND_VALUE_TEMPLATE = "\n" + + "%s\n\n"; + private static final String PREDEFINED_FILTER_TEMPLATE = "\n"; + private static final String OBJECT_OPERAND_TEMPLATE = "\n"; + private static final String RESULT_START_TEMPLATE = "\n"; + private static final String RESULT_END_TEMPLATE = "\n"; + private static final String RESULT_OBJ_TEMPLATE = "\n"; + + private static final String MARKER_EQUAL = "#EqualTo#"; + private static final String MARKER_LESS_EQUAL = "#LessThanOrEqualTo#"; + private static final String MARKER_NOT_EQUAL = "#NotEqualTo#"; + private static final String MARKER_LESS = "#LessThan#"; + private static final String MARKER_GREATER_EQUALS = "#GreaterThanOrEqualTo#"; + private static final String MARKER_GREATER = "#GreaterThan#"; + private static final String MARKER_IN = "#InList#"; + private static final String MARKER_NOT_IN = "#NotInList#"; + private static final String MARKER_NULL = "#IsNull#"; + private static final String MARKER_NOT_NULL = "#IsNotNull#"; + private static final String MARKER_FILTER = "#filter#"; + private static final String MARKER_AND = "#and#"; + private static final String MARKER_OR = "#or#"; + private static final String MARKER_BACKSPACE = "#backspace#"; + private static final String MARKER_LEFT_BRACE = "#left_brace#"; + private static final String MARKER_RIGHT_BRACE = "#right_brace#"; + + + private static final String LEFT_BRACE = "("; + private static final String RIGHT_BRACE = ")"; + + public static final Map OPERATIONS; + + static { + OPERATIONS = new HashMap<>(); + OPERATIONS.put(MARKER_EQUAL, 1); + OPERATIONS.put(MARKER_LESS_EQUAL, 1); + OPERATIONS.put(MARKER_NOT_EQUAL, 1); + OPERATIONS.put(MARKER_LESS, 1); + OPERATIONS.put(MARKER_GREATER_EQUALS, 1); + OPERATIONS.put(MARKER_GREATER, 1); + OPERATIONS.put(MARKER_IN, 1); + OPERATIONS.put(MARKER_NOT_IN, 1); + OPERATIONS.put(MARKER_NULL, 1); + OPERATIONS.put(MARKER_NOT_NULL, 1); + OPERATIONS.put(MARKER_FILTER, 1); + OPERATIONS.put(MARKER_AND, 2); + OPERATIONS.put(MARKER_OR, 3); + } + + public UniverseQuery convertQuery(String text, UniverseClient client, String token) + throws UniverseException { + StringBuilder select = new StringBuilder(); + StringBuilder universe = new StringBuilder(); + StringBuilder buf = new StringBuilder(); + StringBuilder resultObj = new StringBuilder(); + StringBuilder whereBuf = new StringBuilder(); + UniverseInfo universeInfo = null; + String where = null; + boolean singleQuoteClosed = true; + boolean pathClosed = true; + boolean universePart = false; + boolean selectPart = false; + boolean wherePart = false; + boolean listOperator = false; + boolean operatorPosition = false; + Map nodeInfos = null; + + char[] array = text.toCharArray(); + for (int i = 0; i < array.length; i++) { + char c = array[i]; + buf.append(c); + if (c == '\'') { + if (i == 0 || array[i - 1] != '\\') { + singleQuoteClosed = !singleQuoteClosed; + } + } + if (c == '[' && pathClosed && singleQuoteClosed) { + pathClosed = false; + if (wherePart) { + operatorPosition = false; + } + } + if (c == ']' && !pathClosed && singleQuoteClosed) { + pathClosed = true; + if (wherePart) { + operatorPosition = true; + if (i + 1 == array.length || (array[i + 1] != '.' + && isFilter(String.format("%s]", whereBuf.toString()), text.substring(i + 1)))) { + whereBuf.append(c); + whereBuf.append(MARKER_FILTER); + if (i + 1 == array.length) { + wherePart = false; + where = parseWhere(whereBuf.toString(), nodeInfos); + } + continue; + } + } + } + if (c == '(' && wherePart && pathClosed && singleQuoteClosed) { + if (listOperator) { + whereBuf.append(MARKER_LEFT_BRACE); + continue; + } else { + whereBuf.append(c); + continue; + } + } + if (c == ')' && wherePart && pathClosed && singleQuoteClosed) { + if (listOperator) { + whereBuf.append(MARKER_RIGHT_BRACE); + listOperator = false; + continue; + } else { + whereBuf.append(c); + continue; + } + } + + if (!universePart && singleQuoteClosed + && buf.toString().toLowerCase().endsWith("universe")) { + universePart = true; + continue; + } + + if (universePart) { + if (c == ';' && singleQuoteClosed) { + universePart = false; + if (universe.toString().trim().length() > 2) { + String universeName = + universe.toString().trim().substring(1, universe.toString().trim().length() - 1); + universeInfo = client.getUniverseInfo(universeName); + nodeInfos = client.getUniverseNodesInfo(token, universeName); + } + } else { + universe.append(c); + } + continue; + } + + if (!selectPart && pathClosed && singleQuoteClosed + && buf.toString().toLowerCase().endsWith("select")) { + if (StringUtils.isBlank(universe.toString())) { + throw new UniverseException("Not found universe name"); + } + selectPart = true; + select.append(RESULT_START_TEMPLATE); + continue; + } + + if (!wherePart && pathClosed && singleQuoteClosed) { + if (buf.toString().toLowerCase().endsWith("where")) { + wherePart = true; + } + if (buf.toString().toLowerCase().endsWith("where") || i == array.length - 1) { + selectPart = false; + select.append(parseResultObj(resultObj.toString().replaceAll("(?i)wher$", ""), nodeInfos)); + select.append(RESULT_END_TEMPLATE); + continue; + } + } + + if (selectPart) { + if (pathClosed && singleQuoteClosed && c == ',') { + select.append(parseResultObj(resultObj.toString(), nodeInfos)); + resultObj = new StringBuilder(); + } else { + resultObj.append(c); + } + continue; + } + + if (wherePart) { + if (c == ';' && pathClosed && singleQuoteClosed) { + wherePart = false; + where = parseWhere(whereBuf.toString(), nodeInfos); + } else { + if (!singleQuoteClosed || !pathClosed) { + switch (c) { + case ' ': + case '\n': + whereBuf.append(MARKER_BACKSPACE); + break; + case '(': + whereBuf.append(MARKER_LEFT_BRACE); + break; + case ')': + whereBuf.append(MARKER_RIGHT_BRACE); + break; + default: + whereBuf.append(c); + } + } else if (pathClosed) { + if ((c == 'a' || c == 'A') && i < array.length - 2 && + text.substring(i, i + 3).equalsIgnoreCase("and")) { + i += 2; + whereBuf.append(MARKER_AND); + operatorPosition = false; + continue; + } + if ((c == 'o' || c == 'O') && i < array.length - 1 && + text.substring(i, i + 2).equalsIgnoreCase("or")) { + i += 1; + whereBuf.append(MARKER_OR); + operatorPosition = false; + continue; + } + if (operatorPosition) { + switch (c) { + case '=': + whereBuf.append(MARKER_EQUAL); + operatorPosition = false; + break; + case '<': + if (i + 1 < array.length) { + if (array[i + 1] == '=') { + whereBuf.append(MARKER_LESS_EQUAL); + operatorPosition = false; + i++; + break; + } else if (array[i + 1] == '>') { + whereBuf.append(MARKER_NOT_EQUAL); + operatorPosition = false; + i++; + break; + } + } + operatorPosition = false; + whereBuf.append(MARKER_LESS); + break; + case '>': + if (i + 1 < array.length) { + if (array[i + 1] == '=') { + whereBuf.append(MARKER_GREATER_EQUALS); + operatorPosition = false; + i++; + break; + } + } + operatorPosition = false; + whereBuf.append(MARKER_GREATER); + break; + case 'i': + case 'I': + boolean whileI = true; + StringBuilder operI = new StringBuilder(); + operI.append(c); + while (whileI) { + i++; + if (i >= array.length) { + whileI = false; + } + + if (array[i] != ' ' && array[i] != '\n') { + operI.append(array[i]); + } else { + continue; + } + String tmp = operI.toString().toLowerCase(); + if (tmp.equals("in")) { + whereBuf.append(MARKER_IN); + listOperator = true; + whileI = false; + operatorPosition = false; + } else if (tmp.equals("isnull")) { + whereBuf.append(MARKER_NULL); + whileI = false; + operatorPosition = false; + } else if (tmp.equals("isnotnull")) { + whereBuf.append(MARKER_NOT_NULL); + whileI = false; + operatorPosition = false; + } + // longest 9 - isnotnull + if (tmp.length() > 8) { + whileI = false; + } + } + break; + case 'n': + case 'N': + boolean whileN = true; + StringBuilder operN = new StringBuilder(); + operN.append(c); + while (whileN) { + i++; + if (i >= array.length) { + whileN = false; + } + + if (array[i] != ' ' && array[i] != '\n') { + operN.append(array[i]); + } else { + continue; + } + + String tmp = operN.toString().toLowerCase(); + + if (tmp.equals("notin")) { + whereBuf.append(MARKER_NOT_IN); + listOperator = true; + whileN = false; + operatorPosition = false; + } + + // longest 5 - notin + if (tmp.length() > 4) { + whileN = false; + } + } + break; + default: + whereBuf.append(c); + } + } else { + whereBuf.append(c); + } + } else { + whereBuf.append(c); + } + } + } + } + + if (wherePart && StringUtils.isBlank(where)) { + throw new UniverseException("Incorrect block where"); + } + + UniverseQuery universeQuery = new UniverseQuery(select.toString().trim(), + where, universeInfo); + + if (!universeQuery.isCorrect()) { + throw new UniverseException("Incorrect query"); + } + + return universeQuery; + } + + private String parseWhere(String where, Map nodeInfos) + throws UniverseException { + List out = new ArrayList<>(); + Stack stack = new Stack<>(); + + where = where.replaceAll("\\s*", ""); + + Set operationSymbols = new HashSet<>(OPERATIONS.keySet()); + operationSymbols.add(LEFT_BRACE); + operationSymbols.add(RIGHT_BRACE); + + int index = 0; + + boolean findNext = true; + while (findNext) { + int nextOperationIndex = where.length(); + String nextOperation = ""; + for (String operation : operationSymbols) { + int i = where.indexOf(operation, index); + if (i >= 0 && i < nextOperationIndex) { + nextOperation = operation; + nextOperationIndex = i; + } + } + if (nextOperationIndex == where.length()) { + findNext = false; + } else { + if (index != nextOperationIndex) { + out.add(where.substring(index, nextOperationIndex)); + } + if (nextOperation.equals(LEFT_BRACE)) { + stack.push(nextOperation); + } + else if (nextOperation.equals(RIGHT_BRACE)) { + while (!stack.peek().equals(LEFT_BRACE)) { + out.add(stack.pop()); + if (stack.empty()) { + throw new UniverseException("Unmatched brackets"); + } + } + stack.pop(); + } + else { + while (!stack.empty() && !stack.peek().equals(LEFT_BRACE) && + (OPERATIONS.get(nextOperation) >= OPERATIONS.get(stack.peek()))) { + out.add(stack.pop()); + } + stack.push(nextOperation); + } + index = nextOperationIndex + nextOperation.length(); + } + } + if (index != where.length()) { + out.add(where.substring(index)); + } + while (!stack.empty()) { + out.add(stack.pop()); + } + StringBuffer result = new StringBuffer(); + if (!out.isEmpty()) + result.append(out.remove(0)); + while (!out.isEmpty()) + result.append(" ").append(out.remove(0)); + + // result contains the reverse polish notation + return convertWhereToXml(result.toString(), nodeInfos); + } + + private String parseResultObj(String resultObj, Map nodeInfos) + throws UniverseException { + if (StringUtils.isNotBlank(resultObj)) { + UniverseNodeInfo nodeInfo = nodeInfos.get(resultObj.trim()); + if (nodeInfo != null) { + return String.format(RESULT_OBJ_TEMPLATE, nodeInfo.getNodePath(), nodeInfo.getId()); + } + throw new UniverseException(String.format("Not found information about: \"%s\"", + resultObj.trim())); + } + + return StringUtils.EMPTY; + } + + private String convertWhereToXml(String rpn, Map nodeInfos) + throws UniverseException { + StringTokenizer tokenizer = new StringTokenizer(rpn, " "); + + Stack stack = new Stack(); + + while (tokenizer.hasMoreTokens()) { + StringBuilder tmp = new StringBuilder(); + String token = tokenizer.nextToken(); + if (!OPERATIONS.keySet().contains(token)) { + stack.push(token.trim()); + } else { + String rightOperand = revertReplace(stack.pop()); + String operator = token.replaceAll("^#|#$", ""); + + if (token.equalsIgnoreCase(MARKER_NOT_NULL) || token.equalsIgnoreCase(MARKER_NULL)) { + UniverseNodeInfo rightOperandInfo = nodeInfos.get(rightOperand); + stack.push(String.format(COMPARISON_FILTER, rightOperandInfo.getId(), + rightOperandInfo.getNodePath(), operator)); + continue; + } + + if (token.equalsIgnoreCase(MARKER_FILTER)) { + UniverseNodeInfo rightOperandInfo = nodeInfos.get(rightOperand); + stack.push(String.format(PREDEFINED_FILTER_TEMPLATE, rightOperandInfo.getNodePath(), + rightOperandInfo.getId())); + continue; + } + + String leftOperand = stack.empty() ? null : revertReplace(stack.pop()); + + if (token.equalsIgnoreCase(MARKER_AND) || token.equalsIgnoreCase(MARKER_OR)) { + if (rightOperand.matches("^\\[.*\\]$")) { + UniverseNodeInfo rightOperandInfo = nodeInfos.get(rightOperand); + if (rightOperandInfo == null) { + throw new UniverseException(String.format("Not found information about: \"%s\"", + rightOperand)); + } + rightOperand = String.format(PREDEFINED_FILTER_TEMPLATE, + rightOperandInfo.getNodePath(), rightOperandInfo.getId()); + } + if (leftOperand.matches("^\\[.*\\]$")) { + UniverseNodeInfo leftOperandInfo = nodeInfos.get(leftOperand); + if (leftOperandInfo == null) { + throw new UniverseException(String.format("Not found information about: \"%s\"", + leftOperand)); + } + leftOperand = String.format(PREDEFINED_FILTER_TEMPLATE, leftOperandInfo.getNodePath(), + leftOperandInfo.getId()); + } + tmp.append(String.format("<%s>\n", operator)); + tmp.append(leftOperand); + tmp.append("\n"); + tmp.append(rightOperand); + tmp.append("\n"); + tmp.append(String.format("\n", operator)); + stack.push(tmp.toString()); + continue; + } + + UniverseNodeInfo leftOperandInfo = nodeInfos.get(leftOperand); + if (leftOperandInfo == null) { + throw new UniverseException(String.format("Not found information about: \"%s\"", + leftOperand)); + } + if (token.equalsIgnoreCase(MARKER_IN) || token.equalsIgnoreCase(MARKER_NOT_IN)) { + String listValues = rightOperand.replaceAll("^\\(|\\)$", "").trim(); + boolean startItem = false; + List values = new ArrayList<>(); + StringBuilder value = new StringBuilder(); + boolean isNumericList = false; + if (listValues.charAt(0) != '\'') { + isNumericList = true; + } + if (isNumericList) { + String[] nums = listValues.split(","); + for (String num : nums) { + values.add(num.trim()); + } + } else { + for (int i = 0; i < listValues.length(); i++) { + char c = listValues.charAt(i); + if (c == '\'' && (i == 0 || listValues.charAt(i - 1) != '\\')) { + startItem = !startItem; + if (!startItem) { + values.add(value.toString()); + value = new StringBuilder(); + } + continue; + } + if (startItem) { + value.append(c); + } + } + } + + if (!values.isEmpty()) { + tmp.append(String.format(COMPRASION_START_TEMPLATE, leftOperandInfo.getNodePath(), + operator, leftOperandInfo.getId())); + tmp.append(CONST_OPERAND_START_TEMPLATE); + String type = isNumericList ? "Numeric" : "String"; + for (String v : values) { + tmp.append(String.format(CONST_OPERAND_VALUE_TEMPLATE, type, v)); + } + tmp.append(CONST_OPERAND_END_TEMPLATE); + tmp.append(COMPRASION_END_TEMPLATE); + stack.push(tmp.toString()); + } + continue; + } + + // EqualTo, LessThanOrEqualTo, NotEqualTo, LessThan, GreaterThanOrEqualTo, GreaterThan + UniverseNodeInfo rightOperandInfo = null; + if (rightOperand.startsWith("[") && rightOperand.endsWith("]")) { + rightOperandInfo = nodeInfos.get(rightOperand); + if (rightOperandInfo == null) { + throw new UniverseException(String.format("Not found information about: \"%s\"", + rightOperand)); + } + } + if (OPERATIONS.containsKey(token)) { + if (rightOperandInfo != null) { + tmp.append(String.format(COMPRASION_START_TEMPLATE, leftOperandInfo.getNodePath(), + operator, leftOperandInfo.getId())); + tmp.append(String.format(OBJECT_OPERAND_TEMPLATE, rightOperandInfo.getId(), + rightOperandInfo.getNodePath())); + tmp.append(COMPRASION_END_TEMPLATE); + } else { + String type = rightOperand.startsWith("'") ? "String" : "Numeric"; + String value = rightOperand.replaceAll("^'|'$", ""); + tmp.append(String.format(COMPRASION_START_TEMPLATE, leftOperandInfo.getNodePath(), + operator, leftOperandInfo.getId())); + tmp.append(CONST_OPERAND_START_TEMPLATE); + tmp.append(String.format(CONST_OPERAND_VALUE_TEMPLATE, type, value)); + tmp.append(CONST_OPERAND_END_TEMPLATE); + tmp.append(COMPRASION_END_TEMPLATE); + } + stack.push(tmp.toString()); + continue; + } + throw new UniverseException(String.format("Incorrect syntax after: \"%s\"", leftOperand)); + } + } + + return stack.pop(); + } + + private String revertReplace(String s) { + return s.replaceAll(MARKER_BACKSPACE, " ") + .replaceAll(MARKER_LEFT_BRACE, "(") + .replaceAll(MARKER_RIGHT_BRACE, ")"); + } + + private boolean isFilter(String buf, String after) { + boolean result = false; + String[] parts = buf.trim().split("\\s"); + if (parts[parts.length - 1].matches("^\\[.*\\]$")) { + // check before + if (parts.length == 1) { + result = true; + } else { + int count = parts.length - 2; + Set operations = new HashSet(OPERATIONS.keySet()); + operations.remove(MARKER_AND); + operations.remove(MARKER_OR); + while (count >= 0) { + String p = parts[count]; + if (StringUtils.isNotBlank(p)) { + if (!operations.contains(p)) { + result = true; + break; + } else { + return false; + } + } + count--; + } + } + after = after.trim(); + // check after + if (result && !after.startsWith("and") && !after.startsWith("or") && + !after.startsWith(";") && StringUtils.isNotBlank(after)) { + result = false; + } + } + + return result; + } +} diff --git a/sap/src/main/resources/interpreter-setting.json b/sap/src/main/resources/interpreter-setting.json new file mode 100644 index 00000000000..cb5cf946b74 --- /dev/null +++ b/sap/src/main/resources/interpreter-setting.json @@ -0,0 +1,42 @@ +[ + { + "group": "sap", + "name": "universe", + "className": "org.apache.zeppelin.sap.UniverseInterpreter", + "defaultInterpreter": true, + "properties": { + "universe.api.url": { + "envName": null, + "propertyName": "universe.api.url", + "defaultValue": "http://localhost:6405/biprws", + "description": "API url of Universe", + "type": "url" + }, + "universe.user": { + "envName": null, + "propertyName": "universe.user", + "defaultValue": "", + "description": "Username for API of Universe", + "type": "string" + }, + "universe.password": { + "envName": null, + "propertyName": "universe.password", + "defaultValue": "", + "description": "Password for API of Universe", + "type": "password" + }, + "universe.authType": { + "envName": null, + "propertyName": "universe.password", + "defaultValue": "secEnterprise", + "description": "Type of authentication for API of Universe. Available values: secEnterprise, secLDAP, secWinAD, secSAPR3", + "type": "string" + } + }, + "editor": { + "editOnDblClick": false, + "completionKey": "TAB" + } + } +] diff --git a/sap/src/main/resources/universe.keywords b/sap/src/main/resources/universe.keywords new file mode 100644 index 00000000000..0811bfbe3bd --- /dev/null +++ b/sap/src/main/resources/universe.keywords @@ -0,0 +1 @@ +universe,select,where,and,or,is null,is not null,in \ No newline at end of file diff --git a/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseCompleterTest.java b/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseCompleterTest.java new file mode 100644 index 00000000000..91a42179cd6 --- /dev/null +++ b/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseCompleterTest.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.zeppelin.sap.universe; + +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.completer.CachedCompleter; +import org.junit.Before; +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Universe completer unit tests + */ +public class UniverseCompleterTest { + + private UniverseCompleter universeCompleter; + private UniverseUtil universeUtil; + private UniverseClient universeClient; + + @Before + public void beforeTest() throws UniverseException { + universeCompleter = new UniverseCompleter(0); + universeUtil = new UniverseUtil(); + Map universes = new HashMap<>(); + + universes.put("testUniverse", new UniverseInfo("1", "testUniverse", "uvx")); + universes.put("test with space", new UniverseInfo("2", "test with space", "uvx")); + universes.put("(GLOBAL) universe", new UniverseInfo("3", "(GLOBAL) universe", "uvx")); + UniverseInfo universeInfo = new UniverseInfo("1", "testUniverse", "uvx"); + Map testUniverseNodes = new HashMap<>(); + testUniverseNodes.put("[Dimension].[Test].[name1]", + new UniverseNodeInfo("name1id", "name1", "dimension", "Dimension\\Test", + "Dimension|folder\\Test|folder\\name1|dimension")); + testUniverseNodes.put("[Dimension].[Test].[name2]", + new UniverseNodeInfo("name2id", "name2", "dimension", "Dimension\\Test", + "Dimension|folder\\Test|folder\\name2|dimension")); + testUniverseNodes.put("[Filter].[name3]", + new UniverseNodeInfo("name3id", "name3", "filter", "Filter", + "Filter|folder\\name3|filter")); + testUniverseNodes.put("[Filter].[name4]", + new UniverseNodeInfo("name4id", "name4", "filter", "Filter", + "Filter|folder\\name4|filter")); + testUniverseNodes.put("[Measure].[name5]", + new UniverseNodeInfo("name5id", "name5", "measure", "Measure", + "Measure|folder\\name5|measure")); + + universeClient = mock(UniverseClient.class); + when(universeClient.getUniverseInfo(anyString())).thenReturn(universeInfo); + when(universeClient.getUniverseNodesInfo(anyString(), anyString())) + .thenReturn(testUniverseNodes); + when(universeClient.getUniversesMap()).thenReturn(universes); + } + + @Test + public void testCreateUniverseNameCompleter() { + String buffer = "universe ["; + List candidates = new ArrayList<>(); + universeCompleter.createOrUpdate(universeClient, null, buffer, 9); + CachedCompleter completer = universeCompleter.getUniverseCompleter(); + assertNull(completer); + universeCompleter.createOrUpdate(universeClient, null, buffer, 10); + completer = universeCompleter.getUniverseCompleter(); + assertNotNull(completer); + + completer.getCompleter().complete(StringUtils.EMPTY, 0, candidates); + assertEquals(3, candidates.size()); + } + + @Test + public void testCreateUniverseNodesCompleter() { + String buffer = "universe [testUniverse]; select ["; + List candidates = new ArrayList<>(); + universeCompleter.createOrUpdate(universeClient, null, buffer, 32); + Map completerMap = universeCompleter.getUniverseInfoCompletersMap(); + assertFalse(completerMap.containsKey("testUniverse")); + universeCompleter.createOrUpdate(universeClient, null, buffer, 33); + completerMap = universeCompleter.getUniverseInfoCompletersMap(); + assertTrue(completerMap.containsKey("testUniverse")); + CachedCompleter completer = completerMap.get("testUniverse"); + + completer.getCompleter().complete(StringUtils.EMPTY, 0, candidates); + assertEquals(3, candidates.size()); + List candidatesStrings = new ArrayList<>(); + for (Object o : candidates) { + UniverseNodeInfo info = (UniverseNodeInfo) o; + candidatesStrings.add(info.getName()); + } + List expected = Arrays.asList("Filter", "Measure", "Dimension"); + Collections.sort(candidatesStrings); + Collections.sort(expected); + assertEquals(expected, candidatesStrings); + } + + @Test + public void testNestedUniverseNodes() { + String buffer = "universe [testUniverse]; select [Dimension].[Test].[n"; + List candidates = new ArrayList<>(); + + universeCompleter.createOrUpdate(universeClient, null, buffer, 53); + Map completerMap = universeCompleter.getUniverseInfoCompletersMap(); + assertTrue(completerMap.containsKey("testUniverse")); + CachedCompleter completer = completerMap.get("testUniverse"); + + completer.getCompleter().complete("[Dimension].[Test].[n", 21, candidates); + assertEquals(2, candidates.size()); + List candidatesStrings = new ArrayList<>(); + for (Object o : candidates) { + UniverseNodeInfo info = (UniverseNodeInfo) o; + candidatesStrings.add(info.getName()); + } + List expected = Arrays.asList("name1", "name2"); + Collections.sort(candidatesStrings); + Collections.sort(expected); + assertEquals(expected, candidatesStrings); + } +} diff --git a/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseUtilTest.java b/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseUtilTest.java new file mode 100644 index 00000000000..81a027e1ced --- /dev/null +++ b/sap/src/test/java/org/apache/zeppelin/sap/universe/UniverseUtilTest.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.sap.universe; + +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class UniverseUtilTest { + + private UniverseClient universeClient; + private UniverseUtil universeUtil; + + @Before + public void beforeTest() throws UniverseException { + universeUtil = new UniverseUtil(); + UniverseInfo universeInfo = new UniverseInfo("1", "testUniverse", "uvx"); + Map testUniverseNodes = new HashMap<>(); + testUniverseNodes.put("[Dimension].[Test].[name1]", + new UniverseNodeInfo("name1id", "name1", "dimension", "Dimension\\Test", + "Dimension|folder\\Test|folder\\name1|dimension")); + testUniverseNodes.put("[Dimension].[Test].[name2]", + new UniverseNodeInfo("name2id", "name2", "dimension", "Filter\\Test", + "Dimension|folder\\Test|folder\\name2|dimension")); + testUniverseNodes.put("[Filter].[name3]", + new UniverseNodeInfo("name3id", "name3", "filter", "Filter", + "Filter|folder\\name3|filter")); + testUniverseNodes.put("[Filter].[name4]", + new UniverseNodeInfo("name4id", "name4", "filter", "Filter", + "Filter|folder\\name4|filter")); + testUniverseNodes.put("[Measure].[name5]", + new UniverseNodeInfo("name5id", "name5", "measure", "Measure", + "Measure|folder\\name5|measure")); + + universeClient = mock(UniverseClient.class); + when(universeClient.getUniverseInfo(anyString())).thenReturn(universeInfo); + when(universeClient.getUniverseNodesInfo(anyString(), anyString())) + .thenReturn(testUniverseNodes); + } + + @Test + public void testForConvert() throws UniverseException { + String request = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] and [Dimension].[Test].[name2] > 1;"; + UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); + assertNotNull(universeQuery); + assertNotNull(universeQuery.getUniverseInfo()); + assertEquals("\n" + + "\n" + + "", universeQuery.getSelect()); + assertEquals("\n" + + "\n" + + "\n\n" + + "\n" + + "\n" + + "1\n" + + "\n" + + "\n" + + "\n\n" + + "\n", universeQuery.getWhere()); + assertEquals("testUniverse", universeQuery.getUniverseInfo().getName()); + } + + @Test + public void testConvertConditions() throws UniverseException { + String request = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] " + + "and [Dimension].[Test].[name2] >= 1 " + + "and [Dimension].[Test].[name2] < 20 " + + "and [Dimension].[Test].[name1] <> 'test' " + + "and [Dimension].[Test].[name1] is not null " + + "and [Measure].[name5] is null" + + "and [Dimension].[Test].[name1] in ('var1', 'v a r 2') " + + "and [Dimension].[Test].[name1] in ('var1','withoutspaces')" + + "and [Dimension].[Test].[name1] in ('one value')" + + "and [Dimension].[Test].[name2] in (1,3,4);"; + UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); + assertNotNull(universeQuery); + assertEquals("\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "1\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "20\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "test\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n\n" + + "\n\n" + + "\n\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "var1\n" + + "\n" + + "\n" + + "v a r 2\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "var1\n" + + "\n" + + "\n" + + "withoutspaces\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "one value\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "1\n" + + "\n" + + "\n" + + "3\n" + + "\n" + + "\n" + + "4\n" + + "\n" + + "\n" + + "\n\n" + + "\n", + universeQuery.getWhere()); + } + + @Test(expected = UniverseException.class) + public void testFailConvertWithoutUniverse() throws UniverseException { + String request = "universe ;\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] and [Dimension].[Test].[name2] > 1;"; + universeUtil.convertQuery(request, universeClient, null); + } + + @Test(expected = UniverseException.class) + public void testFailConvertWithIncorrectSelect() throws UniverseException { + String request = "universe [testUniverse];\n" + + "select [not].[exist];"; + universeUtil.convertQuery(request, universeClient, null); + } + + + @Test(expected = UniverseException.class) + public void testFailConvertWithIncorrectCondition() throws UniverseException { + String request = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name;"; + universeUtil.convertQuery(request, universeClient, null); + } + + @Test + public void testFiltersConditions() throws UniverseException { + String request1 = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3];"; + String request2 = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Measure].[name5] > 2 and [Filter].[name3];"; + String request3 = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] or [Measure].[name5];"; + String request4 = "universe [testUniverse];\n" + + "select [Measure].[name5]\n" + + "where [Filter].[name3] and [Measure].[name5] is null;"; + UniverseQuery universeQuery = universeUtil.convertQuery(request1, universeClient, null); + assertEquals("\n", + universeQuery.getWhere()); + universeQuery = universeUtil.convertQuery(request2, universeClient, null); + assertEquals("\n" + + "\n" + + "\n" + + "\n" + + "2\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n", + universeQuery.getWhere()); + universeQuery = universeUtil.convertQuery(request3, universeClient, null); + assertEquals("\n" + + "\n\n" + + "\n\n" + + "\n", + universeQuery.getWhere()); + universeQuery = universeUtil.convertQuery(request4, universeClient, null); + assertEquals("\n" + + "\n\n" + + "\n\n" + + "\n", + universeQuery.getWhere()); + } + + @Test + public void testNestedConditions() throws UniverseException { + String request = "universe [testUniverse];\n" + + "select [Dimension].[Test].[name2]\n" + + "where ([Measure].[name5] = 'text' or ([Dimension].[Test].[name1] in ('1','2', '3') and\n" + + "[Dimension].[Test].[name2] is not null)) and ([Filter].[name4] or [Measure].[name5] >=12)\n" + + "or [Dimension].[Test].[name2] not in (31, 65, 77);"; + UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); + assertEquals("\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "text\n" + + "\n" + + "\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "1\n" + + "\n" + + "\n" + + "2\n" + + "\n" + + "\n" + + "3\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n\n" + + "\n\n" + + "\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "12\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + + "31\n" + + "\n" + + "\n" + + "65\n" + + "\n" + + "\n" + + "77\n" + + "\n" + + "\n" + + "\n\n" + + "\n", + universeQuery.getWhere()); + } + + @Test + public void testWithoutConditions() throws UniverseException { + String request = "universe [testUniverse];\n" + + "select [Dimension].[Test].[name2], [Measure].[name5],\n" + + "[Dimension].[Test].[name1] ;"; + UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); + assertNull(universeQuery.getWhere()); + assertEquals("\n" + + "\n" + + "\n" + + "\n" + + "", + universeQuery.getSelect()); + } + + @Test + public void testCaseSensitive() throws UniverseException { + String request = "uniVersE [testUniverse];\n" + + "seLEct [Dimension].[Test].[name2], [Measure].[name5]\n" + + "whERE [Dimension].[Test].[name2] Is NULl Or [Measure].[name5] IN (1,2) aNd [Measure].[name5] is NOT nUll;"; + UniverseQuery universeQuery = universeUtil.convertQuery(request, universeClient, null); + assertEquals("\n" + + "\n" + + "\n" + + "", + universeQuery.getSelect()); + assertEquals("\n" + + "\n\n" + + "\n" + + "\n" + + "\n" + "\n" + "1\n" + + "\n" + + "\n" + + "2\n" + + "\n" + + "\n" + + "\n\n" + + "\n\n" + + "\n\n" + + "\n", + universeQuery.getWhere()); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/CompletionType.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/CompletionType.java index 20cceda8127..fc5f3809bab 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/CompletionType.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/CompletionType.java @@ -24,5 +24,6 @@ public enum CompletionType { setting, command, keyword, - path + path, + universe } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 81f9341e36e..bae0c3875d2 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -692,7 +692,8 @@ public enum ConfVars { + "org.apache.zeppelin.beam.BeamInterpreter," + "org.apache.zeppelin.scio.ScioInterpreter," + "org.apache.zeppelin.groovy.GroovyInterpreter," - + "org.apache.zeppelin.neo4j.Neo4jCypherInterpreter" + + "org.apache.zeppelin.neo4j.Neo4jCypherInterpreter," + + "org.apache.zeppelin.sap.UniverseInterpreter" ), ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 971257cf502..ee1713058b2 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -800,7 +800,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat name: v.name, value: v.value, meta: v.meta, - caption: computeCaption(v.value, v.meta), + caption: computeCaption(v.name, v.meta), score: 300, }); } From 66dada66e25a1d24cc235fd44e58a0507fe07b84 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 2 Mar 2018 15:47:35 -0800 Subject: [PATCH 059/386] [ZEPPELIN-3283] Remove redundant method InterpreterOutput.getInterpreterResultMessages() ### What is this PR for? Remove Redundant method `InterpreterOutput.getInterpreterResultMessages()` ### What type of PR is it? Refactoring ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-3283 ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Lee moon soo Closes #2828 from Leemoonsoo/ZEPPELIN-3283 and squashes the following commits: 12771a145 [Lee moon soo] remove unused import 82bda9c02 [Lee moon soo] Remove redundant method getInterpreterResultMessages() --- .../python/IPythonInterpreterTest.java | 48 +++++++++---------- .../spark/IPySparkInterpreterTest.java | 16 +++---- .../interpreter/InterpreterOutput.java | 11 ----- 3 files changed, 32 insertions(+), 43 deletions(-) diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index 480cae311cd..869d390720f 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -88,7 +88,7 @@ public void testGrpcFrameSize() throws InterpreterException, IOException { InterpreterContext context = getInterpreterContext(); result = interpreter.interpret("print(11111111111111111111111111111)", context); assertEquals(InterpreterResult.Code.ERROR, result.code()); - List interpreterResultMessages = context.out.getInterpreterResultMessages(); + List interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertTrue(interpreterResultMessages.get(0).getData().contains("Frame size 32 exceeds maximum: 4")); @@ -120,7 +120,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("import sys\nprint(sys.version[0])", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); Thread.sleep(100); - List interpreterResultMessages = context.out.getInterpreterResultMessages(); + List interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); boolean isPython2 = interpreterResultMessages.get(0).getData().equals("2\n"); @@ -129,7 +129,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("'hello world'", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertEquals("'hello world'", interpreterResultMessages.get(0).getData()); @@ -138,7 +138,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("print(u'你好')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertEquals("你好\n", interpreterResultMessages.get(0).getData()); @@ -147,7 +147,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("'hello world'\n'hello world2'", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertEquals("'hello world2'", interpreterResultMessages.get(0).getData()); @@ -156,7 +156,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("print('hello world')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertEquals("hello world\n", interpreterResultMessages.get(0).getData()); @@ -165,7 +165,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("print('hello world')\nprint('hello world2')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertEquals("hello world\nhello world2\n", interpreterResultMessages.get(0).getData()); @@ -174,7 +174,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("abc=1",context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(0, interpreterResultMessages.size()); // if block @@ -182,7 +182,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("if abc > 0:\n\tprint('True')\nelse:\n\tprint('False')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertEquals("True\n", interpreterResultMessages.get(0).getData()); @@ -191,7 +191,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("for i in range(3):\n\tprint(i)", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertEquals("0\n1\n2\n", interpreterResultMessages.get(0).getData()); @@ -200,7 +200,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("print(unknown)", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.ERROR, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertTrue(interpreterResultMessages.get(0).getData().contains("name 'unknown' is not defined")); @@ -209,7 +209,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("1/0", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.ERROR, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertTrue(interpreterResultMessages.get(0).getData().contains("ZeroDivisionError")); @@ -220,7 +220,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep "greet('Jack')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertEquals("Hello Jack\n",interpreterResultMessages.get(0).getData()); @@ -229,7 +229,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("print('there is no Error: ok')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertEquals("there is no Error: ok\n", interpreterResultMessages.get(0).getData()); @@ -267,7 +267,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret(st, context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(0, interpreterResultMessages.size()); // now we can get the completion for `a.` @@ -296,7 +296,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("range?", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertTrue(interpreterResultMessages.get(0).getData().contains("range(stop)")); // timeit @@ -304,7 +304,7 @@ public static void testInterpreter(final Interpreter interpreter) throws IOExcep result = interpreter.interpret("%timeit range(100)", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertTrue(interpreterResultMessages.get(0).getData().contains("loops")); // cancel @@ -327,7 +327,7 @@ public void run() { result = interpreter.interpret("import time\ntime.sleep(10)", context2); Thread.sleep(100); assertEquals(InterpreterResult.Code.ERROR, result.code()); - interpreterResultMessages = context2.out.getInterpreterResultMessages(); + interpreterResultMessages = context2.out.toInterpreterResultMessage(); assertTrue(interpreterResultMessages.get(0).getData().contains("KeyboardInterrupt")); // matplotlib @@ -335,7 +335,7 @@ public void run() { result = interpreter.interpret("%matplotlib inline\nimport matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); // the order of IMAGE and TEXT is not determined // check there must be one IMAGE output boolean hasImageOutput = false; @@ -367,7 +367,7 @@ public void run() { "output_notebook(notebook_type='zeppelin')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(2, interpreterResultMessages.size()); assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType()); assertTrue(interpreterResultMessages.get(0).getData().contains("Loading BokehJS")); @@ -384,7 +384,7 @@ public void run() { "show(p)", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(2, interpreterResultMessages.size()); assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType()); assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType()); @@ -399,7 +399,7 @@ public void run() { " facet_wrap(\"clarity\")", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); // the order of IMAGE and TEXT is not determined // check there must be one IMAGE output hasImageOutput = false; @@ -417,7 +417,7 @@ public void run() { result = interpreter.interpret("z.input(name='text_1', defaultValue='value_1')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertTrue(interpreterResultMessages.get(0).getData().contains("'value_1'")); assertEquals(1, context.getGui().getForms().size()); assertTrue(context.getGui().getForms().get("text_1") instanceof TextBox); @@ -453,7 +453,7 @@ public void run() { context = getInterpreterContext(); result = interpreter.interpret("import pandas as pd\ndf = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})\nz.show(df)", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(InterpreterResult.Type.TABLE, interpreterResultMessages.get(0).getType()); assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData()); } diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java index 17c2af8c370..d66f89f966b 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java @@ -103,7 +103,7 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx InterpreterResult result = iPySparkInterpreter.interpret("sc.version", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - String sparkVersion = context.out.getInterpreterResultMessages().get(0).getData(); + String sparkVersion = context.out.toInterpreterResultMessage().get(0).getData(); // spark url is sent verify(mockRemoteEventClient).onMetaInfosReceived(any(Map.class)); @@ -111,7 +111,7 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx result = iPySparkInterpreter.interpret("sc.range(1,10).sum()", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - List interpreterResultMessages = context.out.getInterpreterResultMessages(); + List interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals("45", interpreterResultMessages.get(0).getData()); // spark job url is sent verify(mockRemoteEventClient).onParaInfosReceived(any(String.class), any(String.class), any(Map.class)); @@ -121,7 +121,7 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx if (!isSpark2(sparkVersion)) { result = iPySparkInterpreter.interpret("df = sqlContext.createDataFrame([(1,'a'),(2,'b')])\ndf.show()", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals( "+---+---+\n" + "| _1| _2|\n" + @@ -133,7 +133,7 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx context = getInterpreterContext(); result = iPySparkInterpreter.interpret("z.show(df)", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals( "_1 _2\n" + "1 a\n" + @@ -141,7 +141,7 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx } else { result = iPySparkInterpreter.interpret("df = spark.createDataFrame([(1,'a'),(2,'b')])\ndf.show()", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals( "+---+---+\n" + "| _1| _2|\n" + @@ -153,7 +153,7 @@ public void testBasics() throws InterruptedException, IOException, InterpreterEx context = getInterpreterContext(); result = iPySparkInterpreter.interpret("z.show(df)", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals( "_1 _2\n" + "1 a\n" + @@ -169,7 +169,7 @@ public void run() { assertEquals(InterpreterResult.Code.ERROR, result.code()); List interpreterResultMessages = null; try { - interpreterResultMessages = context2.out.getInterpreterResultMessages(); + interpreterResultMessages = context2.out.toInterpreterResultMessage(); assertTrue(interpreterResultMessages.get(0).getData().contains("KeyboardInterrupt")); } catch (IOException e) { e.printStackTrace(); @@ -207,7 +207,7 @@ public void run() { "ssc.stop(stopSparkContext=False, stopGraceFully=True)", context); Thread.sleep(1000); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - interpreterResultMessages = context.out.getInterpreterResultMessages(); + interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); assertTrue(interpreterResultMessages.get(0).getData().contains("(0, 100)")); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java index d61621ba1bf..44f83f4de45 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.io.OutputStream; import java.net.URL; -import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -120,16 +119,6 @@ public void onUpdate(InterpreterResultMessageOutput out) { }; } - public List getInterpreterResultMessages() throws IOException { - synchronized (resultMessageOutputs) { - List resultMessages = new ArrayList<>(); - for (InterpreterResultMessageOutput output : this.resultMessageOutputs) { - resultMessages.add(output.toInterpreterResultMessage()); - } - return resultMessages; - } - } - public InterpreterResultMessageOutput getCurrentOutput() { synchronized (resultMessageOutputs) { return currentOut; From d90716d4d00f5b83d2bd1f16cadc88118a112f74 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 2 Mar 2018 17:42:41 -0800 Subject: [PATCH 060/386] [ZEPPELIN-3284] z.getInterpreterContext().out().clear() broken in Python interpreter ### What is this PR for? ``` %python import time print("Hello") time.sleep(0.5) # in case of Ipython kernel, print may not immediately flushed and cleared. z.getInterpreterContext().out().clear() print("world") ``` Expected to print `world` only. This worked in zeppelin-0.7.x and it'll be nice keep this feature in the future version. ### What type of PR is it? Bug fix | Improvement ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-3284 ### How should this be tested? * First time? Setup Travis CI as described on https://zeppelin.apache.org/contribution/contributions.html#continuous-integration * Strongly recommended: add automated unit tests for any new or changed behavior * Outline any manual steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? * Is there breaking changes for older versions? * Does this needs documentation? Author: Lee moon soo Closes #2829 from Leemoonsoo/ZEPPELIN-3284 and squashes the following commits: e0dcda61f [Lee moon soo] make z.getInterpreterContext().out().clear() work in IPythonInterprete 224a2df8e [Lee moon soo] make z.getInterpreterContext().out().clear() work in PythonInterpreter --- .../java/org/apache/zeppelin/python/PythonInterpreter.java | 1 + python/src/main/resources/grpc/python/zeppelin_python.py | 3 +++ python/src/main/resources/python/zeppelin_python.py | 2 +- .../org/apache/zeppelin/python/IPythonInterpreterTest.java | 5 +++++ .../org/apache/zeppelin/python/PythonInterpreterTest.java | 6 ++++++ 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index 028f1c6a802..186440901d8 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -388,6 +388,7 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr zeppelinContext.setGui(context.getGui()); zeppelinContext.setNoteGui(context.getNoteGui()); + zeppelinContext.setInterpreterContext(context); if (!pythonscriptRunning) { return new InterpreterResult(Code.ERROR, "python process not running" diff --git a/python/src/main/resources/grpc/python/zeppelin_python.py b/python/src/main/resources/grpc/python/zeppelin_python.py index 1a9e8afa176..6d7c33e9014 100644 --- a/python/src/main/resources/grpc/python/zeppelin_python.py +++ b/python/src/main/resources/grpc/python/zeppelin_python.py @@ -33,6 +33,9 @@ def __init__(self, z): self.javaList = gateway.jvm.java.util.ArrayList self.max_result = z.getMaxResult() + def getInterpreterContext(self): + return self.z.getInterpreterContext() + def input(self, name, defaultValue=""): return self.z.input(name, defaultValue) diff --git a/python/src/main/resources/python/zeppelin_python.py b/python/src/main/resources/python/zeppelin_python.py index 519d1af4557..f4ea2c71619 100644 --- a/python/src/main/resources/python/zeppelin_python.py +++ b/python/src/main/resources/python/zeppelin_python.py @@ -61,7 +61,7 @@ def __init__(self, z): self._setup_matplotlib() def getInterpreterContext(self): - return self.z.getCurrentInterpreterContext() + return self.z.getInterpreterContext() def input(self, name, defaultValue=""): return self.z.input(name, defaultValue) diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index 869d390720f..2d5d832fc98 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -456,6 +456,11 @@ public void run() { interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(InterpreterResult.Type.TABLE, interpreterResultMessages.get(0).getType()); assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData()); + + // clear output + context = getInterpreterContext(); + result = interpreter.interpret("import time\nprint(\"Hello\")\ntime.sleep(0.5)\nz.getInterpreterContext().out().clear()\nprint(\"world\")\n", context); + assertEquals("%text world\n", context.out.getCurrentOutput().toString()); } private static InterpreterContext getInterpreterContext() { diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java index 1143b9e4629..c0beccbd9da 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java @@ -123,6 +123,12 @@ public void testRedefinitionZeppelinContext() throws InterpreterException { assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyValidCode, context).code()); } + @Test + public void testOutputClear() throws InterpreterException { + InterpreterResult result = pythonInterpreter.interpret("print(\"Hello\")\nz.getInterpreterContext().out().clear()\nprint(\"world\")\n", context); + assertEquals("%text world\n", out.getCurrentOutput().toString()); + } + @Override public void onUpdateAll(InterpreterOutput out) { From d6cdd56d8fbc551dee7c334eb403c19bc65d6e1f Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Thu, 22 Feb 2018 21:05:23 +0800 Subject: [PATCH 061/386] ZEPPELIN-3254. Enable one build of zeppelin to work with multiple versions of spark ### What is this PR for? This PR is trying to add test for one build of zeppelin to work with multiple versions of spark. There's 2 main system test * `SparkIntegrationTest` ( It launches spark interpreter from interpreter setting component in 3 modes: local, yarn-client, yarn-cluster) * `ZeppelinSparkClusterTest` (It launch spark interpreter in local mode from Paragraph side ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3254 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2832 from zjffdu/ZEPPELIN-3254 and squashes the following commits: 3b06ed9 [Jeff Zhang] ZEPPELIN-3254. Enable one build of zeppelin to work with multiple versions of spark --- .travis.yml | 30 +- .../zeppelin/spark/SparkRInterpreter.java | 8 +- .../interpreter/InterpreterProperty.java | 1 + zeppelin-server/notebook/.python.recovery.crc | Bin 12 -> 0 bytes zeppelin-server/notebook/python.recovery | 1 - zeppelin-server/pom.xml | 9 + .../zeppelin/rest/AbstractTestRestApi.java | 211 +-- .../rest/ZeppelinSparkClusterTest.java | 1140 +++++++++-------- .../src/test/resources/log4j.properties | 3 +- zeppelin-zengine/pom.xml | 14 + .../interpreter/SparkDownloadUtils.java | 91 ++ ...odeTest.java => SparkIntegrationTest.java} | 49 +- .../src/test/resources/log4j.properties | 6 +- 13 files changed, 832 insertions(+), 731 deletions(-) delete mode 100644 zeppelin-server/notebook/.python.recovery.crc delete mode 100644 zeppelin-server/notebook/python.recovery create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java rename zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/{SparkInterpreterModeTest.java => SparkIntegrationTest.java} (84%) diff --git a/.travis.yml b/.travis.yml index bc568fce2e5..aba29443835 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,7 @@ matrix: - sudo: required jdk: "oraclejdk8" dist: trusty - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk8" @@ -78,32 +78,46 @@ matrix: # Test interpreter modules - jdk: "openjdk7" dist: trusty - env: PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="install -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" + env: PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" + + # Run ZeppelinSparkClusterTest & SparkIntegrationTest in one build would exceed the time limitation of travis, so running them separately + + # Test spark interpreter with different spark versions under python2, only run ZeppelinSparkClusterTest + - sudo: required + jdk: "oraclejdk8" + dist: trusty + env: PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest -DfailIfNoTests=false" + + # Test spark interpreter with different spark versions under python3, only run SparkIntegrationTest + - sudo: required + jdk: "oraclejdk8" + dist: trusty + env: PYTHON="3" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest -DfailIfNoTests=false" # Test spark module for 2.2.0 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 2.1.0 with scala 2.11 - jdk: "openjdk7" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 2.0.2 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 1.6.3 with scala 2.10 - jdk: "openjdk7" dist: trusty - env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 1.6.3 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test python/pyspark with python 2, livy 0.5 - sudo: required @@ -116,7 +130,7 @@ matrix: dist: trusty jdk: "openjdk7" env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="-Dpyspark.test.exclude='' -DfailIfNoTests=false" - + before_install: # check files included in commit range, clear bower_components if a bower.json file has changed. # bower cache clearing can also be forced by putting "bower clear" or "clear bower" in a commit message diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java index dbaeabe9a99..5efff0e7be4 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java @@ -59,12 +59,18 @@ public void open() throws InterpreterException { String sparkRLibPath; if (System.getenv("SPARK_HOME") != null) { + // local or yarn-client mode when SPARK_HOME is specified sparkRLibPath = System.getenv("SPARK_HOME") + "/R/lib"; - } else { + } else if (System.getenv("ZEPPELIN_HOME") != null){ + // embedded mode when SPARK_HOME is not specified sparkRLibPath = System.getenv("ZEPPELIN_HOME") + "/interpreter/spark/R/lib"; // workaround to make sparkr work without SPARK_HOME System.setProperty("spark.test.home", System.getenv("ZEPPELIN_HOME") + "/interpreter/spark"); + } else { + // yarn-cluster mode + sparkRLibPath = "sparkr"; } + synchronized (SparkRBackend.backend()) { if (!SparkRBackend.isStarted()) { SparkRBackend.init(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java index 0bb3d42dd19..92cf3a8febc 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java @@ -34,6 +34,7 @@ public InterpreterProperty(String name, Object value, String type) { public InterpreterProperty(String name, Object value) { this.name = name; this.value = value; + this.type = InterpreterPropertyType.TEXTAREA.getValue(); } public String getName() { diff --git a/zeppelin-server/notebook/.python.recovery.crc b/zeppelin-server/notebook/.python.recovery.crc deleted file mode 100644 index 6bd3e7ae43b861a0504394ec70405b321f2d9b0a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}E^);Is??6cz)X diff --git a/zeppelin-server/notebook/python.recovery b/zeppelin-server/notebook/python.recovery deleted file mode 100644 index eaf4938fdad..00000000000 --- a/zeppelin-server/notebook/python.recovery +++ /dev/null @@ -1 +0,0 @@ -2CZA1DVUG:shared_process 192.168.3.2:55410 \ No newline at end of file diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 970f302fec0..1622791417c 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -341,6 +341,15 @@ + + + org.apache.zeppelin + zeppelin-zengine + ${project.version} + tests + test + + diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 519342012ac..dfb5ac28ea2 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -63,72 +63,70 @@ public abstract class AbstractTestRestApi { static final String restApiUrl = "/api"; static final String url = getUrlToTest(); protected static final boolean wasRunning = checkIfServerIsRunning(); - static boolean pySpark = false; - static boolean sparkR = false; static boolean isRunningWithAuth = false; private static File shiroIni = null; private static String zeppelinShiro = "[users]\n" + - "admin = password1, admin\n" + - "user1 = password2, role1, role2\n" + - "user2 = password3, role3\n" + - "user3 = password4, role2\n" + - "[main]\n" + - "sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager\n" + - "securityManager.sessionManager = $sessionManager\n" + - "securityManager.sessionManager.globalSessionTimeout = 86400000\n" + - "shiro.loginUrl = /api/login\n" + - "[roles]\n" + - "role1 = *\n" + - "role2 = *\n" + - "role3 = *\n" + - "admin = *\n" + - "[urls]\n" + - "/api/version = anon\n" + - "/** = authc"; + "admin = password1, admin\n" + + "user1 = password2, role1, role2\n" + + "user2 = password3, role3\n" + + "user3 = password4, role2\n" + + "[main]\n" + + "sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager\n" + + "securityManager.sessionManager = $sessionManager\n" + + "securityManager.sessionManager.globalSessionTimeout = 86400000\n" + + "shiro.loginUrl = /api/login\n" + + "[roles]\n" + + "role1 = *\n" + + "role2 = *\n" + + "role3 = *\n" + + "admin = *\n" + + "[urls]\n" + + "/api/version = anon\n" + + "/** = authc"; private static String zeppelinShiroKnox = "[users]\n" + - "admin = password1, admin\n" + - "user1 = password2, role1, role2\n" + - "[main]\n" + - "knoxJwtRealm = org.apache.zeppelin.realm.jwt.KnoxJwtRealm\n" + - "knoxJwtRealm.providerUrl = https://domain.example.com/\n" + - "knoxJwtRealm.login = gateway/knoxsso/knoxauth/login.html\n" + - "knoxJwtRealm.logout = gateway/knoxssout/api/v1/webssout\n" + - "knoxJwtRealm.redirectParam = originalUrl\n" + - "knoxJwtRealm.cookieName = hadoop-jwt\n" + - "knoxJwtRealm.publicKeyPath = knox-sso.pem\n" + - "authc = org.apache.zeppelin.realm.jwt.KnoxAuthenticationFilter\n" + - "sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager\n" + - "securityManager.sessionManager = $sessionManager\n" + - "securityManager.sessionManager.globalSessionTimeout = 86400000\n" + - "shiro.loginUrl = /api/login\n" + - "[roles]\n" + - "admin = *\n" + - "[urls]\n" + - "/api/version = anon\n" + - "/** = authc"; + "admin = password1, admin\n" + + "user1 = password2, role1, role2\n" + + "[main]\n" + + "knoxJwtRealm = org.apache.zeppelin.realm.jwt.KnoxJwtRealm\n" + + "knoxJwtRealm.providerUrl = https://domain.example.com/\n" + + "knoxJwtRealm.login = gateway/knoxsso/knoxauth/login.html\n" + + "knoxJwtRealm.logout = gateway/knoxssout/api/v1/webssout\n" + + "knoxJwtRealm.redirectParam = originalUrl\n" + + "knoxJwtRealm.cookieName = hadoop-jwt\n" + + "knoxJwtRealm.publicKeyPath = knox-sso.pem\n" + + "authc = org.apache.zeppelin.realm.jwt.KnoxAuthenticationFilter\n" + + "sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager\n" + + "securityManager.sessionManager = $sessionManager\n" + + "securityManager.sessionManager.globalSessionTimeout = 86400000\n" + + "shiro.loginUrl = /api/login\n" + + "[roles]\n" + + "admin = *\n" + + "[urls]\n" + + "/api/version = anon\n" + + "/** = authc"; private static File knoxSsoPem = null; private static String KNOX_SSO_PEM = "-----BEGIN CERTIFICATE-----\n" - + "MIIChjCCAe+gAwIBAgIJALYrdDEXKwcqMA0GCSqGSIb3DQEBBQUAMIGEMQswCQYD\n" - + "VQQGEwJVUzENMAsGA1UECBMEVGVzdDENMAsGA1UEBxMEVGVzdDEPMA0GA1UEChMG\n" - + "SGFkb29wMQ0wCwYDVQQLEwRUZXN0MTcwNQYDVQQDEy5jdHItZTEzNS0xNTEyMDY5\n" - + "MDMyOTc1LTU0NDctMDEtMDAwMDAyLmh3eC5zaXRlMB4XDTE3MTIwNDA5NTIwMFoX\n" - + "DTE4MTIwNDA5NTIwMFowgYQxCzAJBgNVBAYTAlVTMQ0wCwYDVQQIEwRUZXN0MQ0w\n" - + "CwYDVQQHEwRUZXN0MQ8wDQYDVQQKEwZIYWRvb3AxDTALBgNVBAsTBFRlc3QxNzA1\n" - + "BgNVBAMTLmN0ci1lMTM1LTE1MTIwNjkwMzI5NzUtNTQ0Ny0wMS0wMDAwMDIuaHd4\n" - + "LnNpdGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAILFoXdz3yCy2INncYM2\n" - + "y72fYrONoQIxeeIzeJIibXLTuowSju90Q6aThSyUsQ6NEia2flnlKiCgINTNAodh\n" - + "UPUVGyGT+NMrqJzzpXAll2UUa6gIUPnXYEzYNkMIpbQOAo5BAg7YamaidbPPiT3W\n" - + "wAD1rWo3AMUY+nZJrAi4dEH5AgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAB0R07/lo\n" - + "4hD+WeDEeyLTnsbFnPNXxBT1APMUmmuCjcky/19ZB8OphqTKIITONdOK/XHdjZHG\n" - + "JDOfhBkVknL42lSi45ahUAPS2PZOlQL08MbS8xajP1faterm+aHcdwJVK9dK76RB\n" - + "/bA8TFNPblPxavIOcd+R+RfFmT1YKfYIhco=\n" - + "-----END CERTIFICATE-----"; + + "MIIChjCCAe+gAwIBAgIJALYrdDEXKwcqMA0GCSqGSIb3DQEBBQUAMIGEMQswCQYD\n" + + "VQQGEwJVUzENMAsGA1UECBMEVGVzdDENMAsGA1UEBxMEVGVzdDEPMA0GA1UEChMG\n" + + "SGFkb29wMQ0wCwYDVQQLEwRUZXN0MTcwNQYDVQQDEy5jdHItZTEzNS0xNTEyMDY5\n" + + "MDMyOTc1LTU0NDctMDEtMDAwMDAyLmh3eC5zaXRlMB4XDTE3MTIwNDA5NTIwMFoX\n" + + "DTE4MTIwNDA5NTIwMFowgYQxCzAJBgNVBAYTAlVTMQ0wCwYDVQQIEwRUZXN0MQ0w\n" + + "CwYDVQQHEwRUZXN0MQ8wDQYDVQQKEwZIYWRvb3AxDTALBgNVBAsTBFRlc3QxNzA1\n" + + "BgNVBAMTLmN0ci1lMTM1LTE1MTIwNjkwMzI5NzUtNTQ0Ny0wMS0wMDAwMDIuaHd4\n" + + "LnNpdGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAILFoXdz3yCy2INncYM2\n" + + "y72fYrONoQIxeeIzeJIibXLTuowSju90Q6aThSyUsQ6NEia2flnlKiCgINTNAodh\n" + + "UPUVGyGT+NMrqJzzpXAll2UUa6gIUPnXYEzYNkMIpbQOAo5BAg7YamaidbPPiT3W\n" + + "wAD1rWo3AMUY+nZJrAi4dEH5AgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAB0R07/lo\n" + + "4hD+WeDEeyLTnsbFnPNXxBT1APMUmmuCjcky/19ZB8OphqTKIITONdOK/XHdjZHG\n" + + "JDOfhBkVknL42lSi45ahUAPS2PZOlQL08MbS8xajP1faterm+aHcdwJVK9dK76RB\n" + + "/bA8TFNPblPxavIOcd+R+RfFmT1YKfYIhco=\n" + + "-----END CERTIFICATE-----"; protected static File zeppelinHome; protected static File confDir; @@ -159,7 +157,7 @@ protected static String getUrlToTest() { @Override public void run() { try { - ZeppelinServer.main(new String[] {""}); + ZeppelinServer.main(new String[]{""}); } catch (Exception e) { LOG.error("Exception in WebDriverManager while getWebDriver ", e); throw new RuntimeException(e); @@ -190,7 +188,7 @@ private static void start(boolean withAuth, String testClassName, boolean withKn isRunningWithAuth = true; // Set Anonymous session to false. System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED.getVarName(), "false"); - + // Create a shiro env test. shiroIni = new File(confDir, "shiro.ini"); if (!shiroIni.exists()) { @@ -246,66 +244,10 @@ private static void start(boolean withAuth, String testClassName, boolean withKn throw new RuntimeException("Can not start Zeppelin server"); } LOG.info("Test Zeppelin stared."); - - - // assume first one is spark - InterpreterSetting sparkIntpSetting = null; - for(InterpreterSetting intpSetting : - ZeppelinServer.notebook.getInterpreterSettingManager().get()) { - if (intpSetting.getName().equals("spark")) { - sparkIntpSetting = intpSetting; - } - } - - Map sparkProperties = - (Map) sparkIntpSetting.getProperties(); - // ci environment runs spark cluster for testing - // so configure zeppelin use spark cluster - if ("true".equals(System.getenv("CI"))) { - // set spark master and other properties - sparkProperties.put("master", - new InterpreterProperty("master", "local[2]", InterpreterPropertyType.TEXTAREA.getValue())); - sparkProperties.put("spark.master", - new InterpreterProperty("spark.master", "local[2]", InterpreterPropertyType.TEXTAREA.getValue())); - sparkProperties.put("spark.cores.max", - new InterpreterProperty("spark.cores.max", "2", InterpreterPropertyType.TEXTAREA.getValue())); - sparkProperties.put("zeppelin.spark.useHiveContext", - new InterpreterProperty("zeppelin.spark.useHiveContext", false, InterpreterPropertyType.CHECKBOX.getValue())); - sparkProperties.put("zeppelin.pyspark.useIPython", new InterpreterProperty("zeppelin.pyspark.useIPython", "false", InterpreterPropertyType.TEXTAREA.getValue())); - sparkProperties.put("zeppelin.spark.test", new InterpreterProperty("zeppelin.spark.test", "true", InterpreterPropertyType.TEXTAREA.getValue())); - sparkIntpSetting.setProperties(sparkProperties); - pySpark = true; - sparkR = true; - ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId()); - } else { - String sparkHome = getSparkHome(); - LOG.info("SPARK HOME detected " + sparkHome); - if (sparkHome != null) { - if (System.getenv("SPARK_MASTER") != null) { - sparkProperties.put("master", - new InterpreterProperty("master", System.getenv("SPARK_MASTER"), InterpreterPropertyType.TEXTAREA.getValue())); - } else { - sparkProperties.put("master", - new InterpreterProperty("master", "local[2]", InterpreterPropertyType.TEXTAREA.getValue())); - } - sparkProperties.put("spark.master", - new InterpreterProperty("spark.master", "local[2]", InterpreterPropertyType.TEXTAREA.getValue())); - sparkProperties.put("spark.cores.max", - new InterpreterProperty("spark.cores.max", "2", InterpreterPropertyType.TEXTAREA.getValue())); - sparkProperties.put("zeppelin.spark.useHiveContext", - new InterpreterProperty("zeppelin.spark.useHiveContext", false, InterpreterPropertyType.CHECKBOX.getValue())); - sparkProperties.put("zeppelin.pyspark.useIPython", new InterpreterProperty("zeppelin.pyspark.useIPython", "false", InterpreterPropertyType.TEXTAREA.getValue())); - sparkProperties.put("zeppelin.spark.test", new InterpreterProperty("zeppelin.spark.test", "true", InterpreterPropertyType.TEXTAREA.getValue())); - - pySpark = true; - sparkR = true; - } - - ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId()); - } } } + protected static void startUpWithKnoxEnable(String testClassName) throws Exception { start(true, testClassName, true); } @@ -327,53 +269,12 @@ private static String getHostname() { } } - private static String getSparkHome() { - String sparkHome = System.getenv("SPARK_HOME"); - if (sparkHome != null) { - return sparkHome; - } - sparkHome = getSparkHomeRecursively(new File(System.getProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName()))); - return sparkHome; - } - - boolean isPyspark() { - return pySpark; - } - - boolean isSparkR() { - return sparkR; - } - - private static String getSparkHomeRecursively(File dir) { - if (dir == null) return null; - File files [] = dir.listFiles(); - if (files == null) return null; - - File homeDetected = null; - for (File f : files) { - if (isActiveSparkHome(f)) { - homeDetected = f; - break; - } - } - - if (homeDetected != null) { - return homeDetected.getAbsolutePath(); - } else { - return getSparkHomeRecursively(dir.getParentFile()); - } - } - - private static boolean isActiveSparkHome(File dir) { - return dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+"); - } - protected static void shutDown() throws Exception { shutDown(true); } protected static void shutDown(final boolean deleteConfDir) throws Exception { - if (!wasRunning) { + if (!wasRunning && ZeppelinServer.notebook != null) { // restart interpreter to stop all interpreter processes List settingList = ZeppelinServer.notebook.getInterpreterSettingManager().get(); if (!ZeppelinServer.notebook.getConf().isRecoveryEnabled()) { diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index f3a70996414..bda555a60ca 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -16,588 +16,630 @@ */ package org.apache.zeppelin.rest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterProperty; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.SparkDownloadUtils; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.server.ZeppelinServer; import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; /** * Test against spark cluster. - * Spark cluster is started by CI server using testing/startSparkCluster.sh */ +@RunWith(value = Parameterized.class) public class ZeppelinSparkClusterTest extends AbstractTestRestApi { - AuthenticationInfo anonymous; - - @BeforeClass - public static void init() throws Exception { - AbstractTestRestApi.startUp(ZeppelinSparkClusterTest.class.getSimpleName()); - } - - @AfterClass - public static void destroy() throws Exception { - AbstractTestRestApi.shutDown(); - } - - @Before - public void setUp() { - anonymous = new AuthenticationInfo("anonymous"); + private static Logger LOGGER = LoggerFactory.getLogger(ZeppelinSparkClusterTest.class); + + private String sparkVersion; + private AuthenticationInfo anonymous = new AuthenticationInfo("anonymous"); + + public ZeppelinSparkClusterTest(String sparkVersion) throws Exception { + this.sparkVersion = sparkVersion; + LOGGER.info("Testing SparkVersion: " + sparkVersion); + String sparkHome = SparkDownloadUtils.downloadSpark(sparkVersion); + setupSparkInterpreter(sparkHome); + verifySparkVersionNumber(); + } + + @Parameterized.Parameters + public static List data() { + return Arrays.asList(new Object[][]{ + {"2.2.1"}, + {"2.1.2"}, + {"2.0.2"}, + {"1.6.3"} + }); + } + + public void setupSparkInterpreter(String sparkHome) throws InterpreterException { + InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterSettingManager() + .getInterpreterSettingByName("spark"); + + Map sparkProperties = + (Map) sparkIntpSetting.getProperties(); + LOG.info("SPARK HOME detected " + sparkHome); + if (System.getenv("SPARK_MASTER") != null) { + sparkProperties.put("master", + new InterpreterProperty("master", System.getenv("SPARK_MASTER"))); + } else { + sparkProperties.put("master", new InterpreterProperty("master", "local[2]")); } - - private void waitForFinish(Paragraph p) { - while (p.getStatus() != Status.FINISHED - && p.getStatus() != Status.ERROR - && p.getStatus() != Status.ABORT) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - LOG.error("Exception in WebDriverManager while getWebDriver ", e); - } - } - } - - @Test - public void scalaOutputTest() throws IOException { - // create new note - Note note = ZeppelinServer.notebook.createNote(anonymous); - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%spark import java.util.Date\n" + - "import java.net.URL\n" + - "println(\"hello\")\n" - ); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("import java.util.Date\n" + - "import java.net.URL\n" + - "hello\n", p.getResult().message().get(0).getData()); - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); - } - - - - @Test - public void basicRDDTransformationAndActionTest() throws IOException { - // create new note - Note note = ZeppelinServer.notebook.createNote(anonymous); - - // run markdown paragraph, again - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%spark print(sc.parallelize(1 to 10).reduce(_ + _))"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("55", p.getResult().message().get(0).getData()); - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + sparkProperties.put("SPARK_HOME", new InterpreterProperty("SPARK_HOME", sparkHome)); + sparkProperties.put("spark.master", new InterpreterProperty("spark.master", "local[2]")); + sparkProperties.put("spark.cores.max", + new InterpreterProperty("spark.cores.max", "2")); + sparkProperties.put("zeppelin.spark.useHiveContext", + new InterpreterProperty("zeppelin.spark.useHiveContext", "false")); + sparkProperties.put("zeppelin.pyspark.useIPython", new InterpreterProperty("zeppelin.pyspark.useIPython", "false")); + sparkProperties.put("zeppelin.spark.useNew", new InterpreterProperty("zeppelin.spark.useNew", "true")); + sparkProperties.put("zeppelin.spark.test", new InterpreterProperty("zeppelin.spark.test", "true")); + + ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId()); + } + + @BeforeClass + public static void setUp() throws Exception { + AbstractTestRestApi.startUp(ZeppelinSparkClusterTest.class.getSimpleName()); + } + + @AfterClass + public static void destroy() throws Exception { + AbstractTestRestApi.shutDown(); + } + + private void waitForFinish(Paragraph p) { + while (p.getStatus() != Status.FINISHED + && p.getStatus() != Status.ERROR + && p.getStatus() != Status.ABORT) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + LOG.error("Exception in WebDriverManager while getWebDriver ", e); + } } - - @Test - public void sparkSQLTest() throws IOException { - // create new note - Note note = ZeppelinServer.notebook.createNote(anonymous); - int sparkVersion = getSparkVersionNumber(note); - // DataFrame API is available from spark 1.3 - if (sparkVersion >= 13) { - // test basic dataframe api - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%spark val df=sqlContext.createDataFrame(Seq((\"hello\",20)))\n" + - "df.collect()"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertTrue(p.getResult().message().get(0).getData().contains( - "Array[org.apache.spark.sql.Row] = Array([hello,20])")); - - // test display DataFrame - p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%spark val df=sqlContext.createDataFrame(Seq((\"hello\",20)))\n" + - "z.show(df)"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(1).getType()); - assertEquals("_1\t_2\nhello\t20\n", p.getResult().message().get(1).getData()); - - // test display DataSet - if (sparkVersion >= 20) { - p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%spark val ds=spark.createDataset(Seq((\"hello\",20)))\n" + - "z.show(ds)"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(1).getType()); - assertEquals("_1\t_2\nhello\t20\n", p.getResult().message().get(1).getData()); - } - } - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + @Test + public void scalaOutputTest() throws IOException { + // create new note + Note note = ZeppelinServer.notebook.createNote(anonymous); + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%spark import java.util.Date\n" + + "import java.net.URL\n" + + "println(\"hello\")\n" + ); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals("import java.util.Date\n" + + "import java.net.URL\n" + + "hello\n", p.getResult().message().get(0).getData()); + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + + @Test + public void basicRDDTransformationAndActionTest() throws IOException { + // create new note + Note note = ZeppelinServer.notebook.createNote(anonymous); + + // run markdown paragraph, again + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%spark print(sc.parallelize(1 to 10).reduce(_ + _))"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals("55", p.getResult().message().get(0).getData()); + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + @Test + public void sparkSQLTest() throws IOException { + // create new note + Note note = ZeppelinServer.notebook.createNote(anonymous); + // test basic dataframe api + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%spark val df=sqlContext.createDataFrame(Seq((\"hello\",20)))\n" + + "df.collect()"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertTrue(p.getResult().message().get(0).getData().contains( + "Array[org.apache.spark.sql.Row] = Array([hello,20])")); + + // test display DataFrame + p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%spark val df=sqlContext.createDataFrame(Seq((\"hello\",20)))\n" + + "z.show(df)"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(1).getType()); + assertEquals("_1\t_2\nhello\t20\n", p.getResult().message().get(1).getData()); + + // test display DataSet + if (isSpark2()) { + p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%spark val ds=spark.createDataset(Seq((\"hello\",20)))\n" + + "z.show(ds)"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(1).getType()); + assertEquals("_1\t_2\nhello\t20\n", p.getResult().message().get(1).getData()); } - @Test - public void sparkRTest() throws IOException, InterpreterException { - // create new note - Note note = ZeppelinServer.notebook.createNote(anonymous); - int sparkVersion = getSparkVersionNumber(note); - - if (isSparkR() && sparkVersion >= 14) { // sparkr supported from 1.4.0 - // restart spark interpreter - List settings = - ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId()); - - for (InterpreterSetting setting : settings) { - if (setting.getName().equals("spark")) { - ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId()); - break; - } - } - - String sqlContextName = "sqlContext"; - if (sparkVersion >= 20) { - sqlContextName = "spark"; - } - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%r localDF <- data.frame(name=c(\"a\", \"b\", \"c\"), age=c(19, 23, 18))\n" + - "df <- createDataFrame(" + sqlContextName + ", localDF)\n" + - "count(df)" - ); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - System.err.println("sparkRTest=" + p.getResult().message().get(0).getData()); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("[1] 3", p.getResult().message().get(0).getData().trim()); + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + @Test + public void sparkRTest() throws IOException, InterpreterException { + // create new note + Note note = ZeppelinServer.notebook.createNote(anonymous); + // restart spark interpreter + List settings = + ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId()); + + for (InterpreterSetting setting : settings) { + if (setting.getName().equals("spark")) { + ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId()); + break; } - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); } - @Test - public void pySparkTest() throws IOException { - // create new note - Note note = ZeppelinServer.notebook.createNote(anonymous); - note.setName("note"); - int sparkVersion = getSparkVersionNumber(note); - - if (isPyspark() && sparkVersion >= 12) { // pyspark supported from 1.2.1 - // run markdown paragraph, again - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%pyspark print(sc.parallelize(range(1, 11)).reduce(lambda a, b: a + b))"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("55\n", p.getResult().message().get(0).getData()); - if (sparkVersion >= 13) { - // run sqlContext test - p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%pyspark from pyspark.sql import Row\n" + - "df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" + - "df.collect()"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("[Row(age=20, id=1)]\n", p.getResult().message().get(0).getData()); - - // test display Dataframe - p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%pyspark from pyspark.sql import Row\n" + - "df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" + - "z.show(df)"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(0).getType()); - // TODO (zjffdu), one more \n is appended, need to investigate why. - assertEquals("age\tid\n20\t1\n", p.getResult().message().get(0).getData()); - - // test udf - p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%pyspark sqlContext.udf.register(\"f1\", lambda x: len(x))\n" + - "sqlContext.sql(\"select f1(\\\"abc\\\") as len\").collect()"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertTrue("[Row(len=u'3')]\n".equals(p.getResult().message().get(0).getData()) || - "[Row(len='3')]\n".equals(p.getResult().message().get(0).getData())); - - // test exception - p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - /** - %pyspark - a=1 - - print(a2) - */ - p.setText("%pyspark a=1\n\nprint(a2)"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.ERROR, p.getStatus()); - assertTrue(p.getResult().message().get(0).getData() - .contains("Fail to execute line 3: print(a2)")); - assertTrue(p.getResult().message().get(0).getData() - .contains("name 'a2' is not defined")); - } - if (sparkVersion >= 20) { - // run SparkSession test - p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%pyspark from pyspark.sql import Row\n" + - "df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" + - "df.collect()"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("[Row(age=20, id=1)]\n", p.getResult().message().get(0).getData()); - - // test udf - p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - // use SQLContext to register UDF but use this UDF through SparkSession - p.setText("%pyspark sqlContext.udf.register(\"f1\", lambda x: len(x))\n" + - "spark.sql(\"select f1(\\\"abc\\\") as len\").collect()"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertTrue("[Row(len=u'3')]\n".equals(p.getResult().message().get(0).getData()) || - "[Row(len='3')]\n".equals(p.getResult().message().get(0).getData())); - } - } - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + String sqlContextName = "sqlContext"; + if (isSpark2()) { + sqlContextName = "spark"; } - - @Test - public void pySparkAutoConvertOptionTest() throws IOException { - // create new note - Note note = ZeppelinServer.notebook.createNote(anonymous); - note.setName("note"); - - int sparkVersionNumber = getSparkVersionNumber(note); - - if (isPyspark() && sparkVersionNumber >= 14) { // auto_convert enabled from spark 1.4 - // run markdown paragraph, again - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - - String sqlContextName = "sqlContext"; - if (sparkVersionNumber >= 20) { - sqlContextName = "spark"; - } - - p.setText("%pyspark\nfrom pyspark.sql.functions import *\n" - + "print(" + sqlContextName + ".range(0, 10).withColumn('uniform', rand(seed=10) * 3.14).count())"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("10\n", p.getResult().message().get(0).getData()); - } - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%r localDF <- data.frame(name=c(\"a\", \"b\", \"c\"), age=c(19, 23, 18))\n" + + "df <- createDataFrame(" + sqlContextName + ", localDF)\n" + + "count(df)" + ); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + System.err.println("sparkRTest=" + p.getResult().message().get(0).getData()); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals("[1] 3", p.getResult().message().get(0).getData().trim()); + + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + @Test + public void pySparkTest() throws IOException { + // create new note + Note note = ZeppelinServer.notebook.createNote(anonymous); + note.setName("note"); + + // run markdown paragraph, again + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%pyspark print(sc.parallelize(range(1, 11)).reduce(lambda a, b: a + b))"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals("55\n", p.getResult().message().get(0).getData()); + if (!isSpark2()) { + // run sqlContext test + p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%pyspark from pyspark.sql import Row\n" + + "df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" + + "df.collect()"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals("[Row(age=20, id=1)]\n", p.getResult().message().get(0).getData()); + + // test display Dataframe + p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%pyspark from pyspark.sql import Row\n" + + "df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" + + "z.show(df)"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(0).getType()); + // TODO (zjffdu), one more \n is appended, need to investigate why. + assertEquals("age\tid\n20\t1\n", p.getResult().message().get(0).getData()); + + // test udf + p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%pyspark sqlContext.udf.register(\"f1\", lambda x: len(x))\n" + + "sqlContext.sql(\"select f1(\\\"abc\\\") as len\").collect()"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertTrue("[Row(len=u'3')]\n".equals(p.getResult().message().get(0).getData()) || + "[Row(len='3')]\n".equals(p.getResult().message().get(0).getData())); + + // test exception + p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + /** + %pyspark + a=1 + + print(a2) + */ + p.setText("%pyspark a=1\n\nprint(a2)"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.ERROR, p.getStatus()); + assertTrue(p.getResult().message().get(0).getData() + .contains("Fail to execute line 3: print(a2)")); + assertTrue(p.getResult().message().get(0).getData() + .contains("name 'a2' is not defined")); + } else { + // run SparkSession test + p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%pyspark from pyspark.sql import Row\n" + + "df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" + + "df.collect()"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals("[Row(age=20, id=1)]\n", p.getResult().message().get(0).getData()); + + // test udf + p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + // use SQLContext to register UDF but use this UDF through SparkSession + p.setText("%pyspark sqlContext.udf.register(\"f1\", lambda x: len(x))\n" + + "spark.sql(\"select f1(\\\"abc\\\") as len\").collect()"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertTrue("[Row(len=u'3')]\n".equals(p.getResult().message().get(0).getData()) || + "[Row(len='3')]\n".equals(p.getResult().message().get(0).getData())); } - @Test - public void zRunTest() throws IOException { - // create new note - Note note = ZeppelinServer.notebook.createNote(anonymous); - Paragraph p0 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config0 = p0.getConfig(); - config0.put("enabled", true); - p0.setConfig(config0); - p0.setText("%spark z.run(1)"); - p0.setAuthenticationInfo(anonymous); - Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config1 = p1.getConfig(); - config1.put("enabled", true); - p1.setConfig(config1); - p1.setText("%spark val a=10"); - p1.setAuthenticationInfo(anonymous); - Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config2 = p2.getConfig(); - config2.put("enabled", true); - p2.setConfig(config2); - p2.setText("%spark print(a)"); - p2.setAuthenticationInfo(anonymous); - - note.run(p0.getId()); - waitForFinish(p0); - assertEquals(Status.FINISHED, p0.getStatus()); - - // z.run is not blocking call. So p1 may not be finished when p0 is done. - waitForFinish(p1); - note.run(p2.getId()); - waitForFinish(p2); - assertEquals(Status.FINISHED, p2.getStatus()); - assertEquals("10", p2.getResult().message().get(0).getData()); - - Paragraph p3 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config3 = p3.getConfig(); - config3.put("enabled", true); - p3.setConfig(config3); - p3.setText("%spark println(new java.util.Date())"); - p3.setAuthenticationInfo(anonymous); - - p0.setText(String.format("%%spark z.runNote(\"%s\")", note.getId())); - note.run(p0.getId()); - waitForFinish(p0); - waitForFinish(p1); - waitForFinish(p2); - waitForFinish(p3); - - assertEquals(Status.FINISHED, p3.getStatus()); - String p3result = p3.getResult().message().get(0).getData(); - assertNotEquals(null, p3result); - assertNotEquals("", p3result); - - p0.setText(String.format("%%spark z.run(\"%s\", \"%s\")", note.getId(), p3.getId())); - p3.setText("%%spark println(\"END\")"); - - note.run(p0.getId()); - waitForFinish(p0); - waitForFinish(p3); - - assertNotEquals(p3result, p3.getResult().message()); - - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); - } + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } - @Test - public void pySparkDepLoaderTest() throws IOException, InterpreterException { - // create new note - Note note = ZeppelinServer.notebook.createNote(anonymous); - int sparkVersionNumber = getSparkVersionNumber(note); - - if (isPyspark() && sparkVersionNumber >= 14) { - // restart spark interpreter - List settings = - ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId()); - - for (InterpreterSetting setting : settings) { - if (setting.getName().equals("spark")) { - ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId()); - break; - } - } - - // load dep - Paragraph p0 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config = p0.getConfig(); - config.put("enabled", true); - p0.setConfig(config); - p0.setText("%dep z.load(\"com.databricks:spark-csv_2.11:1.2.0\")"); - p0.setAuthenticationInfo(anonymous); - note.run(p0.getId()); - waitForFinish(p0); - assertEquals(Status.FINISHED, p0.getStatus()); - - // write test csv file - File tmpFile = File.createTempFile("test", "csv"); - FileUtils.write(tmpFile, "a,b\n1,2"); - - // load data using libraries from dep loader - Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - p1.setConfig(config); - - String sqlContextName = "sqlContext"; - if (sparkVersionNumber >= 20) { - sqlContextName = "spark"; - } - p1.setText("%pyspark\n" + - "from pyspark.sql import SQLContext\n" + - "print(" + sqlContextName + ".read.format('com.databricks.spark.csv')" + - ".load('" + tmpFile.getAbsolutePath() +"').count())"); - p1.setAuthenticationInfo(anonymous); - note.run(p1.getId()); - - waitForFinish(p1); - assertEquals(Status.FINISHED, p1.getStatus()); - assertEquals("2\n", p1.getResult().message().get(0).getData()); - } - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); - } + @Test + public void pySparkAutoConvertOptionTest() throws IOException { + // create new note + Note note = ZeppelinServer.notebook.createNote(anonymous); + note.setName("note"); - /** - * Get spark version number as a numerical value. - * eg. 1.1.x => 11, 1.2.x => 12, 1.3.x => 13 ... - */ - private int getSparkVersionNumber(Note note) { - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - note.setName("note"); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%spark print(sc.version)"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - String sparkVersion = p.getResult().message().get(0).getData(); - System.out.println("Spark version detected " + sparkVersion); - String[] split = sparkVersion.split("\\."); - int version = Integer.parseInt(split[0]) * 10 + Integer.parseInt(split[1]); - return version; - } + // run markdown paragraph, again + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); - @Test - public void testSparkZeppelinContextDynamicForms() throws IOException { - Note note = ZeppelinServer.notebook.createNote(anonymous); - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - note.setName("note"); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - String code = "%spark.spark println(z.textbox(\"my_input\", \"default_name\"))\n" + - "println(z.select(\"my_select\", \"1\"," + - "Seq((\"1\", \"select_1\"), (\"2\", \"select_2\"))))\n" + - "val items=z.checkbox(\"my_checkbox\", Seq(\"2\"), " + - "Seq((\"1\", \"check_1\"), (\"2\", \"check_2\")))\n" + - "println(items(0))"; - p.setText(code); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - - assertEquals(Status.FINISHED, p.getStatus()); - Iterator formIter = p.settings.getForms().keySet().iterator(); - assert(formIter.next().equals("my_input")); - assert(formIter.next().equals("my_select")); - assert(formIter.next().equals("my_checkbox")); - - // check dynamic forms values - String[] result = p.getResult().message().get(0).getData().split("\n"); - assertEquals(4, result.length); - assertEquals("default_name", result[0]); - assertEquals("1", result[1]); - assertEquals("items: Seq[Object] = Buffer(2)", result[2]); - assertEquals("2", result[3]); - - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + String sqlContextName = "sqlContext"; + if (isSpark2()) { + sqlContextName = "spark"; } - @Test - public void testPySparkZeppelinContextDynamicForms() throws IOException { - Note note = ZeppelinServer.notebook.createNote(anonymous); - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - note.setName("note"); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - String code = "%spark.pyspark print(z.input('my_input', 'default_name'))\n" + - "print(z.select('my_select', " + - "[('1', 'select_1'), ('2', 'select_2')], defaultValue='1'))\n" + - "items=z.checkbox('my_checkbox', " + - "[('1', 'check_1'), ('2', 'check_2')], defaultChecked=['2'])\n" + - "print(items[0])"; - p.setText(code); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - - assertEquals(Status.FINISHED, p.getStatus()); - Iterator formIter = p.settings.getForms().keySet().iterator(); - assert(formIter.next().equals("my_input")); - assert(formIter.next().equals("my_select")); - assert(formIter.next().equals("my_checkbox")); - - // check dynamic forms values - String[] result = p.getResult().message().get(0).getData().split("\n"); - assertEquals(3, result.length); - assertEquals("default_name", result[0]); - assertEquals("1", result[1]); - assertEquals("2", result[2]); - - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + p.setText("%pyspark\nfrom pyspark.sql.functions import *\n" + + "print(" + sqlContextName + ".range(0, 10).withColumn('uniform', rand(seed=10) * 3.14).count())"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals("10\n", p.getResult().message().get(0).getData()); + + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + @Test + public void zRunTest() throws IOException { + // create new note + Note note = ZeppelinServer.notebook.createNote(anonymous); + Paragraph p0 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config0 = p0.getConfig(); + config0.put("enabled", true); + p0.setConfig(config0); + p0.setText("%spark z.run(1)"); + p0.setAuthenticationInfo(anonymous); + Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config1 = p1.getConfig(); + config1.put("enabled", true); + p1.setConfig(config1); + p1.setText("%spark val a=10"); + p1.setAuthenticationInfo(anonymous); + Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config2 = p2.getConfig(); + config2.put("enabled", true); + p2.setConfig(config2); + p2.setText("%spark print(a)"); + p2.setAuthenticationInfo(anonymous); + + note.run(p0.getId()); + waitForFinish(p0); + assertEquals(Status.FINISHED, p0.getStatus()); + + // z.run is not blocking call. So p1 may not be finished when p0 is done. + waitForFinish(p1); + note.run(p2.getId()); + waitForFinish(p2); + assertEquals(Status.FINISHED, p2.getStatus()); + assertEquals("10", p2.getResult().message().get(0).getData()); + + Paragraph p3 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config3 = p3.getConfig(); + config3.put("enabled", true); + p3.setConfig(config3); + p3.setText("%spark println(new java.util.Date())"); + p3.setAuthenticationInfo(anonymous); + + p0.setText(String.format("%%spark z.runNote(\"%s\")", note.getId())); + note.run(p0.getId()); + waitForFinish(p0); + waitForFinish(p1); + waitForFinish(p2); + waitForFinish(p3); + + assertEquals(Status.FINISHED, p3.getStatus()); + String p3result = p3.getResult().message().get(0).getData(); + assertNotEquals(null, p3result); + assertNotEquals("", p3result); + + p0.setText(String.format("%%spark z.run(\"%s\", \"%s\")", note.getId(), p3.getId())); + p3.setText("%%spark println(\"END\")"); + + note.run(p0.getId()); + waitForFinish(p0); + waitForFinish(p3); + + assertNotEquals(p3result, p3.getResult().message()); + + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + @Test + public void pySparkDepLoaderTest() throws IOException, InterpreterException { + // create new note + Note note = ZeppelinServer.notebook.createNote(anonymous); + + // restart spark interpreter + List settings = + ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId()); + + for (InterpreterSetting setting : settings) { + if (setting.getName().equals("spark")) { + ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId()); + break; + } } - @Test - public void testConfInterpreter() throws IOException { - ZeppelinServer.notebook.getInterpreterSettingManager().close(); - Note note = ZeppelinServer.notebook.createNote(AuthenticationInfo.ANONYMOUS); - Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - Map config = p.getConfig(); - config.put("enabled", true); - p.setConfig(config); - p.setText("%spark.conf spark.jars.packages\tcom.databricks:spark-csv_2.11:1.2.0"); - p.setAuthenticationInfo(anonymous); - note.run(p.getId()); - waitForFinish(p); - assertEquals(Status.FINISHED, p.getStatus()); - - Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); - p1.setConfig(config); - p1.setText("%spark\nimport com.databricks.spark.csv._"); - p1.setAuthenticationInfo(anonymous); - note.run(p1.getId()); - - waitForFinish(p1); - assertEquals(Status.FINISHED, p1.getStatus()); - - ZeppelinServer.notebook.removeNote(note.getId(), anonymous); - + // load dep + Paragraph p0 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p0.getConfig(); + config.put("enabled", true); + p0.setConfig(config); + p0.setText("%dep z.load(\"com.databricks:spark-csv_2.11:1.2.0\")"); + p0.setAuthenticationInfo(anonymous); + note.run(p0.getId()); + waitForFinish(p0); + assertEquals(Status.FINISHED, p0.getStatus()); + + // write test csv file + File tmpFile = File.createTempFile("test", "csv"); + FileUtils.write(tmpFile, "a,b\n1,2"); + + // load data using libraries from dep loader + Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + p1.setConfig(config); + + String sqlContextName = "sqlContext"; + if (isSpark2()) { + sqlContextName = "spark"; } + p1.setText("%pyspark\n" + + "from pyspark.sql import SQLContext\n" + + "print(" + sqlContextName + ".read.format('com.databricks.spark.csv')" + + ".load('" + tmpFile.getAbsolutePath() + "').count())"); + p1.setAuthenticationInfo(anonymous); + note.run(p1.getId()); + + waitForFinish(p1); + assertEquals(Status.FINISHED, p1.getStatus()); + assertEquals("2\n", p1.getResult().message().get(0).getData()); + + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + private void verifySparkVersionNumber() throws IOException { + Note note = ZeppelinServer.notebook.createNote(anonymous); + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + note.setName("note"); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%spark print(sc.version)"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals(sparkVersion, p.getResult().message().get(0).getData()); + } + + private int toIntSparkVersion(String sparkVersion) { + String[] split = sparkVersion.split("\\."); + int version = Integer.parseInt(split[0]) * 10 + Integer.parseInt(split[1]); + return version; + } + + private boolean isSpark2() { + return toIntSparkVersion(sparkVersion) >= 20; + } + + @Test + public void testSparkZeppelinContextDynamicForms() throws IOException { + Note note = ZeppelinServer.notebook.createNote(anonymous); + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + note.setName("note"); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + String code = "%spark.spark println(z.textbox(\"my_input\", \"default_name\"))\n" + + "println(z.select(\"my_select\", \"1\"," + + "Seq((\"1\", \"select_1\"), (\"2\", \"select_2\"))))\n" + + "val items=z.checkbox(\"my_checkbox\", Seq(\"2\"), " + + "Seq((\"1\", \"check_1\"), (\"2\", \"check_2\")))\n" + + "println(items(0))"; + p.setText(code); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + + assertEquals(Status.FINISHED, p.getStatus()); + Iterator formIter = p.settings.getForms().keySet().iterator(); + assert (formIter.next().equals("my_input")); + assert (formIter.next().equals("my_select")); + assert (formIter.next().equals("my_checkbox")); + + // check dynamic forms values + String[] result = p.getResult().message().get(0).getData().split("\n"); + assertEquals(4, result.length); + assertEquals("default_name", result[0]); + assertEquals("1", result[1]); + assertEquals("items: Seq[Object] = Buffer(2)", result[2]); + assertEquals("2", result[3]); + + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + @Test + public void testPySparkZeppelinContextDynamicForms() throws IOException { + Note note = ZeppelinServer.notebook.createNote(anonymous); + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + note.setName("note"); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + String code = "%spark.pyspark print(z.input('my_input', 'default_name'))\n" + + "print(z.select('my_select', " + + "[('1', 'select_1'), ('2', 'select_2')], defaultValue='1'))\n" + + "items=z.checkbox('my_checkbox', " + + "[('1', 'check_1'), ('2', 'check_2')], defaultChecked=['2'])\n" + + "print(items[0])"; + p.setText(code); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + + assertEquals(Status.FINISHED, p.getStatus()); + Iterator formIter = p.settings.getForms().keySet().iterator(); + assert (formIter.next().equals("my_input")); + assert (formIter.next().equals("my_select")); + assert (formIter.next().equals("my_checkbox")); + + // check dynamic forms values + String[] result = p.getResult().message().get(0).getData().split("\n"); + assertEquals(3, result.length); + assertEquals("default_name", result[0]); + assertEquals("1", result[1]); + assertEquals("2", result[2]); + + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + @Test + public void testConfInterpreter() throws IOException { + ZeppelinServer.notebook.getInterpreterSettingManager().close(); + Note note = ZeppelinServer.notebook.createNote(AuthenticationInfo.ANONYMOUS); + Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%spark.conf spark.jars.packages\tcom.databricks:spark-csv_2.11:1.2.0"); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + + Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); + p1.setConfig(config); + p1.setText("%spark\nimport com.databricks.spark.csv._"); + p1.setAuthenticationInfo(anonymous); + note.run(p1.getId()); + + waitForFinish(p1); + assertEquals(Status.FINISHED, p1.getStatus()); + + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + + } } diff --git a/zeppelin-server/src/test/resources/log4j.properties b/zeppelin-server/src/test/resources/log4j.properties index 83689930c1a..aff3b79a078 100644 --- a/zeppelin-server/src/test/resources/log4j.properties +++ b/zeppelin-server/src/test/resources/log4j.properties @@ -42,5 +42,4 @@ log4j.logger.DataNucleus.Datastore=ERROR # Log all JDBC parameters log4j.logger.org.hibernate.type=ALL -log4j.logger.org.apache.zeppelin.interpreter=DEBUG -log4j.logger.org.apache.zeppelin.spark=DEBUG +log4j.logger.org.apache.hadoop=WARN diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 81ce7162b1a..cdb11ddfd75 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -391,6 +391,20 @@ + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + + test-jar + + + + diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java new file mode 100644 index 00000000000..e027bb0d29f --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java @@ -0,0 +1,91 @@ +package org.apache.zeppelin.interpreter; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; + +public class SparkDownloadUtils { + private static Logger LOGGER = LoggerFactory.getLogger(SparkDownloadUtils.class); + + private static String downloadFolder = System.getProperty("user.home") + "/.cache/spark"; + + static { + try { + FileUtils.forceMkdir(new File(downloadFolder)); + } catch (IOException e) { + throw new RuntimeException("Fail to create downloadFolder: " + downloadFolder, e); + } + } + + + public static String downloadSpark(String version) { + File targetSparkHomeFolder = new File(downloadFolder + "/spark-" + version + "-bin-hadoop2.6"); + if (targetSparkHomeFolder.exists()) { + LOGGER.info("Skip to download spark as it is already downloaded."); + return targetSparkHomeFolder.getAbsolutePath(); + } + // Try mirrors a few times until one succeeds + for (int i = 0; i < 3; i++) { + try { + String preferredMirror = IOUtils.toString(new URL("https://www.apache.org/dyn/closer.lua?preferred=true")); + File downloadFile = new File(downloadFolder + "/spark-" + version + "-bin-hadoop2.6.tgz"); + String downloadURL = preferredMirror + "/spark/spark-" + version + "/spark-" + version + "-bin-hadoop2.6.tgz";; + runShellCommand(new String[] {"wget", downloadURL, "-P", downloadFolder}); + runShellCommand(new String[]{"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", downloadFolder}); + break; + } catch (Exception e) { + LOGGER.warn("Failed to download Spark", e); + } + } + return targetSparkHomeFolder.getAbsolutePath(); + } + + private static void runShellCommand(String[] commands) throws IOException, InterruptedException { + LOGGER.info("Starting shell commands: " + StringUtils.join(commands, " ")); + Process process = Runtime.getRuntime().exec(commands); + StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream()); + StreamGobbler outputGobbler = new StreamGobbler(process.getInputStream()); + errorGobbler.start(); + outputGobbler.start(); + if (process.waitFor() != 0) { + throw new IOException("Fail to run shell commands: " + StringUtils.join(commands, " ")); + } + LOGGER.info("Complete shell commands: " + StringUtils.join(commands, " ")); + } + + private static class StreamGobbler extends Thread { + InputStream is; + + // reads everything from is until empty. + StreamGobbler(InputStream is) { + this.is = is; + } + + public void run() { + try { + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); + String line = null; + long startTime = System.currentTimeMillis(); + while ( (line = br.readLine()) != null) { + // logging per 5 seconds + if ((System.currentTimeMillis() - startTime) > 5000) { + LOGGER.info(line); + startTime = System.currentTimeMillis(); + } + } + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest.java similarity index 84% rename from zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java rename to zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest.java index 22bb17e440f..50930a7fa79 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest.java @@ -8,21 +8,47 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.EnumSet; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class SparkInterpreterModeTest { +@RunWith(value = Parameterized.class) +public class SparkIntegrationTest { + private static Logger LOGGER = LoggerFactory.getLogger(SparkIntegrationTest.class); private static MiniHadoopCluster hadoopCluster; private static MiniZeppelin zeppelin; private static InterpreterFactory interpreterFactory; private static InterpreterSettingManager interpreterSettingManager; + private String sparkVersion; + private String sparkHome; + + public SparkIntegrationTest(String sparkVersion) { + LOGGER.info("Testing SparkVersion: " + sparkVersion); + this.sparkVersion = sparkVersion; + this.sparkHome = SparkDownloadUtils.downloadSpark(sparkVersion); + } + + @Parameterized.Parameters + public static List data() { + return Arrays.asList(new Object[][]{ + {"2.2.1"}, + {"2.1.2"}, + {"2.0.2"}, + {"1.6.3"} + }); + } + @BeforeClass public static void setUp() throws IOException { hadoopCluster = new MiniHadoopCluster(); @@ -52,7 +78,8 @@ private void testInterpreterBasics() throws IOException, InterpreterException { InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").getContext(); InterpreterResult interpreterResult = sparkInterpreter.interpret("sc.version", context); assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code); - String sparkVersion = interpreterResult.message().get(0).getData(); + String detectedSparkVersion = interpreterResult.message().get(0).getData(); + assertTrue(detectedSparkVersion +" doesn't contain " + this.sparkVersion, detectedSparkVersion.contains(this.sparkVersion)); interpreterResult = sparkInterpreter.interpret("sc.range(1,10).sum()", context); assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code); assertTrue(interpreterResult.msg.get(0).getData().contains("45")); @@ -69,14 +96,14 @@ private void testInterpreterBasics() throws IOException, InterpreterException { // test SparkSQLInterpreter Interpreter sqlInterpreter = interpreterFactory.getInterpreter("user1", "note1", "spark.sql"); - interpreterResult = sqlInterpreter.interpret("select count(1) from test", context); + interpreterResult = sqlInterpreter.interpret("select count(1) as c from test", context); assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code); assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType()); - assertEquals("count(1)\n2\n", interpreterResult.message().get(0).getData()); + assertEquals("c\n2\n", interpreterResult.message().get(0).getData()); // test SparkRInterpreter Interpreter sparkrInterpreter = interpreterFactory.getInterpreter("user1", "note1", "spark.r"); - if (isSpark2(sparkVersion)) { + if (isSpark2()) { interpreterResult = sparkrInterpreter.interpret("df <- as.DataFrame(faithful)\nhead(df)", context); } else { interpreterResult = sparkrInterpreter.interpret("df <- createDataFrame(sqlContext, faithful)\nhead(df)", context); @@ -90,7 +117,7 @@ private void testInterpreterBasics() throws IOException, InterpreterException { public void testLocalMode() throws IOException, YarnException, InterpreterException { InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark"); sparkInterpreterSetting.setProperty("master", "local[*]"); - sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME")); + sparkInterpreterSetting.setProperty("SPARK_HOME", sparkHome); sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath()); sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false"); sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false"); @@ -110,7 +137,7 @@ public void testYarnClientMode() throws IOException, YarnException, InterruptedE InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark"); sparkInterpreterSetting.setProperty("master", "yarn-client"); sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath()); - sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME")); + sparkInterpreterSetting.setProperty("SPARK_HOME", sparkHome); sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath()); sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false"); sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false"); @@ -132,7 +159,7 @@ public void testYarnClusterMode() throws IOException, YarnException, Interrupted InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark"); sparkInterpreterSetting.setProperty("master", "yarn-cluster"); sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath()); - sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME")); + sparkInterpreterSetting.setProperty("SPARK_HOME", sparkHome); sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath()); sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false"); sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false"); @@ -149,8 +176,8 @@ public void testYarnClusterMode() throws IOException, YarnException, Interrupted interpreterSettingManager.close(); } - private boolean isSpark2(String sparkVersion) { - return sparkVersion.startsWith("2."); + private boolean isSpark2() { + return this.sparkVersion.startsWith("2."); } private String getPythonExec() throws IOException, InterruptedException { diff --git a/zeppelin-zengine/src/test/resources/log4j.properties b/zeppelin-zengine/src/test/resources/log4j.properties index 74f619b70ee..ecfd05e0a2a 100644 --- a/zeppelin-zengine/src/test/resources/log4j.properties +++ b/zeppelin-zengine/src/test/resources/log4j.properties @@ -28,8 +28,6 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n # Root logger option log4j.rootLogger=INFO, stdout -log4j.logger.org.apache.zeppelin.notebook.repo=DEBUG - #mute some noisy guys log4j.logger.org.apache.hadoop.mapred=WARN log4j.logger.org.apache.hadoop.hive.ql=WARN @@ -44,6 +42,6 @@ log4j.logger.DataNucleus.Datastore=ERROR # Log all JDBC parameters log4j.logger.org.hibernate.type=ALL -log4j.logger.org.apache.zeppelin.interpreter=DEBUG -log4j.logger.org.apache.zeppelin.scheduler=DEBUG +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.zeppelin.interpreter=DEBUG From e148c8cfff04a7a299b2d626759a662a3f91087b Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 5 Mar 2018 12:58:19 +0800 Subject: [PATCH 062/386] ZEPPELIN-3291. Throw exception instead of return null for InterpreterFactory.getInterpreter ### What is this PR for? Minor code refactoring to throw exception when no interpreter is found ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3291 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2836 from zjffdu/ZEPPELIN-3291 and squashes the following commits: bee319c [Jeff Zhang] ZEPPELIN-3291. Throw exception instead of return null for InterpreterFactory.getInterpreter --- .../zeppelin/helium/ApplicationException.java | 4 ++ .../zeppelin/socket/NotebookServer.java | 7 ++- .../org/apache/zeppelin/helium/Helium.java | 7 ++- .../helium/HeliumApplicationFactory.java | 24 +++++--- .../interpreter/InterpreterFactory.java | 20 ++++--- .../InterpreterNotFoundException.java | 22 ++++++++ .../apache/zeppelin/notebook/Paragraph.java | 56 +++++++++++-------- .../helium/HeliumApplicationFactoryTest.java | 11 +++- .../interpreter/InterpreterFactoryTest.java | 12 ++-- .../InterpreterSettingManagerTest.java | 8 +-- .../apache/zeppelin/notebook/NoteTest.java | 11 ++-- .../zeppelin/notebook/NotebookTest.java | 12 +++- 12 files changed, 133 insertions(+), 61 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java index 4bf0ac28b94..d3c648886e8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java @@ -31,4 +31,8 @@ public ApplicationException(Exception e) { public ApplicationException() { } + + public ApplicationException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 113dfd60edd..d14a852fbb5 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -2442,7 +2442,12 @@ private void getEditorSetting(NotebookSocket conn, Message fromMessage) throws I Message resp = new Message(OP.EDITOR_SETTING); resp.put("paragraphId", paragraphId); Interpreter interpreter = - notebook().getInterpreterFactory().getInterpreter(user, noteId, replName); + null; + try { + interpreter = notebook().getInterpreterFactory().getInterpreter(user, noteId, replName); + } catch (InterpreterNotFoundException e) { + throw new IOException("Fail to get interpreter: " + replName, e); + } resp.put("editor", notebook().getInterpreterSettingManager(). getEditorSetting(interpreter, user, noteId, replName)); conn.send(serializeMessage(resp)); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java index 0af15abda02..399aea879a7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java @@ -20,6 +20,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterNotFoundException; import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.ManagedInterpreterGroup; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; @@ -340,8 +341,10 @@ private Map getPackagePersistedConfig(String artifact) { public HeliumPackageSuggestion suggestApp(Paragraph paragraph) { HeliumPackageSuggestion suggestion = new HeliumPackageSuggestion(); - Interpreter intp = paragraph.getBindedInterpreter(); - if (intp == null) { + Interpreter intp = null; + try { + intp = paragraph.getBindedInterpreter(); + } catch (InterpreterNotFoundException e) { return suggestion; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java index bdb36141329..50928bb7ab1 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -200,9 +200,11 @@ private void unload(final ApplicationState appsToUnload) throws ApplicationExcep "Can't unload application status " + appsToUnload.getStatus()); } appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADING); - Interpreter intp = paragraph.getBindedInterpreter(); - if (intp == null) { - throw new ApplicationException("No interpreter found"); + Interpreter intp = null; + try { + intp = paragraph.getBindedInterpreter(); + } catch (InterpreterException e) { + throw new ApplicationException("No interpreter found", e); } RemoteInterpreterProcess intpProcess = @@ -280,9 +282,11 @@ private void run(final ApplicationState app) throws ApplicationException { "Can't run application status " + app.getStatus()); } - Interpreter intp = paragraph.getBindedInterpreter(); - if (intp == null) { - throw new ApplicationException("No interpreter found"); + Interpreter intp = null; + try { + intp = paragraph.getBindedInterpreter(); + } catch (InterpreterException e) { + throw new ApplicationException("No interpreter found", e); } RemoteInterpreterProcess intpProcess = @@ -417,7 +421,13 @@ public void onNoteCreate(Note note) { @Override public void onUnbindInterpreter(Note note, InterpreterSetting setting) { for (Paragraph p : note.getParagraphs()) { - Interpreter currentInterpreter = p.getBindedInterpreter(); + Interpreter currentInterpreter = null; + try { + currentInterpreter = p.getBindedInterpreter(); + } catch (InterpreterNotFoundException e) { + logger.warn("Not interpreter found", e); + return; + } List infos = setting.getInterpreterInfos(); for (InterpreterInfo info : infos) { if (currentInterpreter != null && diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 911c2855f27..20b1c2be7af 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -50,14 +50,14 @@ private InterpreterSetting getInterpreterSettingByGroup(List return null; } - public Interpreter getInterpreter(String user, String noteId, String replName) { + public Interpreter getInterpreter(String user, String noteId, String replName) + throws InterpreterNotFoundException { List settings = interpreterSettingManager.getInterpreterSettings(noteId); InterpreterSetting setting; Interpreter interpreter; if (settings == null || settings.size() == 0) { - LOGGER.error("No interpreter is binded to this note: " + noteId); - return null; + throw new InterpreterNotFoundException("No interpreter is binded to this note: " + noteId); } if (StringUtils.isBlank(replName)) { @@ -76,9 +76,10 @@ public Interpreter getInterpreter(String user, String noteId, String replName) { if (null != interpreter) { return interpreter; } - throw new RuntimeException("No such interpreter: " + replName); + throw new InterpreterNotFoundException("No such interpreter: " + replName); } - throw new RuntimeException("Interpreter " + group + " is not binded to this note"); + throw new InterpreterNotFoundException("Interpreter " + group + + " is not binded to this note"); } else if (replNameSplit.length == 1){ // first assume replName is 'name' of interpreter. ('groupName' is ommitted) // search 'name' from first (default) interpreter group @@ -97,11 +98,12 @@ public Interpreter getInterpreter(String user, String noteId, String replName) { if (null != setting) { return setting.getDefaultInterpreter(user, noteId); } else { - throw new RuntimeException("Either no interpreter named " + replName + " or it is not " + - "binded to this note"); + throw new InterpreterNotFoundException("Either no interpreter named " + replName + + " or it is not binded to this note"); } } - //TODO(zjffdu) throw InterpreterException instead of return null - return null; + + throw new InterpreterNotFoundException("No such interpreter " + replName + " for note " + + noteId); } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java new file mode 100644 index 00000000000..192e8221664 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java @@ -0,0 +1,22 @@ +package org.apache.zeppelin.interpreter; + +/** + * Exception for no interpreter is found + */ +public class InterpreterNotFoundException extends InterpreterException { + + public InterpreterNotFoundException() { + } + + public InterpreterNotFoundException(String message) { + super(message); + } + + public InterpreterNotFoundException(String message, Throwable cause) { + super(message, cause); + } + + public InterpreterNotFoundException(Throwable cause) { + super(cause); + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 76c48c844ca..23d2d4794eb 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -43,6 +43,7 @@ import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterNotFoundException; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterOutputListener; @@ -237,7 +238,7 @@ public boolean isEnabled() { return enabled == null || enabled.booleanValue(); } - public Interpreter getBindedInterpreter() { + public Interpreter getBindedInterpreter() throws InterpreterNotFoundException { return this.interpreterFactory.getInterpreter(user, note.getId(), intpText); } @@ -253,8 +254,11 @@ public List completion(String buffer, int cursor) { return note.getInterpreterCompletion(); } } - this.interpreter = getBindedInterpreter(); - + try { + this.interpreter = getBindedInterpreter(); + } catch (InterpreterNotFoundException e) { + return null; + } setText(buffer); cursor = calculateCursorPosition(buffer, cursor); @@ -342,9 +346,27 @@ public boolean execute(boolean blocking) { } clearRuntimeInfo(null); - this.interpreter = getBindedInterpreter(); + try { + this.interpreter = getBindedInterpreter(); + setStatus(Status.READY); + if (getConfig().get("enabled") == null || (Boolean) getConfig().get("enabled")) { + setAuthenticationInfo(getAuthenticationInfo()); + interpreter.getScheduler().submit(this); + } - if (interpreter == null) { + if (blocking) { + while (!getStatus().isCompleted()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return getStatus() == Status.FINISHED; + } else { + return true; + } + } catch (InterpreterNotFoundException e) { String intpExceptionMsg = getJobName() + "'s Interpreter " + getIntpText() + " not found"; RuntimeException intpException = new RuntimeException(intpExceptionMsg); @@ -354,24 +376,6 @@ public boolean execute(boolean blocking) { setStatus(Job.Status.ERROR); throw intpException; } - setStatus(Status.READY); - if (getConfig().get("enabled") == null || (Boolean) getConfig().get("enabled")) { - setAuthenticationInfo(getAuthenticationInfo()); - interpreter.getScheduler().submit(this); - } - - if (blocking) { - while (!getStatus().isCompleted()) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - return getStatus() == Status.FINISHED; - } else { - return true; - } } @Override @@ -702,7 +706,11 @@ String extractVariablesFromAngularRegistry(String scriptBody, Map } public boolean isValidInterpreter(String replName) { - return interpreterFactory.getInterpreter(user, note.getId(), replName) != null; + try { + return interpreterFactory.getInterpreter(user, note.getId(), replName) != null; + } catch (InterpreterNotFoundException e) { + return false; + } } public void updateRuntimeInfos(String label, String tooltip, Map infos, diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 0952de67d40..03de533a2b6 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -20,6 +20,7 @@ import org.apache.zeppelin.interpreter.AbstractInterpreterTest; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterNotFoundException; import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.ApplicationState; @@ -44,6 +45,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; public class HeliumApplicationFactoryTest extends AbstractInterpreterTest implements JobListenerFactory { @@ -229,8 +231,13 @@ public void testInterpreterUnbindOfNullReplParagraph() throws IOException { p1.setText("%fake "); // make sure that p1's repl is null - Interpreter intp = p1.getBindedInterpreter(); - assertEquals(intp, null); + Interpreter intp = null; + try { + intp = p1.getBindedInterpreter(); + fail("Should throw InterpreterNotFoundException"); + } catch (InterpreterNotFoundException e) { + + } // Unbind all interpreter from note // NullPointerException shouldn't occur here diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 760bdf09dda..2fef05d8482 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -31,10 +31,14 @@ public class InterpreterFactoryTest extends AbstractInterpreterTest { @Test - public void testGetFactory() throws IOException { + public void testGetFactory() throws IOException, InterpreterException { // no default interpreter because there's no interpreter setting binded to this note - assertNull(interpreterFactory.getInterpreter("user1", "note1", "")); + try { + interpreterFactory.getInterpreter("user1", "note1", ""); + fail("Should throw InterpreterNotFoundException"); + } catch (InterpreterNotFoundException e) { + } interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds()); assertTrue(interpreterFactory.getInterpreter("user1", "note1", "") instanceof RemoteInterpreter); RemoteInterpreter remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", ""); @@ -61,7 +65,7 @@ public void testUnknownRepl1() throws IOException { try { interpreterFactory.getInterpreter("user1", "note1", "test.unknown_repl"); fail("should fail due to no such interpreter"); - } catch (RuntimeException e) { + } catch (InterpreterNotFoundException e) { assertEquals("No such interpreter: test.unknown_repl", e.getMessage()); } } @@ -72,7 +76,7 @@ public void testUnknownRepl2() throws IOException { try { interpreterFactory.getInterpreter("user1", "note1", "unknown_repl"); fail("should fail due to no such interpreter"); - } catch (RuntimeException e) { + } catch (InterpreterNotFoundException e) { assertEquals("Either no interpreter named unknown_repl or it is not binded to this note", e.getMessage()); } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java index ec79ada626f..a52601c40b1 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java @@ -200,7 +200,7 @@ public void testInterpreterBinding() throws IOException { } @Test - public void testUpdateInterpreterBinding_PerNoteShared() throws IOException { + public void testUpdateInterpreterBinding_PerNoteShared() throws IOException, InterpreterNotFoundException { InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0); defaultInterpreterSetting.getOption().setPerNote("shared"); @@ -220,7 +220,7 @@ public void testUpdateInterpreterBinding_PerNoteShared() throws IOException { } @Test - public void testUpdateInterpreterBinding_PerNoteIsolated() throws IOException { + public void testUpdateInterpreterBinding_PerNoteIsolated() throws IOException, InterpreterNotFoundException { InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0); defaultInterpreterSetting.getOption().setPerNote("isolated"); @@ -241,7 +241,7 @@ public void testUpdateInterpreterBinding_PerNoteIsolated() throws IOException { } @Test - public void testUpdateInterpreterBinding_PerNoteScoped() throws IOException { + public void testUpdateInterpreterBinding_PerNoteScoped() throws IOException, InterpreterNotFoundException { InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0); defaultInterpreterSetting.getOption().setPerNote("scoped"); @@ -266,7 +266,7 @@ public void testUpdateInterpreterBinding_PerNoteScoped() throws IOException { } @Test - public void testGetEditor() throws IOException { + public void testGetEditor() throws IOException, InterpreterNotFoundException { interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds()); Interpreter echoInterpreter = interpreterFactory.getInterpreter("user1", "note1", "test.echo"); // get editor setting from interpreter-setting.json diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java index 286affdd79b..824fe10d653 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java @@ -23,6 +23,7 @@ import org.apache.zeppelin.display.ui.TextBox; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterNotFoundException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.repo.NotebookRepo; @@ -73,7 +74,7 @@ public class NoteTest { private AuthenticationInfo anonymous = new AuthenticationInfo("anonymous"); @Test - public void runNormalTest() { + public void runNormalTest() throws InterpreterNotFoundException { when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter); when(interpreter.getScheduler()).thenReturn(scheduler); @@ -101,7 +102,7 @@ public void addParagraphWithEmptyReplNameTest() { } @Test - public void addParagraphWithLastReplNameTest() { + public void addParagraphWithLastReplNameTest() throws InterpreterNotFoundException { when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter); Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); @@ -113,7 +114,7 @@ public void addParagraphWithLastReplNameTest() { } @Test - public void insertParagraphWithLastReplNameTest() { + public void insertParagraphWithLastReplNameTest() throws InterpreterNotFoundException { when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter); Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); @@ -125,7 +126,7 @@ public void insertParagraphWithLastReplNameTest() { } @Test - public void insertParagraphWithInvalidReplNameTest() { + public void insertParagraphWithInvalidReplNameTest() throws InterpreterNotFoundException { when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("invalid"))).thenReturn(null); Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); @@ -144,7 +145,7 @@ public void insertParagraphwithUser() { } @Test - public void clearAllParagraphOutputTest() { + public void clearAllParagraphOutputTest() throws InterpreterNotFoundException { when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("md"))).thenReturn(interpreter); when(interpreter.getScheduler()).thenReturn(scheduler); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 02490ea685c..122395cc611 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -26,6 +26,7 @@ import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterNotFoundException; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResultMessage; @@ -145,7 +146,12 @@ public void testReloadAndSetInterpreter() throws IOException { // then interpreter factory should be injected into all the paragraphs Note note = notebook.getAllNotes().get(0); - assertNull(note.getParagraphs().get(0).getBindedInterpreter()); + try { + note.getParagraphs().get(0).getBindedInterpreter(); + fail("Should throw InterpreterNotFoundException"); + } catch (InterpreterNotFoundException e) { + + } } @Test @@ -523,7 +529,7 @@ private void terminateScheduledNote(Note note) { @Test - public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{ + public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException, InterpreterNotFoundException { // create a note and a paragraph Note note = notebook.createNote(anonymous); interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); @@ -573,7 +579,7 @@ public void testAutoRestartInterpreterAfterSchedule() throws InterruptedExceptio @Test public void testCronWithReleaseResourceClosesOnlySpecificInterpreters() - throws IOException, InterruptedException { + throws IOException, InterruptedException, InterpreterNotFoundException { // create a cron scheduled note. Note cronNote = notebook.createNote(anonymous); interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), cronNote.getId(), From fe660abb9663173998d6e02504ef3238f869c080 Mon Sep 17 00:00:00 2001 From: Andreas Weise Date: Sun, 18 Feb 2018 12:55:32 +0100 Subject: [PATCH 063/386] [ZEPPELIN-3243] fix IndexOutOfBoundsException when 'No interpreter is binded to this note' ### What is this PR for? I observed IndexOutOfBoundsException when using zeppelin locally on an existing notebook repository and with fresh interpreter.json file. The default Zeppelin behavior when using existing notebooks is the manual binding of the interpreter settings, which works fine. This PR simply avoids the IndexOutOfBoundsException. ### What type of PR is it? Bug Fix ### Todos - ### What is the Jira issue? [https://issues.apache.org/jira/browse/ZEPPELIN-3243](https://issues.apache.org/jira/browse/ZEPPELIN-3243) ### How should this be tested? - manual ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Andreas Weise Closes #2808 from weand/ZEPPELIN-3243 and squashes the following commits: b97a38e [Andreas Weise] ZEPPELIN-3243 fix IndexOutOfBoundsException when 'No interpreter is binded to this note' --- .../zeppelin/interpreter/InterpreterSettingManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 711812e07f4..c3ec81bbb44 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -412,7 +412,8 @@ private void registerInterpreterSetting(List registeredIn @VisibleForTesting public InterpreterSetting getDefaultInterpreterSetting(String noteId) { - return getInterpreterSettings(noteId).get(0); + List allInterpreterSettings = getInterpreterSettings(noteId); + return allInterpreterSettings.size() > 0 ? allInterpreterSettings.get(0) : null; } public List getInterpreterSettings(String noteId) { @@ -468,7 +469,7 @@ public Map getEditorSetting(Interpreter interpreter, String user group = replNameSplit[0]; } // when replName is 'name' of interpreter - if (defaultSettingName.equals(intpSetting.getName())) { + if (intpSetting.getName().equals(defaultSettingName)) { editor = intpSetting.getEditorFromSettingByClassName(interpreter.getClassName()); } // when replName is 'alias name' of interpreter or 'group' of interpreter From a6cd0e24cc795985e200ebebf04de981e13de97c Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 5 Mar 2018 14:38:00 +0800 Subject: [PATCH 064/386] ZEPPELIN-3279. [FlakyTest] NotebookTest.testPerSessionInterpreterCloseOnUnbindInterpreterSetting ### What is this PR for? The root cause of this flaky test is that hashCode doesn't represent an unique id of object. This PR just use IdGenerator for unique id and also combine process id. This PR also fix the flaky test caused by matplotlib. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3279 ### How should this be tested? * CI Pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2837 from zjffdu/ZEPPELIN-3279 and squashes the following commits: 063d4ee [Jeff Zhang] ZEPPELIN-3279. [FlakyTest] NotebookTest.testPerSessionInterpreterCloseOnUnbindInterpreterSetting --- testing/install_external_dependencies.sh | 2 +- .../zeppelin/interpreter/mock/MockInterpreter1.java | 11 ++++++++++- .../org/apache/zeppelin/notebook/NotebookTest.java | 5 +++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/testing/install_external_dependencies.sh b/testing/install_external_dependencies.sh index d6c07368b9d..a120d6174dc 100755 --- a/testing/install_external_dependencies.sh +++ b/testing/install_external_dependencies.sh @@ -44,6 +44,6 @@ if [[ -n "$PYTHON" ]] ; then conda update -q conda conda info -a conda config --add channels conda-forge - conda install -q matplotlib pandasql ipython=5.4.1 jupyter_client ipykernel matplotlib bokeh=0.12.10 + conda install -q matplotlib=2.1.2 pandasql ipython=5.4.1 jupyter_client ipykernel matplotlib bokeh=0.12.10 pip install -q grpcio ggplot bkzep==0.4.0 fi diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java index a533c12913e..500c4f72094 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java @@ -24,18 +24,27 @@ import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; +import java.lang.management.ManagementFactory; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; public class MockInterpreter1 extends Interpreter { + private static AtomicInteger IdGenerator = new AtomicInteger(); + + private int object_id; + private String pid; Map vars = new HashMap<>(); public MockInterpreter1(Properties property) { super(property); + this.object_id = IdGenerator.getAndIncrement(); + this.pid = ManagementFactory.getRuntimeMXBean().getName().split("@")[0]; } + boolean open; @@ -60,7 +69,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) { if ("getId".equals(st)) { // get unique id of this interpreter instance - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode()); + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.object_id + "-" + this.pid); } else if (st.startsWith("sleep")) { try { Thread.sleep(Integer.parseInt(st.split(" ")[1])); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 122395cc611..ee04291be5b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -44,6 +44,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.JUnitCore; +import org.junit.runner.Request; +import org.junit.runner.Result; +import org.mockito.internal.runners.JUnit44RunnerImpl; import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1450,4 +1454,5 @@ public void afterStatusChange(Job job, Status before, Status after) { private interface StatusChangedListener { void onStatusChanged(Job job, Status before, Status after); } + } From c77c549e3f6bfa7b08ebc0bfaacb2bd8719c875f Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 5 Mar 2018 15:52:14 +0800 Subject: [PATCH 065/386] ZEPPELIN-3277. NotebookServerTest.testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent fails ### What is this PR for? Trivial fix for flaky test, see the comment for details. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3277 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2838 from zjffdu/ZEPPELIN-3277 and squashes the following commits: 68827d7 [Jeff Zhang] ZEPPELIN-3277. NotebookServerTest.testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent fails --- .../java/org/apache/zeppelin/socket/NotebookServerTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java index a0cb58745be..0b68ca64b19 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java @@ -123,6 +123,8 @@ public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() thr } Thread.sleep(100); } + // sleep for 1 second to make sure job running thread finish to fire event. See ZEPPELIN-3277 + Thread.sleep(1000); // add angularObject interpreterGroup.getAngularObjectRegistry().add("object1", "value1", note1.getId(), null); From 0cff6f0f06570fdf331c102377402ac9732282e5 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 6 Mar 2018 22:29:41 +0800 Subject: [PATCH 066/386] ZEPPELIN-3295. Remove build of profile spark-1.6 and scala-2.11 ### What is this PR for? Trivial PR to remove the build of profile spark-1.6 and scala-2.11, The travis build of profile spark-1.6 and scala-2.11 only test the embedded mode of spark instead real spark binary distribution of spark 1.6 with scala-2.11, so IMHO it is safe for us to just remove that build and only keep spark 1.6 with scala-2.10 ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3295 ### How should this be tested? * Travis CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2840 from zjffdu/ZEPPELIN-3295 and squashes the following commits: 946cf5d [Jeff Zhang] ZEPPELIN-3295. Remove build of profile spark-1.6 and scala-2.11 --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index aba29443835..a6f72c87a79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -114,11 +114,6 @@ matrix: dist: trusty env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test spark module for 1.6.3 with scala 2.11 - - jdk: "oraclejdk8" - dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test python/pyspark with python 2, livy 0.5 - sudo: required dist: trusty From 483dc3f2bb46d18b7bbb41d72118c356bd9de403 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 6 Mar 2018 17:17:35 +0800 Subject: [PATCH 067/386] ZEPPELIN-3296. Reorg livy integration test to minimize livy session ### What is this PR for? Just refactor livy integration test to minuze livy session so that we can reduce the livy build time. ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3296 ### How should this be tested? * Travis CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2844 from zjffdu/ZEPPELIN-3296 and squashes the following commits: 206ea3e [Jeff Zhang] ZEPPELIN-3296. Reorg livy integration test to minimize livy session --- .travis.yml | 9 +- .../zeppelin/livy/LivyInterpreterIT.java | 646 +++++++----------- 2 files changed, 250 insertions(+), 405 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6f72c87a79..9edb1986841 100644 --- a/.travis.yml +++ b/.travis.yml @@ -114,17 +114,16 @@ matrix: dist: trusty env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test python/pyspark with python 2, livy 0.5 - sudo: required dist: trusty jdk: "openjdk7" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="-Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="2" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="" - # Test python/pyspark with python 3, livy 0.5 + # Test livy 0.5 with spark 2.2.0 under python3 - sudo: required dist: trusty - jdk: "openjdk7" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="-Dpyspark.test.exclude='' -DfailIfNoTests=false" + jdk: "openjdk8" + env: PYTHON="3" SPARK_VER="2.2.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS="" before_install: # check files included in commit range, clear bower_components if a bower.json file has changed. diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 3dfeb363f8f..96fdbea876b 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -18,17 +18,27 @@ package org.apache.zeppelin.livy; +import org.apache.commons.io.IOUtils; import org.apache.livy.test.framework.Cluster; import org.apache.livy.test.framework.Cluster$; -import org.apache.commons.io.IOUtils; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterOutputListener; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.user.AuthenticationInfo; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Properties; @@ -78,128 +88,7 @@ public static boolean checkPreCondition() { @Test - public void testSparkInterpreterRDD() throws InterpreterException { - if (!checkPreCondition()) { - return; - } - InterpreterGroup interpreterGroup = new InterpreterGroup("group_1"); - interpreterGroup.put("session_1", new ArrayList()); - final LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties); - sparkInterpreter.setInterpreterGroup(interpreterGroup); - interpreterGroup.get("session_1").add(sparkInterpreter); - AuthenticationInfo authInfo = new AuthenticationInfo("user1"); - MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener(); - InterpreterOutput output = new InterpreterOutput(outputListener); - final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark", - "title", "text", authInfo, null, null, null, null, null, null, output); - sparkInterpreter.open(); - - try { - // detect spark version - InterpreterResult result = sparkInterpreter.interpret("sc.version", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - - boolean isSpark2 = isSpark2(sparkInterpreter, context); - - // test RDD api - result = sparkInterpreter.interpret("sc.parallelize(1 to 10).sum()", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData().contains("Double = 55.0")); - - // single line comment - String singleLineComment = "println(1)// my comment"; - result = sparkInterpreter.interpret(singleLineComment, context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - - // multiple line comment - String multipleLineComment = "println(1)/* multiple \n" + "line \n" + "comment */"; - result = sparkInterpreter.interpret(multipleLineComment, context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - - // multi-line string - String multiLineString = "val str = \"\"\"multiple\n" + - "line\"\"\"\n" + - "println(str)"; - result = sparkInterpreter.interpret(multiLineString, context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData().contains("multiple\nline")); - - // case class - String caseClassCode = "case class Person(id:Int, \n" + - "name:String)\n" + - "val p=Person(1, \"name_a\")"; - result = sparkInterpreter.interpret(caseClassCode, context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData().contains("p: Person = Person(1,name_a)")); - - // object class - String objectClassCode = "object Person {}"; - result = sparkInterpreter.interpret(objectClassCode, context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - if (!isSpark2) { - assertTrue(result.message().get(0).getData().contains("defined module Person")); - } else { - assertTrue(result.message().get(0).getData().contains("defined object Person")); - } - - // html output - String htmlCode = "println(\"%html

    hello

    \")"; - result = sparkInterpreter.interpret(htmlCode, context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType()); - - // error - result = sparkInterpreter.interpret("println(a)", context); - assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); - assertTrue(result.message().get(0).getData().contains("error: not found: value a")); - - // incomplete code - result = sparkInterpreter.interpret("if(true){", context); - assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); - assertTrue(result.message().get(0).getData().contains("incomplete statement")); - - // cancel - if (sparkInterpreter.livyVersion.newerThanEquals(LivyVersion.LIVY_0_3_0)) { - Thread cancelThread = new Thread() { - @Override - public void run() { - // invoke cancel after 1 millisecond to wait job starting - try { - Thread.sleep(1); - } catch (InterruptedException e) { - e.printStackTrace(); - } - sparkInterpreter.cancel(context); - } - }; - cancelThread.start(); - result = sparkInterpreter - .interpret("sc.parallelize(1 to 10).foreach(e=>Thread.sleep(10*1000))", context); - assertEquals(InterpreterResult.Code.ERROR, result.code()); - String message = result.message().get(0).getData(); - // 2 possibilities, sometimes livy doesn't return the real cancel exception - assertTrue(message.contains("cancelled part of cancelled job group") || - message.contains("Job is cancelled")); - } - - } finally { - sparkInterpreter.close(); - } - } - - - @Test - public void testSparkInterpreterDataFrame() throws InterpreterException { + public void testSparkInterpreter() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -227,309 +116,229 @@ public void testSparkInterpreterDataFrame() throws InterpreterException { assertEquals(1, result.message().size()); boolean isSpark2 = isSpark2(sparkInterpreter, context); + testRDD(sparkInterpreter, isSpark2); + testDataFrame(sparkInterpreter, sqlInterpreter, isSpark2); - // test DataFrame api - if (!isSpark2) { - result = sparkInterpreter.interpret( - "val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n" - + "df.collect()", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData() - .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])")); - } else { - result = sparkInterpreter.interpret( - "val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n" - + "df.collect()", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData() - .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])")); - } - sparkInterpreter.interpret("df.registerTempTable(\"df\")", context); - // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter - result = sqlInterpreter.interpret("select * from df where col_1='hello'", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - assertEquals("col_1\tcol_2\nhello\t20", result.message().get(0).getData()); - // double quotes - result = sqlInterpreter.interpret("select * from df where col_1=\"hello\"", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - assertEquals("col_1\tcol_2\nhello\t20", result.message().get(0).getData()); - - // only enable this test in spark2 as spark1 doesn't work for this case - if (isSpark2) { - result = sqlInterpreter.interpret("select * from df where col_1=\"he\\\"llo\" ", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - } - - // single quotes inside attribute value - result = sqlInterpreter.interpret("select * from df where col_1=\"he'llo\"", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - - // test sql with syntax error - result = sqlInterpreter.interpret("select * from df2", context); - assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); - - if (!isSpark2) { - assertTrue(result.message().get(0).getData().contains("Table not found")); - } else { - assertTrue(result.message().get(0).getData().contains("Table or view not found")); - } } finally { sparkInterpreter.close(); sqlInterpreter.close(); } } - @Test - public void testSparkSQLInterpreter() throws InterpreterException { - if (!checkPreCondition()) { - return; - } - InterpreterGroup interpreterGroup = new InterpreterGroup("group_1"); - interpreterGroup.put("session_1", new ArrayList()); - LazyOpenInterpreter sparkInterpreter = new LazyOpenInterpreter( - new LivySparkInterpreter(properties)); - sparkInterpreter.setInterpreterGroup(interpreterGroup); - interpreterGroup.get("session_1").add(sparkInterpreter); - LazyOpenInterpreter sqlInterpreter = new LazyOpenInterpreter( - new LivySparkSQLInterpreter(properties)); - interpreterGroup.get("session_1").add(sqlInterpreter); - sqlInterpreter.setInterpreterGroup(interpreterGroup); - sqlInterpreter.open(); - - try { - AuthenticationInfo authInfo = new AuthenticationInfo("user1"); - MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener(); - InterpreterOutput output = new InterpreterOutput(outputListener); - InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.sql", - "title", "text", authInfo, null, null, null, null, null, null, output); - InterpreterResult result = sqlInterpreter.interpret("show tables", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - assertTrue(result.message().get(0).getData().contains("tableName")); - int r = sqlInterpreter.getProgress(context); - assertTrue(r == 0); - } finally { - sqlInterpreter.close(); - } - } - - - @Test - public void testSparkSQLCancellation() throws InterpreterException { - if (!checkPreCondition()) { - return; - } - InterpreterGroup interpreterGroup = new InterpreterGroup("group_1"); - interpreterGroup.put("session_1", new ArrayList()); - LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties); - sparkInterpreter.setInterpreterGroup(interpreterGroup); - interpreterGroup.get("session_1").add(sparkInterpreter); + private void testRDD(final LivySparkInterpreter sparkInterpreter, boolean isSpark2) { AuthenticationInfo authInfo = new AuthenticationInfo("user1"); MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener(); InterpreterOutput output = new InterpreterOutput(outputListener); final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark", "title", "text", authInfo, null, null, null, null, null, null, output); - sparkInterpreter.open(); - - final LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties); - interpreterGroup.get("session_1").add(sqlInterpreter); - sqlInterpreter.setInterpreterGroup(interpreterGroup); - sqlInterpreter.open(); - - try { - // detect spark version - InterpreterResult result = sparkInterpreter.interpret("sc.version", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - - boolean isSpark2 = isSpark2(sparkInterpreter, context); - - // test DataFrame api - if (!isSpark2) { - result = sparkInterpreter.interpret( - "val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n" - + "df.collect()", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData() - .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])")); - } else { - result = sparkInterpreter.interpret( - "val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n" - + "df.collect()", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData() - .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])")); - } - sparkInterpreter.interpret("df.registerTempTable(\"df\")", context); - // cancel - if (sqlInterpreter.getLivyVersion().newerThanEquals(LivyVersion.LIVY_0_3_0)) { - Thread cancelThread = new Thread() { - @Override - public void run() { - sqlInterpreter.cancel(context); + InterpreterResult result = sparkInterpreter.interpret("sc.parallelize(1 to 10).sum()", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData().contains("Double = 55.0")); + + // single line comment + String singleLineComment = "println(1)// my comment"; + result = sparkInterpreter.interpret(singleLineComment, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // multiple line comment + String multipleLineComment = "println(1)/* multiple \n" + "line \n" + "comment */"; + result = sparkInterpreter.interpret(multipleLineComment, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // multi-line string + String multiLineString = "val str = \"\"\"multiple\n" + + "line\"\"\"\n" + + "println(str)"; + result = sparkInterpreter.interpret(multiLineString, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData().contains("multiple\nline")); + + // case class + String caseClassCode = "case class Person(id:Int, \n" + + "name:String)\n" + + "val p=Person(1, \"name_a\")"; + result = sparkInterpreter.interpret(caseClassCode, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData().contains("p: Person = Person(1,name_a)")); + + // object class + String objectClassCode = "object Person {}"; + result = sparkInterpreter.interpret(objectClassCode, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + if (!isSpark2) { + assertTrue(result.message().get(0).getData().contains("defined module Person")); + } else { + assertTrue(result.message().get(0).getData().contains("defined object Person")); + } + + // html output + String htmlCode = "println(\"%html

    hello

    \")"; + result = sparkInterpreter.interpret(htmlCode, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType()); + + // error + result = sparkInterpreter.interpret("println(a)", context); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); + assertTrue(result.message().get(0).getData().contains("error: not found: value a")); + + // incomplete code + result = sparkInterpreter.interpret("if(true){", context); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); + assertTrue(result.message().get(0).getData().contains("incomplete statement")); + + // cancel + if (sparkInterpreter.livyVersion.newerThanEquals(LivyVersion.LIVY_0_3_0)) { + Thread cancelThread = new Thread() { + @Override + public void run() { + // invoke cancel after 1 millisecond to wait job starting + try { + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); } - }; - cancelThread.start(); - //sleep so that cancelThread performs a cancel. - try { - Thread.sleep(1); - } catch (InterruptedException e) { - e.printStackTrace(); - } - result = sqlInterpreter - .interpret("select count(1) from df", context); - if (result.code().equals(InterpreterResult.Code.ERROR)) { - String message = result.message().get(0).getData(); - // 2 possibilities, sometimes livy doesn't return the real cancel exception - assertTrue(message.contains("cancelled part of cancelled job group") || - message.contains("Job is cancelled")); + sparkInterpreter.cancel(context); } - } - } catch (LivyException e) { - } finally { - sparkInterpreter.close(); - sqlInterpreter.close(); + }; + cancelThread.start(); + result = sparkInterpreter + .interpret("sc.parallelize(1 to 10).foreach(e=>Thread.sleep(10*1000))", context); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + String message = result.message().get(0).getData(); + // 2 possibilities, sometimes livy doesn't return the real cancel exception + assertTrue(message.contains("cancelled part of cancelled job group") || + message.contains("Job is cancelled")); } } - @Test - public void testStringWithTruncation() throws InterpreterException { - if (!checkPreCondition()) { - return; - } - InterpreterGroup interpreterGroup = new InterpreterGroup("group_1"); - interpreterGroup.put("session_1", new ArrayList()); - LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties); - sparkInterpreter.setInterpreterGroup(interpreterGroup); - interpreterGroup.get("session_1").add(sparkInterpreter); + private void testDataFrame(LivySparkInterpreter sparkInterpreter, + final LivySparkSQLInterpreter sqlInterpreter, + boolean isSpark2) throws LivyException { AuthenticationInfo authInfo = new AuthenticationInfo("user1"); MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener(); InterpreterOutput output = new InterpreterOutput(outputListener); - InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark", + final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark", "title", "text", authInfo, null, null, null, null, null, null, output); - sparkInterpreter.open(); - - LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties); - interpreterGroup.get("session_1").add(sqlInterpreter); - sqlInterpreter.setInterpreterGroup(interpreterGroup); - sqlInterpreter.open(); - try { - // detect spark version - InterpreterResult result = sparkInterpreter.interpret("sc.version", context); + InterpreterResult result = null; + // test DataFrame api + if (!isSpark2) { + result = sparkInterpreter.interpret( + "val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n" + + "df.collect()", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(1, result.message().size()); - - boolean isSpark2 = isSpark2(sparkInterpreter, context); - - // test DataFrame api - if (!isSpark2) { - result = sparkInterpreter.interpret( - "val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20))).toDF(\"col_1\", \"col_2\")\n" - + "df.collect()", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData() - .contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])")); - } else { - result = sparkInterpreter.interpret( - "val df=spark.createDataFrame(Seq((\"12characters12characters\",20))).toDF(\"col_1\", \"col_2\")\n" - + "df.collect()", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData() - .contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])")); - } - sparkInterpreter.interpret("df.registerTempTable(\"df\")", context); - // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter - result = sqlInterpreter.interpret("select * from df where col_1='12characters12characters'", context); + assertTrue(result.message().get(0).getData() + .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])")); + } else { + result = sparkInterpreter.interpret( + "val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n" + + "df.collect()", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData() + .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])")); + } + sparkInterpreter.interpret("df.registerTempTable(\"df\")", context); + // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter + result = sqlInterpreter.interpret("select * from df where col_1='hello'", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); + assertEquals("col_1\tcol_2\nhello\t20", result.message().get(0).getData()); + // double quotes + result = sqlInterpreter.interpret("select * from df where col_1=\"hello\"", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); + assertEquals("col_1\tcol_2\nhello\t20", result.message().get(0).getData()); + + // only enable this test in spark2 as spark1 doesn't work for this case + if (isSpark2) { + result = sqlInterpreter.interpret("select * from df where col_1=\"he\\\"llo\" ", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - assertEquals("col_1\tcol_2\n12characters12cha...\t20", result.message().get(0).getData()); - } finally { - sparkInterpreter.close(); - sqlInterpreter.close(); } - } + // single quotes inside attribute value + result = sqlInterpreter.interpret("select * from df where col_1=\"he'llo\"", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - @Test - public void testStringWithoutTruncation() throws InterpreterException { - if (!checkPreCondition()) { - return; - } - InterpreterGroup interpreterGroup = new InterpreterGroup("group_1"); - interpreterGroup.put("session_1", new ArrayList()); - Properties newProps = new Properties(); - for (Object name: properties.keySet()) { - newProps.put(name, properties.get(name)); + // test sql with syntax error + result = sqlInterpreter.interpret("select * from df2", context); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); + + if (!isSpark2) { + assertTrue(result.message().get(0).getData().contains("Table not found")); + } else { + assertTrue(result.message().get(0).getData().contains("Table or view not found")); } - newProps.put(LivySparkSQLInterpreter.ZEPPELIN_LIVY_SPARK_SQL_FIELD_TRUNCATE, "false"); - LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(newProps); - sparkInterpreter.setInterpreterGroup(interpreterGroup); - interpreterGroup.get("session_1").add(sparkInterpreter); - AuthenticationInfo authInfo = new AuthenticationInfo("user1"); - MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener(); - InterpreterOutput output = new InterpreterOutput(outputListener); - InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark", - "title", "text", authInfo, null, null, null, null, null, null, output); - sparkInterpreter.open(); - LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(newProps); - interpreterGroup.get("session_1").add(sqlInterpreter); - sqlInterpreter.setInterpreterGroup(interpreterGroup); - sqlInterpreter.open(); + // test sql cancel + if (sqlInterpreter.getLivyVersion().newerThanEquals(LivyVersion.LIVY_0_3_0)) { + Thread cancelThread = new Thread() { + @Override + public void run() { + sqlInterpreter.cancel(context); + } + }; + cancelThread.start(); + //sleep so that cancelThread performs a cancel. + try { + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); + } + result = sqlInterpreter + .interpret("select count(1) from df", context); + if (result.code().equals(InterpreterResult.Code.ERROR)) { + String message = result.message().get(0).getData(); + // 2 possibilities, sometimes livy doesn't return the real cancel exception + assertTrue(message.contains("cancelled part of cancelled job group") || + message.contains("Job is cancelled")); + } + } - try { - // detect spark version - InterpreterResult result = sparkInterpreter.interpret("sc.version", context); + // test result string truncate + if (!isSpark2) { + result = sparkInterpreter.interpret( + "val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20))).toDF(\"col_1\", \"col_2\")\n" + + "df.collect()", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(1, result.message().size()); - - boolean isSpark2 = isSpark2(sparkInterpreter, context); - - // test DataFrame api - if (!isSpark2) { - result = sparkInterpreter.interpret( - "val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20))).toDF(\"col_1\", \"col_2\")\n" - + "df.collect()", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData() - .contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])")); - } else { - result = sparkInterpreter.interpret( - "val df=spark.createDataFrame(Seq((\"12characters12characters\",20))).toDF(\"col_1\", \"col_2\")\n" - + "df.collect()", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(1, result.message().size()); - assertTrue(result.message().get(0).getData() - .contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])")); - } - sparkInterpreter.interpret("df.registerTempTable(\"df\")", context); - // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter - result = sqlInterpreter.interpret("select * from df where col_1='12characters12characters'", context); + assertTrue(result.message().get(0).getData() + .contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])")); + } else { + result = sparkInterpreter.interpret( + "val df=spark.createDataFrame(Seq((\"12characters12characters\",20))).toDF(\"col_1\", \"col_2\")\n" + + "df.collect()", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); - assertEquals("col_1\tcol_2\n12characters12characters\t20", result.message().get(0).getData()); - } finally { - sparkInterpreter.close(); - sqlInterpreter.close(); + assertEquals(1, result.message().size()); + assertTrue(result.message().get(0).getData() + .contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])")); } + sparkInterpreter.interpret("df.registerTempTable(\"df\")", context); + // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter + result = sqlInterpreter.interpret("select * from df where col_1='12characters12characters'", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); + assertEquals("col_1\tcol_2\n12characters12cha...\t20", result.message().get(0).getData()); + } @Test - public void testPySparkInterpreter() throws LivyException, InterpreterException { + public void testPySparkInterpreter() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -549,7 +358,7 @@ public void testPySparkInterpreter() throws LivyException, InterpreterException // for livy version >=0.3 , input some erroneous spark code, check the shown result is more than one line InterpreterResult result = pysparkInterpreter.interpret("sc.parallelize(wrongSyntax(1, 2)).count()", context); assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertTrue(result.message().get(0).getData().split("\n").length>1); + assertTrue(result.message().get(0).getData().split("\n").length > 1); assertTrue(result.message().get(0).getData().contains("Traceback")); } catch (APINotFoundException e) { // only livy 0.2 can throw this exception since it doesn't have /version endpoint @@ -557,17 +366,17 @@ public void testPySparkInterpreter() throws LivyException, InterpreterException // traceback InterpreterResult result = pysparkInterpreter.interpret("print(a)", context); assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertTrue(result.message().get(0).getData().split("\n").length>1); + assertTrue(result.message().get(0).getData().split("\n").length > 1); assertTrue(result.message().get(0).getData().contains("Traceback")); } // test utf-8 Encoding try { String utf8Str = "你你你你你你好"; - InterpreterResult result = pysparkInterpreter.interpret("print(\""+utf8Str+"\")", context); + InterpreterResult result = pysparkInterpreter.interpret("print(\"" + utf8Str + "\")", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains(utf8Str)); - }catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); } @@ -650,7 +459,7 @@ public void run() { } @Test - public void testSparkInterpreterWithDisplayAppInfo() throws InterpreterException { + public void testSparkInterpreterWithDisplayAppInfo_StringWithoutTruncation() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -660,6 +469,7 @@ public void testSparkInterpreterWithDisplayAppInfo() throws InterpreterException properties2.put("zeppelin.livy.displayAppInfo", "true"); // enable spark ui because it is disabled by livy integration test properties2.put("livy.spark.ui.enabled", "true"); + properties2.put(LivySparkSQLInterpreter.ZEPPELIN_LIVY_SPARK_SQL_FIELD_TRUNCATE, "false"); LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties2); sparkInterpreter.setInterpreterGroup(interpreterGroup); interpreterGroup.get("session_1").add(sparkInterpreter); @@ -670,6 +480,11 @@ public void testSparkInterpreterWithDisplayAppInfo() throws InterpreterException "title", "text", authInfo, null, null, null, null, null, null, output); sparkInterpreter.open(); + LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties2); + interpreterGroup.get("session_1").add(sqlInterpreter); + sqlInterpreter.setInterpreterGroup(interpreterGroup); + sqlInterpreter.open(); + try { InterpreterResult result = sparkInterpreter.interpret("sc.version", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -683,13 +498,44 @@ public void testSparkInterpreterWithDisplayAppInfo() throws InterpreterException assertEquals(2, result.message().size()); assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType()); + // detect spark version + result = sparkInterpreter.interpret("sc.version", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(2, result.message().size()); + + boolean isSpark2 = isSpark2(sparkInterpreter, context); + + if (!isSpark2) { + result = sparkInterpreter.interpret( + "val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20))).toDF(\"col_1\", \"col_2\")\n" + + "df.collect()", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(2, result.message().size()); + assertTrue(result.message().get(0).getData() + .contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])")); + } else { + result = sparkInterpreter.interpret( + "val df=spark.createDataFrame(Seq((\"12characters12characters\",20))).toDF(\"col_1\", \"col_2\")\n" + + "df.collect()", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(2, result.message().size()); + assertTrue(result.message().get(0).getData() + .contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])")); + } + sparkInterpreter.interpret("df.registerTempTable(\"df\")", context); + // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter + result = sqlInterpreter.interpret("select * from df where col_1='12characters12characters'", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); + assertEquals("col_1\tcol_2\n12characters12characters\t20", result.message().get(0).getData()); } finally { sparkInterpreter.close(); + sqlInterpreter.close(); } } @Test - public void testSparkRInterpreter() throws LivyException, InterpreterException { + public void testSparkRInterpreter() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -847,7 +693,7 @@ public void testSharedInterpreter() throws InterpreterException { assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(1, result.message().size()); - boolean isSpark2 = isSpark2((BaseLivyInterpreter)sparkInterpreter.getInnerInterpreter(), context); + boolean isSpark2 = isSpark2((BaseLivyInterpreter) sparkInterpreter.getInnerInterpreter(), context); if (!isSpark2) { result = sparkInterpreter.interpret( @@ -891,10 +737,10 @@ public void testSharedInterpreter() throws InterpreterException { assertEquals(1, result.message().size()); assertTrue(result.message().get(0).getData() .contains("+-----+-----+\n" + - "|col_1|col_2|\n" + - "+-----+-----+\n" + - "|hello| 20|\n" + - "+-----+-----+")); + "|col_1|col_2|\n" + + "+-----+-----+\n" + + "|hello| 20|\n" + + "+-----+-----+")); // access table from sparkr result = sparkRInterpreter.interpret("head(sql(\"select * from df\"))", context); From 733a202109d9087556f24822347a3903da997a88 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 6 Mar 2018 22:33:30 +0800 Subject: [PATCH 068/386] ZEPPELIN-3299. Combine spark integration test with its unit test in one build ### What is this PR for? In ZEPPELIN-3254, we introduce 2 build for spark integration test. And we can run unit test with these system test together so that we could save 2 build in travis and also save time to set up enviroment. ### What type of PR is it? [Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3299 ### How should this be tested? * Travis CI Pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2843 from zjffdu/ZEPPELIN-3299 and squashes the following commits: dcbbb10 [Jeff Zhang] ZEPPELIN-3299. Combine spark integration test with its unit test in one build --- .travis.yml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9edb1986841..bcef80a1712 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,22 +82,17 @@ matrix: # Run ZeppelinSparkClusterTest & SparkIntegrationTest in one build would exceed the time limitation of travis, so running them separately - # Test spark interpreter with different spark versions under python2, only run ZeppelinSparkClusterTest + # Integration test of spark interpreter with different spark versions under python2, only run ZeppelinSparkClusterTest. Also run spark unit test of spark 2.2 in this build. - sudo: required jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest -DfailIfNoTests=false" + env: PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test spark interpreter with different spark versions under python3, only run SparkIntegrationTest + # Integration test of spark interpreter with different spark versions under python3, only run SparkIntegrationTest. Also run spark unit test of spark 1.6 in this build. - sudo: required jdk: "oraclejdk8" dist: trusty - env: PYTHON="3" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest -DfailIfNoTests=false" - - # Test spark module for 2.2.0 with scala 2.11 - - jdk: "oraclejdk8" - dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="3" PROFILE="-Pspark-1.6" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 2.1.0 with scala 2.11 - jdk: "openjdk7" @@ -109,11 +104,7 @@ matrix: dist: trusty env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test spark module for 1.6.3 with scala 2.10 - - jdk: "openjdk7" - dist: trusty - env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" - + # Test python/pyspark with python 2, livy 0.5 - sudo: required dist: trusty jdk: "openjdk7" From db3774640f446db9c6b00ee8ca5375405dd027be Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 7 Mar 2018 11:21:19 +0800 Subject: [PATCH 069/386] ZEPPELIN-3302. Update SparkVersion.java to support Spark 2.3 ### What is this PR for? Trivial PR to support 2.3 via updating SparkVersion.java. Unfortunately we cannot download spark 2.3 from http://d3kbcqa49mib13.cloudfront.net, will leave it in ZEPPELIN-3305 ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3302 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2846 from zjffdu/ZEPPELIN-3302-2 and squashes the following commits: 212eb68 [Jeff Zhang] ZEPPELIN-3302. Update SparkVersion.java to support Spark 2.3 --- .../src/main/java/org/apache/zeppelin/spark/SparkVersion.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java index 4b027989e01..09ea3325ac5 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java @@ -34,10 +34,10 @@ public class SparkVersion { public static final SparkVersion SPARK_1_6_0 = SparkVersion.fromVersionString("1.6.0"); public static final SparkVersion SPARK_2_0_0 = SparkVersion.fromVersionString("2.0.0"); - public static final SparkVersion SPARK_2_3_0 = SparkVersion.fromVersionString("2.3.0"); + public static final SparkVersion SPARK_2_4_0 = SparkVersion.fromVersionString("2.4.0"); public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_1_0_0; - public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_2_3_0; + public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_2_4_0; private int version; private String versionString; From a3952ea95c72543ca767d2c9dd2f4c6cab63fe1b Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 7 Mar 2018 11:54:51 +0800 Subject: [PATCH 070/386] ZEPPELIN-3304. Move sap module before zeppelin-distribution ### What is this PR for? 1. Move sap module before zeppelin-distribution, otherwise sap module will be built after zeppelin-distribution 2. Fix code style issue of sap module 3. Update travis to include sap module ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3304 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2845 from zjffdu/ZEPPELIN-3304 and squashes the following commits: 533a1b0 [Jeff Zhang] ZEPPELIN-3304. Move sap module before zeppelin-distribution --- .travis.yml | 2 +- pom.xml | 2 +- .../java/org/apache/zeppelin/sap/universe/UniverseUtil.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcef80a1712..dcd6cea8437 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ addons: env: global: # Interpreters does not required by zeppelin-server integration tests - - INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy' + - INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy,!sap' matrix: include: diff --git a/pom.xml b/pom.xml index dc6d64b89a3..d6087a871dc 100644 --- a/pom.xml +++ b/pom.xml @@ -77,11 +77,11 @@ alluxio scio neo4j + sap zeppelin-web zeppelin-server zeppelin-jupyter zeppelin-distribution - sap diff --git a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseUtil.java b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseUtil.java index dc9099d8cb8..31fbf512f9f 100644 --- a/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseUtil.java +++ b/sap/src/main/java/org/apache/zeppelin/sap/universe/UniverseUtil.java @@ -188,7 +188,8 @@ && isFilter(String.format("%s]", whereBuf.toString()), text.substring(i + 1)))) } if (buf.toString().toLowerCase().endsWith("where") || i == array.length - 1) { selectPart = false; - select.append(parseResultObj(resultObj.toString().replaceAll("(?i)wher$", ""), nodeInfos)); + select.append(parseResultObj(resultObj.toString().replaceAll("(?i)wher$", ""), + nodeInfos)); select.append(RESULT_END_TEMPLATE); continue; } From 2afcb944e2240b477387498aeacc2dabab8f2842 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 7 Mar 2018 13:50:45 +0800 Subject: [PATCH 071/386] ZEPPELIN-3303. Avoid download spark for travis build ### What is this PR for? It is not necessary to download spark distribution for several travis build. This PR would remove that to save time. ### What type of PR is it? [ Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3303 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2847 from zjffdu/ZEPPELIN-3303 and squashes the following commits: b5c4049 [Jeff Zhang] ZEPPELIN-3303. Avoid download spark for travis build --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index dcd6cea8437..24476fbfc2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ matrix: # Test License compliance using RAT tool - jdk: "openjdk7" dist: trusty - env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS="" + env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS="" # Run e2e tests (in zeppelin-web) # chrome dropped the support for precise (ubuntu 12.04), so need to use trusty @@ -66,7 +66,7 @@ matrix: - sudo: required jdk: "oraclejdk8" dist: trusty - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" + env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk8" @@ -97,12 +97,12 @@ matrix: # Test spark module for 2.1.0 with scala 2.11 - jdk: "openjdk7" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.1 -Phadoop2 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test spark module for 2.0.2 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.0 -Phadoop3 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test python/pyspark with python 2, livy 0.5 - sudo: required From c904e56a8cfd2f848ccf9612c99edf4a1037a128 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Sun, 4 Mar 2018 13:11:50 +0800 Subject: [PATCH 072/386] ZEPPELIN-3286. Run All Paragraphs stops if there is a disabled paragraph mid-run ### What is this PR for? Trivial fix that just skip the disable paragraph when run all paragraphs. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3286 ### How should this be tested? * Manually tested ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2833 from zjffdu/ZEPPELIN-3286 and squashes the following commits: 924d1fc [Jeff Zhang] ZEPPELIN-3286. Run All Paragraphs stops if there is a disabled paragraph mid-run --- .../main/java/org/apache/zeppelin/socket/NotebookServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index d14a852fbb5..0888874fcb8 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -1723,7 +1723,7 @@ private void runAllParagraphs(NotebookSocket conn, HashSet userAndRoles, Paragraph p = setParagraphUsingMessage(note, fromMessage, paragraphId, text, title, params, config); - if (!persistAndExecuteSingleParagraph(conn, note, p, true)) { + if (p.isEnabled() && !persistAndExecuteSingleParagraph(conn, note, p, true)) { // stop execution when one paragraph fails. break; } From 0267ecf76d313ba3c3ccea2d35e9b1919c7c9956 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Tue, 6 Mar 2018 12:55:35 +0530 Subject: [PATCH 073/386] [ZEPPELIN-3289] Table not using full height after paragraph height has been changed Changing paragraph height does not work correctly anymore in conjunction with table output. Refer screen shot: ![image](https://user-images.githubusercontent.com/674497/36977669-f28c77a2-20a7-11e8-854a-0b6d40973700.png) [Bug Fix] * [ZEPPELIN-3289](https://issues.apache.org/jira/browse/ZEPPELIN-3289) * [ZEPPELIN-3272](https://issues.apache.org/jira/browse/ZEPPELIN-3272) * Does the licenses files need update? N/A * Is there breaking changes for older versions? N/A * Does this needs documentation? N/A Author: Prabhjyot Singh Closes #2835 from prabhjyotsingh/ZEPPELIN-3289 and squashes the following commits: 40322606a [Prabhjyot Singh] data duplication fix d63c5e70c [Prabhjyot Singh] fix `core` undefined dd6a6d121 [Prabhjyot Singh] ZEPPELIN-3289: refresh() also updates height Change-Id: I4056f796cd34cf23af12dcf4af6ee9c3ae6067e7 --- zeppelin-web/package.json | 2 +- .../paragraph/result/result.controller.js | 55 ++++++++++--------- .../builtins/visualization-table.js | 5 ++ 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/zeppelin-web/package.json b/zeppelin-web/package.json index 5e8762cbaca..3ff642b1fcc 100644 --- a/zeppelin-web/package.json +++ b/zeppelin-web/package.json @@ -25,7 +25,7 @@ "test": "karma start karma.conf.js" }, "dependencies": { - "angular-ui-grid": "^4.0.4", + "angular-ui-grid": "^4.2.4", "angular-viewport-watch": "github:shahata/angular-viewport-watch", "ansi_up": "^2.0.2", "github-markdown-css": "2.6.0", diff --git a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js index 29465e5bd5c..75a80b2402f 100644 --- a/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/result/result.controller.js @@ -544,32 +544,35 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location }; function appendTableOutput(data) { - if (!$scope.$parent.result.data) { - $scope.$parent.result.data = []; - tableData = undefined; - } - if (!$scope.$parent.result.data[data.index]) { - $scope.$parent.result.data[data.index] = ''; - } - if (!tableData) { - $scope.$parent.result.data[data.index] = $scope.$parent.result.data[data.index].concat(data.data); - $rootScope.$broadcast( - 'updateResult', - {'data': $scope.$parent.result.data[data.index], 'type': 'TABLE'}, - undefined, - paragraph, - data.index); - let elemId = `p${$scope.id}_table`; - renderGraph(elemId, 'table', true); - } else { - let textRows = data.data.split('\n'); - for (let i = 0; i < textRows.length; i++) { - if (textRows[i] !== '') { - let row = textRows[i].split('\t'); - tableData.rows.push(row); - let builtInViz = builtInVisualizations['table']; - if (builtInViz.instance !== undefined) { - builtInViz.instance.append([row], tableData.columns); + if (ParagraphStatus.FINISHED !== paragraph.status) { + if (!$scope.$parent.result.data) { + $scope.$parent.result.data = []; + tableData = undefined; + } + if (!$scope.$parent.result.data[data.index]) { + $scope.$parent.result.data[data.index] = ''; + } + if (!tableData) { + $scope.$parent.result.data[data.index] = $scope.$parent.result.data[data.index].concat( + data.data); + $rootScope.$broadcast( + 'updateResult', + {'data': $scope.$parent.result.data[data.index], 'type': 'TABLE'}, + undefined, + paragraph, + data.index); + let elemId = `p${$scope.id}_table`; + renderGraph(elemId, 'table', true); + } else { + let textRows = data.data.split('\n'); + for (let i = 0; i < textRows.length; i++) { + if (textRows[i] !== '') { + let row = textRows[i].split('\t'); + tableData.rows.push(row); + let builtInViz = builtInVisualizations['table']; + if (builtInViz.instance !== undefined) { + builtInViz.instance.append([row], tableData.columns); + } } } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-table.js b/zeppelin-web/src/app/visualization/builtins/visualization-table.js index 723bb3aca13..5837575c3a2 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-table.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-table.js @@ -158,6 +158,11 @@ export default class TableVisualization extends Visualization { if (gridElem) { gridElem.css('height', this.targetEl.height() - 10); + const gridApiId = this.getGridApiId(); + const scope = this.getScope(); + if(scope[gridApiId]!==undefined) { + scope[gridApiId].core.handleWindowResize(); + } } } From 4fc0482385a150ab6712d7f44e00e37d16cd4bdf Mon Sep 17 00:00:00 2001 From: Savalek Date: Tue, 27 Feb 2018 14:08:18 +0300 Subject: [PATCH 074/386] [ZEPPELIN-3180] BUGFIX - save operation on interpreter add whitespace to permissions ### What is this PR for? This PR fixes a bug when unnecessary spaces are added to the permissions. ### What type of PR is it? [Bug Fix] ### What is the Jira issue? [ZEPPELIN-3180](https://issues.apache.org/jira/browse/ZEPPELIN-3180) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Savalek Closes #2749 from Savalek/ZEPPELIN-3180 and squashes the following commits: b24f1d4 [Savalek] Merge branch 'master' into ZEPPELIN-3180 b2f58fb [Savalek] Merge remote-tracking branch 'upstream/master' into ZEPPELIN-3180 84ee2cb [Savalek] Merge remote-tracking branch 'upstream/master' into ZEPPELIN-3180 28c537e [Savalek] [ZEPPELIN-3212] add test 0d4a900 [Savalek] [ZEPPELIN-3180] FIX - interpreter add whitespace to permissions --- zeppelin-web/e2e/home.spec.js | 48 +++++++++++++++++++ .../app/interpreter/interpreter.controller.js | 3 ++ 2 files changed, 51 insertions(+) diff --git a/zeppelin-web/e2e/home.spec.js b/zeppelin-web/e2e/home.spec.js index 7fc9499ed63..299fbb5adb5 100644 --- a/zeppelin-web/e2e/home.spec.js +++ b/zeppelin-web/e2e/home.spec.js @@ -1,4 +1,26 @@ describe('Home e2e Test', function() { + /*Common methods for interact with elements*/ + let clickOn = function(elem) { + browser.actions().mouseMove(elem).click().perform() + } + + let sendKeysToInput = function(input, keys) { + cleanInput(input) + input.sendKeys(keys) + } + + let cleanInput = function(inputElem) { + inputElem.sendKeys(protractor.Key.chord(protractor.Key.CONTROL, "a")) + inputElem.sendKeys(protractor.Key.BACK_SPACE) + } + + let scrollToElementAndClick = function(elem) { + browser.executeScript("arguments[0].scrollIntoView(false);", elem.getWebElement()) + browser.sleep(100) + clickOn(elem) + } + + //tests it('should have a welcome message', function() { browser.get('http://localhost:8080'); var welcomeElem = element(by.id('welcome')) @@ -15,4 +37,30 @@ describe('Home e2e Test', function() { var btn = element(by.cssContainingText('a', 'Create new note')) expect(btn.isPresent()).toBe(true) }) + + it('correct save permission in interpreter', function() { + var ownerName = 'admin' + var interpreterName = 'interpreter_e2e_test' + clickOn(element(by.xpath('//span[@class="username ng-binding"]'))) + clickOn(element(by.xpath('//a[@href="#/interpreter"]'))) + clickOn(element(by.xpath('//button[@ng-click="showAddNewSetting = !showAddNewSetting"]'))) + sendKeysToInput(element(by.xpath('//input[@id="newInterpreterSettingName"]')), interpreterName) + clickOn(element(by.xpath('//select[@ng-model="newInterpreterSetting.group"]'))) + browser.sleep(500) + browser.actions().sendKeys(protractor.Key.ARROW_DOWN).perform() + browser.actions().sendKeys(protractor.Key.ENTER).perform() + clickOn(element(by.xpath('//div[@ng-show="showAddNewSetting"]//input[@id="idShowPermission"]'))) + sendKeysToInput(element(by.xpath('//div[@ng-show="showAddNewSetting"]//input[@class="select2-search__field"]')), ownerName) + browser.sleep(500) + browser.actions().sendKeys(protractor.Key.ENTER).perform() + scrollToElementAndClick(element(by.xpath('//span[@ng-click="addNewInterpreterSetting()"]'))) + scrollToElementAndClick(element(by.xpath('//*[@id="' + interpreterName + '"]//span[@class="fa fa-pencil"]'))) + scrollToElementAndClick(element(by.xpath('//*[@id="' + interpreterName + '"]//button[@type="submit"]'))) + clickOn(element(by.xpath('//div[@class="bootstrap-dialog-footer-buttons"]//button[contains(text(), \'OK\')]'))) + browser.get('http://localhost:8080/#/interpreter'); + var text = element(by.xpath('//*[@id="' + interpreterName + '"]//li[contains(text(), \'admin\')]')).getText() + scrollToElementAndClick(element(by.xpath('//*[@id="' + interpreterName + '"]//span//span[@class="fa fa-trash"]'))) + clickOn(element(by.xpath('//div[@class="bootstrap-dialog-footer-buttons"]//button[contains(text(), \'OK\')]'))) + expect(text).toEqual(ownerName); + }) }) diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 060c6b6132b..d220dba035f 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -379,6 +379,9 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeou setting.option.remote = true; } setting.option.owners = angular.element('#' + setting.name + 'Owners').val(); + for (let i = 0; i < setting.option.owners.length; i++) { + setting.option.owners[i] = setting.option.owners[i].trim(); + } let request = { option: angular.copy(setting.option), From f4a798b6230b4f2389ad7ea9b41818b15d9ab5a7 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Fri, 9 Mar 2018 15:45:10 +0530 Subject: [PATCH 075/386] [ZEPPELIN-3306] Web e2e test fails ### What is this PR for? This is one of the flaky tests that fails with "should have a welcome message" because Z-Server did not start by the time test started executing. ### What type of PR is it? [Bug Fix] ### What is the Jira issue? * [ZEPPELIN-3306](https://issues.apache.org/jira/browse/ZEPPELIN-3306) ### How should this be tested? * CI should pass ### Questions: * Does the licenses files need update? N/A * Is there breaking changes for older versions? N/A * Does this needs documentation? N/A Author: Prabhjyot Singh Closes #2853 from prabhjyotsingh/ZEPPELIN-3306 and squashes the following commits: 38a4297 [Prabhjyot Singh] upgrade chrome=2.35 f82a00a [Prabhjyot Singh] set spawn as false for both zeppelin-web and zeppelin-integration 8742b11 [Prabhjyot Singh] ZEPPELIN-3306: Web e2e test fails --- .travis.yml | 4 ++-- zeppelin-integration/pom.xml | 2 +- zeppelin-web/package.json | 2 +- zeppelin-web/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 24476fbfc2e..e113ae30ce8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,7 +53,7 @@ matrix: sudo: false dist: trusty jdk: "oraclejdk8" - env: PYTHON="2" WEB_E2E="true" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e" + env: CI="true" WEB_E2E="true" PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e" addons: apt: packages: @@ -73,7 +73,7 @@ matrix: dist: trusty addons: firefox: "31.0" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false" + env: CI="true" PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false" # Test interpreter modules - jdk: "openjdk7" diff --git a/zeppelin-integration/pom.xml b/zeppelin-integration/pom.xml index eafa48540cf..5240a2c4a2c 100644 --- a/zeppelin-integration/pom.xml +++ b/zeppelin-integration/pom.xml @@ -163,7 +163,7 @@ + spawn="false"> diff --git a/zeppelin-web/package.json b/zeppelin-web/package.json index 3ff642b1fcc..71c691b2c41 100644 --- a/zeppelin-web/package.json +++ b/zeppelin-web/package.json @@ -19,7 +19,7 @@ "dev:watch": "grunt watch-webpack-dev", "dev": "npm-run-all --parallel dev:server lint:watch dev:watch", "test:watch": "karma start karma.conf.js --single-run=false", - "pree2e": "webdriver-manager update --gecko false --versions.chrome=2.30", + "pree2e": "webdriver-manager update --gecko false --versions.chrome=2.35", "e2e": "protractor protractor.conf.js", "pretest": "npm rebuild phantomjs-prebuilt", "test": "karma start karma.conf.js" diff --git a/zeppelin-web/pom.xml b/zeppelin-web/pom.xml index 444f8b29a4e..2a6c4328ae9 100644 --- a/zeppelin-web/pom.xml +++ b/zeppelin-web/pom.xml @@ -140,7 +140,7 @@ ${web.e2e.disabled} - + From 803116fd0d4ad5baaf3d85b7c903d75fd04b2030 Mon Sep 17 00:00:00 2001 From: gss2002 Date: Thu, 8 Mar 2018 17:06:06 -0500 Subject: [PATCH 076/386] ZEPPELIN-3309. Import/Clone user not set in Paragraph causes NPE. What is this PR for? During Import/Clone Paragraph set "user" to eliminate NPEs thrown in Helium and other functions leaving unusable notebooks. What type of PR is it? [Bug Fix] Todos What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-3309 How should this be tested? Manually tested using Import/Clone of Notebooks and attempt to adjust bound interpreters Screenshots (if appropriate) Questions: Does the licenses files need update? No Is there breaking changes for older versions? No Does this needs documentation? No Author: Greg Senia gseniaapache.org Author: gss2002 Closes #2855 from gss2002/ZEPPELIN-3309 and squashes the following commits: b8c35c9 [gss2002] ZEPPELIN-3309. Import/Clone user not set in Paragraph causes NPE. --- .../org/apache/zeppelin/notebook/Note.java | 8 ++++++- .../apache/zeppelin/notebook/Notebook.java | 4 ++-- .../zeppelin/notebook/NotebookTest.java | 23 ++++++++++++++++++- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 495f670eb55..fc70c706ab7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -362,7 +362,7 @@ public Paragraph addNewParagraph(AuthenticationInfo authenticationInfo) { * * @param srcParagraph source paragraph */ - void addCloneParagraph(Paragraph srcParagraph) { + void addCloneParagraph(Paragraph srcParagraph, AuthenticationInfo subject) { // Keep paragraph original ID final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory); @@ -371,11 +371,17 @@ void addCloneParagraph(Paragraph srcParagraph) { Map param = srcParagraph.settings.getParams(); LinkedHashMap form = srcParagraph.settings.getForms(); + logger.debug("srcParagraph user: " + srcParagraph.getUser()); + + newParagraph.setAuthenticationInfo(subject); newParagraph.setConfig(config); newParagraph.settings.setParams(param); newParagraph.settings.setForms(form); newParagraph.setText(srcParagraph.getText()); newParagraph.setTitle(srcParagraph.getTitle()); + + logger.debug("newParagraph user: " + newParagraph.getUser()); + try { Gson gson = new Gson(); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index a9168a35f21..7467e315cde 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -211,7 +211,7 @@ public Note importNote(String sourceJson, String noteName, AuthenticationInfo su newNote.setCronSupported(getConf()); List paragraphs = oldNote.getParagraphs(); for (Paragraph p : paragraphs) { - newNote.addCloneParagraph(p); + newNote.addCloneParagraph(p, subject); } notebookAuthorization.setNewNotePermissions(newNote.getId(), subject); @@ -252,7 +252,7 @@ public Note cloneNote(String sourceNoteId, String newNoteName, AuthenticationInf List paragraphs = sourceNote.getParagraphs(); for (Paragraph p : paragraphs) { - newNote.addCloneParagraph(p); + newNote.addCloneParagraph(p, subject); } noteSearchService.addIndexDoc(newNote); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index ee04291be5b..cfb2e1660ce 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -1233,7 +1233,7 @@ public void onParagraphStatusChange(Paragraph p, Status status) { Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS); assertEquals(1, onParagraphCreate.get()); - note1.addCloneParagraph(p1); + note1.addCloneParagraph(p1, AuthenticationInfo.ANONYMOUS); assertEquals(2, onParagraphCreate.get()); note1.removeParagraph(anonymous.getUser(), p1.getId()); @@ -1401,6 +1401,27 @@ public void testPublicPrivateNewNote() throws IOException, SchedulerException { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true"); ZeppelinConfiguration.create(); } + + @Test + public void testCloneImportCheck() throws IOException { + Note sourceNote = notebook.createNote(new AuthenticationInfo("user")); + sourceNote.setName("TestNote"); + + assertEquals("TestNote",sourceNote.getName()); + + Paragraph sourceParagraph = sourceNote.addNewParagraph(AuthenticationInfo.ANONYMOUS); + assertEquals("anonymous", sourceParagraph.getUser()); + + Note destNote = notebook.createNote(new AuthenticationInfo("user")); + destNote.setName("ClonedNote"); + assertEquals("ClonedNote",destNote.getName()); + + List paragraphs = sourceNote.getParagraphs(); + for (Paragraph p : paragraphs) { + destNote.addCloneParagraph(p, AuthenticationInfo.ANONYMOUS); + assertEquals("anonymous", p.getUser()); + } + } private void delete(File file){ if(file.isFile()) file.delete(); From 4463967a520af2bc9950fdbd2987ed220d8c0344 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 13 Mar 2018 14:54:58 +0800 Subject: [PATCH 077/386] ZEPPELIN-3322. Update interpreterBind when restarting zeppelin server ### What is this PR for? When loading interpreter.json, also update interpreterBindings as we change change interpreter setting id in ZEPPELIN-3208. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3322 ### How should this be tested? * Unit test is added. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2861 from zjffdu/ZEPPELIN-3322 and squashes the following commits: 94f4882 [Jeff Zhang] ZEPPELIN-3322. Update interpreterBind when restarting zeppelin server --- .../InterpreterSettingManager.java | 15 +++++++++++-- .../InterpreterSettingManagerTest.java | 6 ++++++ .../src/test/resources/conf/interpreter.json | 21 +------------------ 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index c3ec81bbb44..9587a881430 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -208,6 +208,19 @@ private void loadFromFile() throws IOException { return; } + // update interpreter binding first as we change interpreter setting id in ZEPPELIN-3208. + Map> newBindingMap = new HashMap<>(); + for (Map.Entry> entry : infoSaving.interpreterBindings.entrySet()) { + String noteId = entry.getKey(); + List oldSettingIdList = entry.getValue(); + List newSettingIdList = new ArrayList<>(); + for (String oldId : oldSettingIdList) { + newSettingIdList.add(infoSaving.interpreterSettings.get(oldId).getName()); + } + newBindingMap.put(noteId, newSettingIdList); + } + interpreterBindings.putAll(newBindingMap); + //TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving) for (InterpreterSetting savedInterpreterSetting : infoSaving.interpreterSettings.values()) { savedInterpreterSetting.setProperties(InterpreterSetting.convertInterpreterProperties( @@ -259,8 +272,6 @@ private void loadFromFile() throws IOException { interpreterSettings.put(savedInterpreterSetting.getId(), savedInterpreterSetting); } - interpreterBindings.putAll(infoSaving.interpreterBindings); - if (infoSaving.interpreterRepositories != null) { for (RemoteRepository repo : infoSaving.interpreterRepositories) { if (!dependencyResolver.getRepos().contains(repo)) { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java index a52601c40b1..10dfb03b833 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java @@ -72,6 +72,12 @@ public void testInitInterpreterSettingManager() throws IOException, RepositoryEx assertEquals(2, repositories.size()); assertEquals("central", repositories.get(0).getId()); + // verify interpreter binding + List interpreterSettingIds = interpreterSettingManager.getInterpreterBinding("2C6793KRV"); + assertEquals(2, interpreterSettingIds.size()); + assertEquals("test", interpreterSettingIds.get(0)); + assertEquals("test2", interpreterSettingIds.get(1)); + // Load it again InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf, mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); diff --git a/zeppelin-zengine/src/test/resources/conf/interpreter.json b/zeppelin-zengine/src/test/resources/conf/interpreter.json index 0c5e829c3cb..528921ca5b4 100644 --- a/zeppelin-zengine/src/test/resources/conf/interpreter.json +++ b/zeppelin-zengine/src/test/resources/conf/interpreter.json @@ -124,27 +124,8 @@ }, "interpreterBindings": { "2C6793KRV": [ - "2C48Y7FSJ", - "2C63XW4XE", - "2C66GE1VB", - "2C5VH924X", - "2C4BJDRRZ", - "2C3SQSB7V", - "2C4HKDCQW", - "2C3DR183X", - "2C66Z9XPQ", - "2C3PTPMUH", - "2C69WE69N", - "2C5SRRXHM", - "2C4ZD49PF", - "2C6V3D44K", - "2C4UB1UZA", - "2C5S1R21W", - "2C5DCRVGM", - "2C686X8ZH", "2C3RWCVAG", - "2C3JKFMJU", - "2C3VECEG2" + "2CKWE7B19" ] }, "interpreterRepositories": [ From b9d2f40bd737163be55681970b1f467f668c8fab Mon Sep 17 00:00:00 2001 From: Akhil Subhash Naik Date: Tue, 13 Mar 2018 16:10:07 +0530 Subject: [PATCH 078/386] ZEPPELIN-3099 Livy Interpreter doesn't support German Special Character - Encoding Issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What is this PR for? ZEPPELIN-3099 Livy Interpreter doesn't support German Special Character - Encoding Issue ### What type of PR is it? BUG FIX for : ZEPPELIN-3099 ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-3099 ### How should this be tested? test with German Special Characters like ß, ü, Ü, ä, Ä, ö, Ö . Author: Akhil Subhash Naik Closes #2860 from Akhilsnaik/ZEPPELIN-3099-master and squashes the following commits: eee2f5a [Akhil Subhash Naik] ZEPPELIN-3099 Implement the UT cases and fix review comments (asnaik) ecf4b54 [Akhil Subhash Naik] ZEPPELIN-3099 Implement the UT cases and fix review comments (asnaik) 7a65776 [Akhil Subhash Naik] ZEPPELIN-3099 Livy Interpreter doesn't support German Special Character - Encoding Issue (asnaik) --- .../zeppelin/livy/BaseLivyInterpreter.java | 15 ++++++++++----- .../zeppelin/livy/LivyInterpreterIT.java | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index 5fe7ce40c9b..1eba9c2885e 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -19,6 +19,7 @@ import java.io.FileInputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.security.KeyStore; import java.security.Principal; import java.util.ArrayList; @@ -52,6 +53,7 @@ import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; @@ -68,6 +70,7 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.security.kerberos.client.KerberosRestTemplate; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.HttpServerErrorException; @@ -652,19 +655,21 @@ public Principal getUserPrincipal() { } } - + RestTemplate restTemplate = null; if (isSpnegoEnabled) { if (httpClient == null) { - return new KerberosRestTemplate(keytabLocation, principal); + restTemplate = new KerberosRestTemplate(keytabLocation, principal); } else { - return new KerberosRestTemplate(keytabLocation, principal, httpClient); + restTemplate = new KerberosRestTemplate(keytabLocation, principal, httpClient); } } if (httpClient == null) { - return new RestTemplate(); + restTemplate = new RestTemplate(); } else { - return new RestTemplate(new HttpComponentsClientHttpRequestFactory(httpClient)); + restTemplate = new RestTemplate(new HttpComponentsClientHttpRequestFactory(httpClient)); } + restTemplate.getMessageConverters().add(0, new StringHttpMessageConverter(Charset.forName("UTF-8"))); + return restTemplate; } private String callRestAPI(String targetURL, String method) throws LivyException { diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 96fdbea876b..0c8bea936fc 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -371,14 +371,16 @@ public void testPySparkInterpreter() throws InterpreterException { } // test utf-8 Encoding - try { - String utf8Str = "你你你你你你好"; - InterpreterResult result = pysparkInterpreter.interpret("print(\"" + utf8Str + "\")", context); - assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertTrue(result.message().get(0).getData().contains(utf8Str)); - } catch (Exception e) { - e.printStackTrace(); - } + String utf8Str = "你你你你你你好"; + InterpreterResult reslt = pysparkInterpreter.interpret("print(\"" + utf8Str + "\")", context); + assertEquals(InterpreterResult.Code.SUCCESS, reslt.code()); + assertTrue(reslt.message().get(0).getData().contains(utf8Str)); + + //test special characters + String charStr = "açñiñíûÑoç"; + InterpreterResult res = pysparkInterpreter.interpret("print(\"" + charStr + "\")", context); + assertEquals(InterpreterResult.Code.SUCCESS, res.code()); + assertTrue(res.message().get(0).getData().contains(charStr)); try { InterpreterResult result = pysparkInterpreter.interpret("sc.version", context); From 6919c4ad32e872f3658506c74447ab353d89c89e Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Wed, 14 Feb 2018 14:38:58 +0100 Subject: [PATCH 079/386] ZEPPELIN-3159. Fixed Checkstyle errors and warnings in jdbc module ### What is this PR for? Fixed the Checkstyle errors and warnings in the jdbc module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3159 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2801 from HorizonNet/ZEPPELIN-3159 and squashes the following commits: 7771c26 [Jan Hentschel] ZEPPELIN-3159. Fixed Checkstyle errors and warnings in jdbc module --- jdbc/pom.xml | 7 + .../apache/zeppelin/jdbc/JDBCInterpreter.java | 65 ++++--- .../zeppelin/jdbc/JDBCUserConfigurations.java | 5 +- .../apache/zeppelin/jdbc/SqlCompleter.java | 58 +++--- .../jdbc/security/JDBCSecurityImpl.java | 12 +- .../zeppelin/jdbc/JDBCInterpreterTest.java | 74 ++++---- .../zeppelin/jdbc/SqlCompleterTest.java | 171 +++++++++++------- 7 files changed, 214 insertions(+), 178 deletions(-) diff --git a/jdbc/pom.xml b/jdbc/pom.xml index ba745dd632d..eef9aa84ce1 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -362,6 +362,13 @@ maven-resources-plugin + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index 0265e2d6d99..56afe6f00f4 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -14,6 +14,27 @@ */ package org.apache.zeppelin.jdbc; +import static org.apache.commons.lang.StringUtils.containsIgnoreCase; +import static org.apache.commons.lang.StringUtils.isEmpty; +import static org.apache.commons.lang.StringUtils.isNotEmpty; +import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; + +import org.apache.commons.dbcp2.ConnectionFactory; +import org.apache.commons.dbcp2.DriverManagerConnectionFactory; +import org.apache.commons.dbcp2.PoolableConnectionFactory; +import org.apache.commons.dbcp2.PoolingDriver; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.commons.lang.mutable.MutableBoolean; +import org.apache.commons.pool2.ObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.sql.Connection; @@ -33,19 +54,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.apache.commons.dbcp2.ConnectionFactory; -import org.apache.commons.dbcp2.DriverManagerConnectionFactory; -import org.apache.commons.dbcp2.PoolableConnectionFactory; -import org.apache.commons.dbcp2.PoolingDriver; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.commons.lang.mutable.MutableBoolean; -import org.apache.commons.pool2.ObjectPool; -import org.apache.commons.pool2.impl.GenericObjectPool; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.alias.CredentialProvider; -import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; @@ -58,13 +66,6 @@ import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.user.UserCredentials; import org.apache.zeppelin.user.UsernamePassword; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.commons.lang.StringUtils.containsIgnoreCase; -import static org.apache.commons.lang.StringUtils.isEmpty; -import static org.apache.commons.lang.StringUtils.isNotEmpty; -import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; /** * JDBC interpreter for Zeppelin. This interpreter can also be used for accessing HAWQ, @@ -89,7 +90,6 @@ *

    */ public class JDBCInterpreter extends KerberosInterpreter { - private Logger logger = LoggerFactory.getLogger(JDBCInterpreter.class); static final String INTERPRETER_NAME = "jdbc"; @@ -128,9 +128,10 @@ public class JDBCInterpreter extends KerberosInterpreter { static final String EMPTY_COLUMN_VALUE = ""; - private final String CONCURRENT_EXECUTION_KEY = "zeppelin.jdbc.concurrent.use"; - private final String CONCURRENT_EXECUTION_COUNT = "zeppelin.jdbc.concurrent.max_connection"; - private final String DBCP_STRING = "jdbc:apache:commons:dbcp:"; + private static final String CONCURRENT_EXECUTION_KEY = "zeppelin.jdbc.concurrent.use"; + private static final String CONCURRENT_EXECUTION_COUNT = + "zeppelin.jdbc.concurrent.max_connection"; + private static final String DBCP_STRING = "jdbc:apache:commons:dbcp:"; private final HashMap basePropretiesMap; private final HashMap jdbcUserConfigurationsMap; @@ -206,7 +207,6 @@ public void open() { setMaxLineResults(); } - protected boolean isKerboseEnabled() { if (!isEmpty(getProperty("zeppelin.jdbc.auth.type"))) { UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(properties); @@ -217,7 +217,6 @@ protected boolean isKerboseEnabled() { return false; } - private void setMaxLineResults() { if (basePropretiesMap.containsKey(COMMON_KEY) && basePropretiesMap.get(COMMON_KEY).containsKey(MAX_LINE_KEY)) { @@ -334,8 +333,7 @@ private UsernamePassword getUsernamePassword(InterpreterContext interpreterConte } public JDBCUserConfigurations getJDBCConfiguration(String user) { - JDBCUserConfigurations jdbcUserConfigurations = - jdbcUserConfigurationsMap.get(user); + JDBCUserConfigurations jdbcUserConfigurations = jdbcUserConfigurationsMap.get(user); if (jdbcUserConfigurations == null) { jdbcUserConfigurations = new JDBCUserConfigurations(); @@ -357,8 +355,7 @@ private void setUserProperty(String propertyKey, InterpreterContext interpreterC String user = interpreterContext.getAuthenticationInfo().getUser(); - JDBCUserConfigurations jdbcUserConfigurations = - getJDBCConfiguration(user); + JDBCUserConfigurations jdbcUserConfigurations = getJDBCConfiguration(user); if (basePropretiesMap.get(propertyKey).containsKey(USER_KEY) && !basePropretiesMap.get(propertyKey).getProperty(USER_KEY).isEmpty()) { String password = getPassword(basePropretiesMap.get(propertyKey)); @@ -373,7 +370,7 @@ private void setUserProperty(String propertyKey, InterpreterContext interpreterC jdbcUserConfigurations.cleanUserProperty(propertyKey); UsernamePassword usernamePassword = getUsernamePassword(interpreterContext, - getEntityName(interpreterContext.getReplName())); + getEntityName(interpreterContext.getReplName())); if (usernamePassword != null) { jdbcUserConfigurations.setUserProperty(propertyKey, usernamePassword); } else { @@ -384,10 +381,10 @@ private void setUserProperty(String propertyKey, InterpreterContext interpreterC private void createConnectionPool(String url, String user, String propertyKey, Properties properties) throws SQLException, ClassNotFoundException { ConnectionFactory connectionFactory = - new DriverManagerConnectionFactory(url, properties); + new DriverManagerConnectionFactory(url, properties); PoolableConnectionFactory poolableConnectionFactory = new PoolableConnectionFactory( - connectionFactory, null); + connectionFactory, null); ObjectPool connectionPool = new GenericObjectPool(poolableConnectionFactory); poolableConnectionFactory.setPool(connectionPool); @@ -803,7 +800,7 @@ public void cancel(InterpreterContext context) { logger.info("Cancel current query statement."); String paragraphId = context.getParagraphId(); JDBCUserConfigurations jdbcUserConfigurations = - getJDBCConfiguration(context.getAuthenticationInfo().getUser()); + getJDBCConfiguration(context.getAuthenticationInfo().getUser()); try { jdbcUserConfigurations.cancelStatement(paragraphId); } catch (SQLException e) { diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCUserConfigurations.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCUserConfigurations.java index 057938035bb..4eac9fc739f 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCUserConfigurations.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCUserConfigurations.java @@ -15,15 +15,15 @@ package org.apache.zeppelin.jdbc; import org.apache.commons.dbcp2.PoolingDriver; -import org.apache.zeppelin.user.UsernamePassword; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import java.util.Properties; +import org.apache.zeppelin.user.UsernamePassword; + /** * UserConfigurations for JDBC impersonation. */ @@ -106,5 +106,4 @@ public boolean isConnectionInDBDriverPoolSuccessful(String key) { } return false; } - } diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java index 6103fc7bc16..9f52ecba491 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java @@ -4,6 +4,11 @@ * This source file is based on code taken from SQLLine 1.0.2 See SQLLine notice in LICENSE */ +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.NumberUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -22,28 +27,22 @@ import java.util.TreeSet; import java.util.regex.Pattern; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.NumberUtils; +import jline.console.completer.ArgumentCompleter.ArgumentList; +import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter; + import org.apache.zeppelin.completer.CachedCompleter; import org.apache.zeppelin.completer.CompletionType; import org.apache.zeppelin.completer.StringsCompleter; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import jline.console.completer.ArgumentCompleter.ArgumentList; -import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter; /** * SQL auto complete functionality for the JdbcInterpreter. */ public class SqlCompleter { - private static Logger logger = LoggerFactory.getLogger(SqlCompleter.class); - /** - * Delimiter that can split SQL statement in keyword list + * Delimiter that can split SQL statement in keyword list. */ private WhitespaceArgumentDelimiter sqlDelimiter = new WhitespaceArgumentDelimiter() { @@ -57,35 +56,33 @@ public boolean isDelimiterChar(CharSequence buffer, int pos) { }; /** - * Schema completer + * Schema completer. */ private CachedCompleter schemasCompleter; /** - * Contain different completer with table list for every schema name + * Contain different completer with table list for every schema name. */ private Map tablesCompleters = new HashMap<>(); /** * Contains different completer with column list for every table name - * Table names store as schema_name.table_name + * Table names store as schema_name.table_name. */ private Map columnsCompleters = new HashMap<>(); /** - * Completer for sql keywords + * Completer for sql keywords. */ private CachedCompleter keywordCompleter; private int ttlInSeconds; - public SqlCompleter(int ttlInSeconds) { this.ttlInSeconds = ttlInSeconds; } public int complete(String buffer, int cursor, List candidates) { - logger.debug("Complete with buffer = " + buffer + ", cursor = " + cursor); // The delimiter breaks the buffer into separate words (arguments), separated by the @@ -110,7 +107,7 @@ public int complete(String buffer, int cursor, List candi } /** - * Return list of schema names within the database + * Return list of schema names within the database. * * @param meta metadata from connection to database * @param schemaFilters a schema name patterns; must match the schema name @@ -146,7 +143,7 @@ private static Set getSchemaNames(DatabaseMetaData meta, List sc } /** - * Return list of catalog names within the database + * Return list of catalog names within the database. * * @param meta metadata from connection to database * @param schemaFilters a catalog name patterns; must match the catalog name @@ -177,7 +174,6 @@ private static Set getCatalogNames(DatabaseMetaData meta, List s return res; } - private static void fillTableNames(String schema, DatabaseMetaData meta, Set tables) { try (ResultSet tbls = meta.getTables(schema, schema, "%", new String[]{"TABLE", "VIEW", "ALIAS", "SYNONYM", "GLOBAL TEMPORARY", "LOCAL TEMPORARY"})) { @@ -191,7 +187,7 @@ private static void fillTableNames(String schema, DatabaseMetaData meta, Set getSqlKeywordsCompletions(DatabaseMetaData meta) throws IOException, SQLException { - // Add the default SQL completions String keywords = new BufferedReader(new InputStreamReader( @@ -222,7 +217,6 @@ public static Set getSqlKeywordsCompletions(DatabaseMetaData meta) throw Set completions = new TreeSet<>(); if (null != meta) { - // Add the driver specific SQL completions String driverSpecificKeywords = "/" + meta.getDriverName().replace(" ", "-").toLowerCase() + "-sql.keywords"; @@ -240,7 +234,6 @@ public static Set getSqlKeywordsCompletions(DatabaseMetaData meta) throw driverSpecificKeywords + " : " + e, e); } - // Add the keywords from the current JDBC connection try { keywords += "," + meta.getSQLKeywords(); @@ -270,7 +263,6 @@ public static Set getSqlKeywordsCompletions(DatabaseMetaData meta) throw // Set all keywords to lower-case versions keywords = keywords.toLowerCase(); - } StringTokenizer tok = new StringTokenizer(keywords, ", "); @@ -282,7 +274,7 @@ public static Set getSqlKeywordsCompletions(DatabaseMetaData meta) throw } /** - * Initializes all local completers from database connection + * Initializes all local completers from database connection. * * @param connection database connection * @param schemaFiltersString a comma separated schema name patterns, supports '%' symbol; @@ -350,8 +342,6 @@ public void createOrUpdateFromConnection(Connection connection, String schemaFil } } - - public void initKeywords(Set keywords) { if (keywords != null && !keywords.isEmpty()) { keywordCompleter = new CachedCompleter(new StringsCompleter(keywords), 0); @@ -380,7 +370,7 @@ public void initColumns(String schemaTable, Set columns) { } /** - * Find aliases in sql command + * Find aliases in sql command. * * @param sqlArguments sql command divided on arguments * @return for every alias contains table name in format schema_name.table_name @@ -397,7 +387,7 @@ public Map findAliasesInSQL(String[] sqlArguments) { } /** - * Complete buffer in case it is a keyword + * Complete buffer in case it is a keyword. * * @return -1 in case of no candidates found, 0 otherwise */ @@ -406,7 +396,7 @@ private int completeKeyword(String buffer, int cursor, List candid } /** - * Complete buffer in case it is a schema name + * Complete buffer in case it is a schema name. * * @return -1 in case of no candidates found, 0 otherwise */ @@ -415,22 +405,22 @@ private int completeSchema(String buffer, int cursor, List candida } /** - * Complete buffer in case it is a table name + * Complete buffer in case it is a table name. * * @return -1 in case of no candidates found, 0 otherwise */ private int completeTable(String schema, String buffer, int cursor, List candidates) { // Wrong schema - if (schema == null || !tablesCompleters.containsKey(schema)) + if (schema == null || !tablesCompleters.containsKey(schema)) { return -1; - else { + } else { return tablesCompleters.get(schema).getCompleter().complete(buffer, cursor, candidates); } } /** - * Complete buffer in case it is a column name + * Complete buffer in case it is a column name. * * @return -1 in case of no candidates found, 0 otherwise */ diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/security/JDBCSecurityImpl.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/security/JDBCSecurityImpl.java index 6e9470f4edc..ef2c729abd0 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/security/JDBCSecurityImpl.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/security/JDBCSecurityImpl.java @@ -16,24 +16,23 @@ */ package org.apache.zeppelin.jdbc.security; +import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; +import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.SIMPLE; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Properties; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; -import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; -import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.SIMPLE; - /** * Created for org.apache.zeppelin.jdbc.security on 09/07/16. */ public class JDBCSecurityImpl { - - private static Logger LOGGER = LoggerFactory.getLogger(JDBCSecurityImpl.class); + private static final Logger LOGGER = LoggerFactory.getLogger(JDBCSecurityImpl.class); /*** * @param properties @@ -70,5 +69,4 @@ public static AuthenticationMethod getAuthtype(Properties properties) { } return authType; } - } diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java index 64412670ef9..a6e394f9d4b 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java @@ -14,15 +14,23 @@ */ package org.apache.zeppelin.jdbc; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + import static java.lang.String.format; + +import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE; import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_DRIVER; import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD; -import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER; -import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL; import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PRECODE; +import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL; +import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER; import static org.apache.zeppelin.jdbc.JDBCInterpreter.PRECODE_KEY_TEMPLATE; -import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE; -import static org.junit.Assert.*; + +import org.junit.Before; +import org.junit.Test; import java.io.IOException; import java.nio.file.Files; @@ -35,6 +43,8 @@ import java.util.List; import java.util.Properties; +import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter; + import org.apache.zeppelin.completer.CompletionType; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -46,21 +56,16 @@ import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.UserCredentials; import org.apache.zeppelin.user.UsernamePassword; -import org.junit.Before; -import org.junit.Test; - -import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter; /** - * JDBC interpreter unit tests + * JDBC interpreter unit tests. */ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter { - static String jdbcConnection; InterpreterContext interpreterContext; private static String getJdbcConnection() throws IOException { - if(null == jdbcConnection) { + if (null == jdbcConnection) { Path tmpDir = Files.createTempDirectory("h2-test-"); tmpDir.toFile().deleteOnExit(); jdbcConnection = format("jdbc:h2:%s", tmpDir); @@ -88,11 +93,12 @@ public void setUp() throws Exception { "DROP TABLE IF EXISTS test_table; " + "CREATE TABLE test_table(id varchar(255), name varchar(255));"); - PreparedStatement insertStatement = connection.prepareStatement("insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);"); + PreparedStatement insertStatement = connection.prepareStatement( + "insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);"); insertStatement.setString(1, null); insertStatement.execute(); interpreterContext = new InterpreterContext("", "1", null, "", "", - new AuthenticationInfo("testUser"), null, null, null,null, null, null, null); + new AuthenticationInfo("testUser"), null, null, null, null, null, null, null); } @@ -110,8 +116,8 @@ public void testForParsePropertyKey() throws IOException { "default"); // when you use a %jdbc(prefix1), prefix1 is the propertyKey as form part of the cmd string - assertEquals(t.getPropertyKey("(prefix1)\n select max(cant) from test_table where id >= 2452640"), - "prefix1"); + assertEquals(t.getPropertyKey( + "(prefix1)\n select max(cant) from test_table where id >= 2452640"), "prefix1"); assertEquals(t.getPropertyKey("(prefix2) select max(cant) from test_table where id >= 2452640"), "prefix2"); @@ -201,7 +207,7 @@ public void testSplitSqlQuery() throws SQLException, IOException { } @Test - public void testQueryWithEsсapedCharacters() throws SQLException, IOException { + public void testQueryWithEscapedCharacters() throws SQLException, IOException { String sqlQuery = "select '\\n', ';';" + "select replace('A\\;B', '\\', 'text');" + "select '\\', ';';" + @@ -229,7 +235,6 @@ public void testSplitSqlQuery() throws SQLException, IOException { assertEquals("'Atext;B'\nAtext;B\n", interpreterResult.message().get(1).getData()); assertEquals("'\\'\t';'\n\\\t;\n", interpreterResult.message().get(2).getData()); assertEquals("''''\t';'\n'\t;\n", interpreterResult.message().get(3).getData()); - } @Test @@ -252,7 +257,8 @@ public void testSelectMultipleQueries() throws SQLException, IOException { assertEquals(2, interpreterResult.message().size()); assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType()); - assertEquals("ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n", interpreterResult.message().get(0).getData()); + assertEquals("ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n", + interpreterResult.message().get(0).getData()); assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(1).getType()); assertEquals("ID\tNAME\n", interpreterResult.message().get(1).getData()); @@ -277,7 +283,8 @@ public void testDefaultSplitQuries() throws SQLException, IOException { assertEquals(1, interpreterResult.message().size()); assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType()); - assertEquals("ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n", interpreterResult.message().get(0).getData()); + assertEquals("ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n", + interpreterResult.message().get(0).getData()); } @Test @@ -304,7 +311,6 @@ public void testSelectQueryWithNull() throws SQLException, IOException { @Test public void testSelectQueryMaxResult() throws SQLException, IOException { - Properties properties = new Properties(); properties.setProperty("common.max_count", "1"); properties.setProperty("common.max_retry", "3"); @@ -363,9 +369,11 @@ public void testAutoCompletion() throws SQLException, IOException, InterpreterEx jdbcInterpreter.interpret("", interpreterContext); - List completionList = jdbcInterpreter.completion("sel", 3, interpreterContext); + List completionList = jdbcInterpreter.completion("sel", 3, + interpreterContext); - InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select", "select", CompletionType.keyword.name()); + InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select", "select", + CompletionType.keyword.name()); assertEquals(1, completionList.size()); assertEquals(true, completionList.contains(correctCompletionKeyword)); @@ -386,7 +394,8 @@ private Properties getDBProperty(String dbUser, String dbPassowrd) throws IOExce return properties; } - private AuthenticationInfo getUserAuth(String user, String entityName, String dbUser, String dbPassword){ + private AuthenticationInfo getUserAuth(String user, String entityName, String dbUser, + String dbPassword) { UserCredentials userCredentials = new UserCredentials(); if (entityName != null && dbUser != null && dbPassword != null) { UsernamePassword up = new UsernamePassword(dbUser, dbPassword); @@ -400,8 +409,7 @@ private AuthenticationInfo getUserAuth(String user, String entityName, String db @Test public void testMultiTenant() throws SQLException, IOException { - - /** + /* * assume that the database user is 'dbuser' and password is 'dbpassword' * 'jdbc1' interpreter has user('dbuser')/password('dbpassword') property * 'jdbc2' interpreter doesn't have user/password property @@ -413,12 +421,12 @@ public void testMultiTenant() throws SQLException, IOException { JDBCInterpreter jdbc2 = new JDBCInterpreter(getDBProperty("", "")); AuthenticationInfo user1Credential = getUserAuth("user1", null, null, null); - AuthenticationInfo user2Credential = getUserAuth("user2", "jdbc.jdbc2", "user2Id","user2Pw"); + AuthenticationInfo user2Credential = getUserAuth("user2", "jdbc.jdbc2", "user2Id", "user2Pw"); // user1 runs jdbc1 jdbc1.open(); InterpreterContext ctx1 = new InterpreterContext("", "1", "jdbc1", "", "", user1Credential, - null, null, null, null, null, null, null); + null, null, null, null, null, null, null); jdbc1.interpret("", ctx1); JDBCUserConfigurations user1JDBC1Conf = jdbc1.getJDBCConfiguration("user1"); @@ -429,7 +437,7 @@ public void testMultiTenant() throws SQLException, IOException { // user1 runs jdbc2 jdbc2.open(); InterpreterContext ctx2 = new InterpreterContext("", "1", "jdbc2", "", "", user1Credential, - null, null, null, null, null, null, null); + null, null, null, null, null, null, null); jdbc2.interpret("", ctx2); JDBCUserConfigurations user1JDBC2Conf = jdbc2.getJDBCConfiguration("user1"); @@ -440,7 +448,7 @@ public void testMultiTenant() throws SQLException, IOException { // user2 runs jdbc1 jdbc1.open(); InterpreterContext ctx3 = new InterpreterContext("", "1", "jdbc1", "", "", user2Credential, - null, null, null, null, null, null, null); + null, null, null, null, null, null, null); jdbc1.interpret("", ctx3); JDBCUserConfigurations user2JDBC1Conf = jdbc1.getJDBCConfiguration("user2"); @@ -451,7 +459,7 @@ public void testMultiTenant() throws SQLException, IOException { // user2 runs jdbc2 jdbc2.open(); InterpreterContext ctx4 = new InterpreterContext("", "1", "jdbc2", "", "", user2Credential, - null, null, null, null, null, null, null); + null, null, null, null, null, null, null); jdbc2.interpret("", ctx4); JDBCUserConfigurations user2JDBC2Conf = jdbc2.getJDBCConfiguration("user2"); @@ -467,7 +475,8 @@ public void testPrecode() throws SQLException, IOException { properties.setProperty("default.url", getJdbcConnection()); properties.setProperty("default.user", ""); properties.setProperty("default.password", ""); - properties.setProperty(DEFAULT_PRECODE, "create table test_precode (id int); insert into test_precode values (1);"); + properties.setProperty(DEFAULT_PRECODE, + "create table test_precode (id int); insert into test_precode values (1);"); JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties); jdbcInterpreter.open(); jdbcInterpreter.executePrecode(interpreterContext); @@ -509,7 +518,8 @@ public void testPrecodeWithAnotherPrefix() throws SQLException, IOException { properties.setProperty("anotherPrefix.url", getJdbcConnection()); properties.setProperty("anotherPrefix.user", ""); properties.setProperty("anotherPrefix.password", ""); - properties.setProperty(String.format(PRECODE_KEY_TEMPLATE, "anotherPrefix"), "create table test_precode_2 (id int); insert into test_precode_2 values (2);"); + properties.setProperty(String.format(PRECODE_KEY_TEMPLATE, "anotherPrefix"), + "create table test_precode_2 (id int); insert into test_precode_2 values (2);"); JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties); jdbcInterpreter.open(); jdbcInterpreter.executePrecode(interpreterContext); diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java index 19150ccebc9..1ec3ae4a41a 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java @@ -14,6 +14,20 @@ */ package org.apache.zeppelin.jdbc; +import static com.google.common.collect.Sets.newHashSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.common.base.Joiner; + +import org.apache.commons.lang.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; @@ -23,29 +37,16 @@ import java.util.Map; import java.util.Set; -import org.apache.commons.lang.StringUtils; -import org.apache.zeppelin.completer.CompletionType; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Joiner; import jline.console.completer.ArgumentCompleter; -import static com.google.common.collect.Sets.newHashSet; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.zeppelin.completer.CompletionType; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; /** - * SQL completer unit tests + * SQL completer unit tests. */ public class SqlCompleterTest { - public class CompleterTester { - private SqlCompleter completer; private String buffer; @@ -133,7 +134,7 @@ private String explain(String buffer, int cursor, List ca private Logger logger = LoggerFactory.getLogger(SqlCompleterTest.class); - private final static Set EMPTY = new HashSet<>(); + private static final Set EMPTY = new HashSet<>(); private CompleterTester tester; @@ -162,35 +163,35 @@ public void beforeTest() throws IOException, SQLException { sqlCompleter.initSchemas(schemas); - Set prod_dds_tables = new HashSet<>(); - prod_dds_tables.add("financial_account"); - prod_dds_tables.add("customer"); + Set prodDdsTables = new HashSet<>(); + prodDdsTables.add("financial_account"); + prodDdsTables.add("customer"); - sqlCompleter.initTables("prod_dds", prod_dds_tables); + sqlCompleter.initTables("prod_dds", prodDdsTables); - Set prod_emart_tables = new HashSet<>(); - prod_emart_tables.add("financial_account"); + Set prodEmartTables = new HashSet<>(); + prodEmartTables.add("financial_account"); - sqlCompleter.initTables("prod_emart", prod_emart_tables); + sqlCompleter.initTables("prod_emart", prodEmartTables); - Set prod_dds_financial_account_columns = new HashSet<>(); - prod_dds_financial_account_columns.add("account_rk"); - prod_dds_financial_account_columns.add("account_id"); + Set prodDdsFinancialAccountColumns = new HashSet<>(); + prodDdsFinancialAccountColumns.add("account_rk"); + prodDdsFinancialAccountColumns.add("account_id"); - sqlCompleter.initColumns("prod_dds.financial_account", prod_dds_financial_account_columns); + sqlCompleter.initColumns("prod_dds.financial_account", prodDdsFinancialAccountColumns); - Set prod_dds_customer_columns = new HashSet<>(); - prod_dds_customer_columns.add("customer_rk"); - prod_dds_customer_columns.add("name"); - prod_dds_customer_columns.add("birth_dt"); + Set prodDdsCustomerColumns = new HashSet<>(); + prodDdsCustomerColumns.add("customer_rk"); + prodDdsCustomerColumns.add("name"); + prodDdsCustomerColumns.add("birth_dt"); - sqlCompleter.initColumns("prod_dds.customer", prod_dds_customer_columns); + sqlCompleter.initColumns("prod_dds.customer", prodDdsCustomerColumns); - Set prod_emart_financial_account_columns = new HashSet<>(); - prod_emart_financial_account_columns.add("account_rk"); - prod_emart_financial_account_columns.add("balance_amt"); + Set prodEmartFinancialAccountColumns = new HashSet<>(); + prodEmartFinancialAccountColumns.add("account_rk"); + prodEmartFinancialAccountColumns.add("balance_amt"); - sqlCompleter.initColumns("prod_emart.financial_account", prod_emart_financial_account_columns); + sqlCompleter.initColumns("prod_emart.financial_account", prodEmartFinancialAccountColumns); tester = new CompleterTester(sqlCompleter); } @@ -198,7 +199,8 @@ public void beforeTest() throws IOException, SQLException { @Test public void testFindAliasesInSQL_Simple() { String sql = "select * from prod_emart.financial_account a"; - Map res = sqlCompleter.findAliasesInSQL(delimiter.delimit(sql, 0).getArguments()); + Map res = sqlCompleter.findAliasesInSQL( + delimiter.delimit(sql, 0).getArguments()); assertEquals(1, res.size()); assertTrue(res.get("a").equals("prod_emart.financial_account")); } @@ -206,7 +208,8 @@ public void testFindAliasesInSQL_Simple() { @Test public void testFindAliasesInSQL_Two() { String sql = "select * from prod_dds.financial_account a, prod_dds.customer b"; - Map res = sqlCompleter.findAliasesInSQL(sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments()); + Map res = sqlCompleter.findAliasesInSQL( + sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments()); assertEquals(2, res.size()); assertTrue(res.get("a").equals("prod_dds.financial_account")); assertTrue(res.get("b").equals("prod_dds.customer")); @@ -215,7 +218,8 @@ public void testFindAliasesInSQL_Two() { @Test public void testFindAliasesInSQL_WrongTables() { String sql = "select * from prod_ddsxx.financial_account a, prod_dds.customerxx b"; - Map res = sqlCompleter.findAliasesInSQL(sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments()); + Map res = sqlCompleter.findAliasesInSQL( + sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments()); assertEquals(0, res.size()); } @@ -227,15 +231,24 @@ public void testCompleteName_Empty() { Map aliases = new HashMap<>(); sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(9, candidates.size()); - assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", + CompletionType.schema.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", + CompletionType.schema.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("SUM", "SUM", + CompletionType.keyword.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("SUBSTRING", "SUBSTRING", + CompletionType.keyword.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", + CompletionType.keyword.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("SELECT", "SELECT", + CompletionType.keyword.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("ORDER", "ORDER", + CompletionType.keyword.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("LIMIT", "LIMIT", + CompletionType.keyword.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("FROM", "FROM", + CompletionType.keyword.name()))); } @Test @@ -246,8 +259,10 @@ public void testCompleteName_SimpleSchema() { Map aliases = new HashMap<>(); sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(2, candidates.size()); - assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", + CompletionType.schema.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", + CompletionType.schema.name()))); } @Test @@ -258,7 +273,9 @@ public void testCompleteName_SimpleTable() { Map aliases = new HashMap<>(); sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(1, candidates.size()); - assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()))); + assertTrue(candidates.contains( + new InterpreterCompletion("financial_account", "financial_account", + CompletionType.table.name()))); } @Test @@ -269,8 +286,10 @@ public void testCompleteName_SimpleColumn() { Map aliases = new HashMap<>(); sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(2, candidates.size()); - assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", + CompletionType.column.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", + CompletionType.column.name()))); } @Test @@ -282,8 +301,10 @@ public void testCompleteName_WithAlias() { aliases.put("a", "prod_dds.financial_account"); sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(2, candidates.size()); - assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", + CompletionType.column.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", + CompletionType.column.name()))); } @Test @@ -295,24 +316,32 @@ public void testCompleteName_WithAliasAndPoint() { aliases.put("a", "prod_dds.financial_account"); sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(2, candidates.size()); - assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", + CompletionType.column.name()))); + assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", + CompletionType.column.name()))); } @Test public void testSchemaAndTable() { String buffer = "select * from prod_emart.fi"; - tester.buffer(buffer).from(20).to(23).expect(newHashSet(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()))).test(); - tester.buffer(buffer).from(25).to(27).expect(newHashSet(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()))).test(); + tester.buffer(buffer).from(20).to(23).expect(newHashSet( + new InterpreterCompletion("prod_emart", "prod_emart", + CompletionType.schema.name()))).test(); + tester.buffer(buffer).from(25).to(27).expect(newHashSet( + new InterpreterCompletion("financial_account", "financial_account", + CompletionType.table.name()))).test(); } @Test public void testEdges() { String buffer = " ORDER "; - tester.buffer(buffer).from(3).to(7).expect(newHashSet(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(3).to(7).expect(newHashSet( + new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()))).test(); tester.buffer(buffer).from(0).to(1).expect(newHashSet( new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()), - new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()), + new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", + CompletionType.keyword.name()), new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()), new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()), new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()), @@ -326,23 +355,29 @@ public void testEdges() { @Test public void testMultipleWords() { String buffer = "SELE FRO LIM"; - tester.buffer(buffer).from(2).to(4).expect(newHashSet(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test(); - tester.buffer(buffer).from(6).to(8).expect(newHashSet(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test(); - tester.buffer(buffer).from(10).to(12).expect(newHashSet(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(2).to(4).expect(newHashSet( + new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(6).to(8).expect(newHashSet( + new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(10).to(12).expect(newHashSet( + new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()))).test(); } @Test public void testMultiLineBuffer() { String buffer = " \n SELE\nFRO"; - tester.buffer(buffer).from(5).to(7).expect(newHashSet(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test(); - tester.buffer(buffer).from(9).to(11).expect(newHashSet(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(5).to(7).expect(newHashSet( + new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(9).to(11).expect(newHashSet( + new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test(); } @Test public void testMultipleCompletionSuggestions() { String buffer = "SU"; tester.buffer(buffer).from(2).to(2).expect(newHashSet( - new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()), + new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", + CompletionType.keyword.name()), new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()), new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name())) ).test(); From 5e37eed1270b1aa4a2ed9f540fee16fad29127b4 Mon Sep 17 00:00:00 2001 From: Jan Hentschel Date: Tue, 13 Feb 2018 14:59:53 +0100 Subject: [PATCH 080/386] ZEPPELIN-3158. Fixed Checkstyle errors and warnings in ignite module ### What is this PR for? Fixed all Checkstyle errors and warnings in the ignite module. ### What type of PR is it? Improvement ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3158 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jan Hentschel Closes #2797 from HorizonNet/ZEPPELIN-3158 and squashes the following commits: a4bb15b [Jan Hentschel] ZEPPELIN-3158. Fixed Checkstyle errors and warnings in ignite module --- ignite/pom.xml | 7 ++++++ .../zeppelin/ignite/IgniteInterpreter.java | 17 +++++++------ .../zeppelin/ignite/IgniteSqlInterpreter.java | 20 ++++++++-------- .../ignite/IgniteInterpreterTest.java | 23 ++++++++++-------- .../ignite/IgniteSqlInterpreterTest.java | 24 +++++++++++-------- .../org/apache/zeppelin/ignite/Person.java | 4 ++-- 6 files changed, 56 insertions(+), 39 deletions(-) diff --git a/ignite/pom.xml b/ignite/pom.xml index cd778b8c11a..7d7b3db948c 100644 --- a/ignite/pom.xml +++ b/ignite/pom.xml @@ -114,6 +114,13 @@ maven-resources-plugin + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + diff --git a/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java b/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java index ac385ea55c8..e1040399466 100644 --- a/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java +++ b/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java @@ -21,11 +21,6 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,13 +31,12 @@ import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Properties; import scala.Console; -import scala.None; import scala.Some; import scala.collection.JavaConversions; import scala.tools.nsc.Settings; @@ -51,6 +45,15 @@ import scala.tools.nsc.settings.MutableSettings.BooleanSetting; import scala.tools.nsc.settings.MutableSettings.PathSetting; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterUtils; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; + /** * Apache Ignite interpreter (http://ignite.incubator.apache.org/). * diff --git a/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java b/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java index 6af8eb549e5..100a733c7ba 100644 --- a/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java +++ b/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java @@ -16,15 +16,6 @@ */ package org.apache.zeppelin.ignite; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,12 +23,21 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.ResultSetMetaData; -import java.sql.Statement; import java.sql.SQLException; +import java.sql.Statement; import java.util.LinkedList; import java.util.List; import java.util.Properties; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; + /** * Apache Ignite SQL interpreter (http://ignite.incubator.apache.org/). * diff --git a/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java b/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java index e8f226588a3..1f6c49c4015 100644 --- a/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java +++ b/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java @@ -16,22 +16,23 @@ */ package org.apache.zeppelin.ignite; -import java.util.Collections; -import java.util.Properties; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import org.apache.ignite.Ignite; import org.apache.ignite.Ignition; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterResult; import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import java.util.Collections; +import java.util.Properties; + +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; /** * Tests for Apache Ignite interpreter ({@link IgniteInterpreter}). @@ -40,7 +41,8 @@ public class IgniteInterpreterTest { private static final String HOST = "127.0.0.1:47500..47509"; private static final InterpreterContext INTP_CONTEXT = - new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null, null, null); + new InterpreterContext(null, null, null, null, null, null, null, null, null, null, + null, null, null); private IgniteInterpreter intp; private Ignite ignite; @@ -61,7 +63,8 @@ public void setUp() { ignite = Ignition.start(cfg); Properties props = new Properties(); - props.setProperty(IgniteSqlInterpreter.IGNITE_JDBC_URL, "jdbc:ignite:cfg://cache=person@default-ignite-jdbc.xml"); + props.setProperty(IgniteSqlInterpreter.IGNITE_JDBC_URL, + "jdbc:ignite:cfg://cache=person@default-ignite-jdbc.xml"); props.setProperty(IgniteInterpreter.IGNITE_CLIENT_MODE, "false"); props.setProperty(IgniteInterpreter.IGNITE_PEER_CLASS_LOADING_ENABLED, "false"); props.setProperty(IgniteInterpreter.IGNITE_ADDRESSES, HOST); @@ -84,7 +87,8 @@ public void testInterpret() { "val " + sizeVal + " = ignite.cluster().nodes().size()", INTP_CONTEXT); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertTrue(result.message().get(0).getData().contains(sizeVal + ": Int = " + ignite.cluster().nodes().size())); + assertTrue(result.message().get(0).getData().contains(sizeVal + ": Int = " + + ignite.cluster().nodes().size())); result = intp.interpret("\"123\"\n .toInt", INTP_CONTEXT); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -96,5 +100,4 @@ public void testInterpretInvalidInput() { assertEquals(InterpreterResult.Code.ERROR, result.code()); } - } diff --git a/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java b/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java index 4cb2cbb0e2d..5006fa3805e 100644 --- a/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java +++ b/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java @@ -16,8 +16,7 @@ */ package org.apache.zeppelin.ignite; -import java.util.Collections; -import java.util.Properties; +import static org.junit.Assert.assertEquals; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -26,16 +25,18 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.Properties; + import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; /** * Tests for Apache Ignite SQL interpreter ({@link IgniteSqlInterpreter}). @@ -44,7 +45,8 @@ public class IgniteSqlInterpreterTest { private static final String HOST = "127.0.0.1:47500..47509"; private static final InterpreterContext INTP_CONTEXT = - new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null, null, null); + new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null, + null, null); private Ignite ignite; private IgniteSqlInterpreter intp; @@ -66,7 +68,8 @@ public void setUp() { ignite = Ignition.start(cfg); Properties props = new Properties(); - props.setProperty(IgniteSqlInterpreter.IGNITE_JDBC_URL, "jdbc:ignite:cfg://cache=person@default-ignite-jdbc.xml"); + props.setProperty(IgniteSqlInterpreter.IGNITE_JDBC_URL, + "jdbc:ignite:cfg://cache=person@default-ignite-jdbc.xml"); intp = new IgniteSqlInterpreter(props); @@ -90,7 +93,8 @@ public void tearDown() throws InterpreterException { @Test public void testSql() { - InterpreterResult result = intp.interpret("select name, age from person where age > 10", INTP_CONTEXT); + InterpreterResult result = intp.interpret("select name, age from person where age > 10", + INTP_CONTEXT); assertEquals(Code.SUCCESS, result.code()); assertEquals(Type.TABLE, result.message().get(0).getType()); diff --git a/ignite/src/test/java/org/apache/zeppelin/ignite/Person.java b/ignite/src/test/java/org/apache/zeppelin/ignite/Person.java index e0b43674879..e8bef93c37e 100644 --- a/ignite/src/test/java/org/apache/zeppelin/ignite/Person.java +++ b/ignite/src/test/java/org/apache/zeppelin/ignite/Person.java @@ -16,10 +16,10 @@ */ package org.apache.zeppelin.ignite; -import java.io.Serializable; - import org.apache.ignite.cache.query.annotations.QuerySqlField; +import java.io.Serializable; + public class Person implements Serializable { @QuerySqlField private String name; From 514ba659bbe8706bccf539b09712362377bfd588 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 14 Mar 2018 10:53:28 +0800 Subject: [PATCH 081/386] [HOTFIX] More proper error message when interpreter is not binded ### What is this PR for? Minor fix for more proper error message when interpreter is not binded ### What type of PR is it? [Bug Fix | Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * NO jira created ### How should this be tested? * CI pass ### Screenshots (if appropriate) Before ![screen shot 2018-03-14 at 10 13 50 am](https://user-images.githubusercontent.com/164491/37381029-1e27ac9a-2776-11e8-9c2d-1c696aa07807.png) After ![screen shot 2018-03-14 at 10 52 24 am](https://user-images.githubusercontent.com/164491/37381033-20926466-2776-11e8-823d-211160bbcf76.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2865 from zjffdu/HOTFIX_INTP and squashes the following commits: 0a22913 [Jeff Zhang] [HOTFIX] More proper error message when interpreter is not binded --- .../apache/zeppelin/interpreter/InterpreterSetting.java | 5 +++-- .../java/org/apache/zeppelin/notebook/Paragraph.java | 9 +++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index ff4626d13a1..a3b7f8dd0cc 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -677,8 +677,9 @@ public void clearNoteIdAndParaMap() { List createInterpreters(String user, String interpreterGroupId, String sessionId) { List interpreters = new ArrayList<>(); List interpreterInfos = getInterpreterInfos(); + Properties intpProperties = getJavaProperties(); for (InterpreterInfo info : interpreterInfos) { - Interpreter interpreter = new RemoteInterpreter(getJavaProperties(), sessionId, + Interpreter interpreter = new RemoteInterpreter(intpProperties, sessionId, info.getClassName(), user, lifecycleManager); if (info.isDefaultInterpreter()) { interpreters.add(0, interpreter); @@ -688,7 +689,7 @@ List createInterpreters(String user, String interpreterGroupId, Str LOGGER.info("Interpreter {} created for user: {}, sessionId: {}", interpreter.getClassName(), user, sessionId); } - interpreters.add(new ConfInterpreter(getJavaProperties(), interpreterGroupId, this)); + interpreters.add(new ConfInterpreter(intpProperties, interpreterGroupId, this)); return interpreters; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 23d2d4794eb..57756b8e183 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -367,14 +367,11 @@ public boolean execute(boolean blocking) { return true; } } catch (InterpreterNotFoundException e) { - String intpExceptionMsg = - getJobName() + "'s Interpreter " + getIntpText() + " not found"; - RuntimeException intpException = new RuntimeException(intpExceptionMsg); InterpreterResult intpResult = - new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage()); - setReturn(intpResult, intpException); + new InterpreterResult(InterpreterResult.Code.ERROR); + setReturn(intpResult, e); setStatus(Job.Status.ERROR); - throw intpException; + throw new RuntimeException(e); } } From 9ec6eb5a4e422e4ac73a97e38e47512811d5b29d Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 14 Mar 2018 13:27:09 +0800 Subject: [PATCH 082/386] ZEPPELIN-3328. Add plotting test for LivyInterpreter ### What is this PR for? Add plotting for LivyInterpreter (Python & R) ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3328 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2866 from zjffdu/ZEPPELIN-3328 and squashes the following commits: fd394e3 [Jeff Zhang] ZEPPELIN-3328. Add plotting test for LivyInterpreter --- .../zeppelin/livy/LivyInterpreterIT.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 0c8bea936fc..c7fbc8d4896 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -750,6 +750,26 @@ public void testSharedInterpreter() throws InterpreterException { assertEquals(1, result.message().size()); assertTrue(result.message().get(0).getData().contains("col_1 col_2\n1 hello 20")); } + + // test plotting of python + result = pysparkInterpreter.interpret( + "import matplotlib.pyplot as plt\n" + + "plt.switch_backend('agg')\n" + + "data=[1,2,3,4]\n" + + "plt.figure()\n" + + "plt.plot(data)\n" + + "%matplot plt", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertEquals(InterpreterResult.Type.IMG, result.message().get(0).getType()); + + // test plotting of R + result = sparkRInterpreter.interpret( + "hist(mtcars$mpg)", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertEquals(InterpreterResult.Type.IMG, result.message().get(0).getType()); + } finally { sparkInterpreter.close(); sqlInterpreter.close(); From 1effc63507f819f2b99589a34605afb53226282d Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 12 Mar 2018 09:41:01 +0800 Subject: [PATCH 083/386] ZEPPELIN-3314. Only apply maven-dependency-plugin for interpreter component ### What is this PR for? only interpreter component need to copy interpreter dependency and interpreter-setting.json ### What type of PR is it? [Bug Fix | Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3314 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2857 from zjffdu/ZEPPELIN-3314 and squashes the following commits: 322530f [Jeff Zhang] ZEPPELIN-3314. Only apply maven-dependency-plugin for interpreter component --- interpreter-parent/pom.xml | 8 -------- pom.xml | 36 ++++++++++++++++-------------------- zeppelin-integration/pom.xml | 4 ++++ zeppelin-interpreter/pom.xml | 18 +++++------------- zeppelin-server/pom.xml | 6 ++++++ zeppelin-zengine/pom.xml | 5 +++++ 6 files changed, 36 insertions(+), 41 deletions(-) diff --git a/interpreter-parent/pom.xml b/interpreter-parent/pom.xml index cbfedd5d474..0aa20261376 100644 --- a/interpreter-parent/pom.xml +++ b/interpreter-parent/pom.xml @@ -78,14 +78,6 @@ maven-dependency-plugin - - copy-dependencies - none - - true - - - copy-interpreter-dependencies package diff --git a/pom.xml b/pom.xml index d6087a871dc..ca6c4095de2 100644 --- a/pom.xml +++ b/pom.xml @@ -449,26 +449,6 @@ - - maven-dependency-plugin - - - copy-dependencies - process-test-resources - - copy-dependencies - - - ${project.build.directory}/lib - false - false - true - runtime - - - - - maven-jar-plugin ${plugin.jar.version} @@ -589,6 +569,22 @@ maven-dependency-plugin ${plugin.dependency.version} + + + copy-dependencies + process-test-resources + + copy-dependencies + + + ${project.build.directory}/lib + false + false + true + runtime + + + diff --git a/zeppelin-integration/pom.xml b/zeppelin-integration/pom.xml index 5240a2c4a2c..94ed3a29ecc 100644 --- a/zeppelin-integration/pom.xml +++ b/zeppelin-integration/pom.xml @@ -189,6 +189,10 @@ + + org.apache.maven.plugins + maven-dependency-plugin + diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 80864f9285d..7fe75a75d6a 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -247,20 +247,12 @@ false     + + + org.apache.maven.plugins + maven-dependency-plugin + - - - - maven-enforcer-plugin - - - enforce - none - - - - - diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 1622791417c..3ee83bd9836 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -453,6 +453,12 @@ + + + org.apache.maven.plugins + maven-dependency-plugin + + diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index cdb11ddfd75..a7d9d5f915d 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -405,6 +405,11 @@ + + + org.apache.maven.plugins + maven-dependency-plugin + From 4ee09717cfb2d09b55f20fbd8245b6c2eca37fda Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 12 Mar 2018 10:12:51 +0800 Subject: [PATCH 084/386] ZEPPELIN-3315. Merge beforeStatusChange and afterStatusChange to onStatusChange ### What is this PR for? The signature of `beforeStatusChange` & `afterStatusChange` include both the status of before and after, so it is not necessary to create both `beforeStatusChange` & `afterStatusChange`. Only one method `onStatusChange` is sufficient. ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3315 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2858 from zjffdu/ZEPPELIN-3315 and squashes the following commits: f0fe594 [Jeff Zhang] ZEPPELIN-3315. Merge beforeStatusChange and afterStatusChange to onStatusChange --- .../remote/RemoteInterpreterServer.java | 6 +----- .../java/org/apache/zeppelin/scheduler/Job.java | 7 ++----- .../apache/zeppelin/scheduler/JobListener.java | 6 ++---- .../apache/zeppelin/socket/NotebookServer.java | 6 +----- .../java/org/apache/zeppelin/notebook/Note.java | 16 +++------------- .../zeppelin/scheduler/RemoteScheduler.java | 10 ++-------- .../helium/HeliumApplicationFactoryTest.java | 7 +------ .../apache/zeppelin/notebook/NotebookTest.java | 6 +----- .../notebook/repo/NotebookRepoSyncTest.java | 5 +---- 9 files changed, 14 insertions(+), 55 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index d50d0ed3e2b..766ed16514d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -513,11 +513,7 @@ public void onProgressUpdate(Job job, int progress) { } @Override - public void beforeStatusChange(Job job, Status before, Status after) { - } - - @Override - public void afterStatusChange(Job job, Status before, Status after) { + public void onStatusChange(Job job, Status before, Status after) { synchronized (this) { notifyAll(); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java index 8e25f7b1f2d..579b60441a3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java @@ -152,12 +152,9 @@ public void setStatus(Status status) { } Status before = this.status; Status after = status; - if (listener != null) { - listener.beforeStatusChange(this, before, after); - } this.status = status; - if (listener != null) { - listener.afterStatusChange(this, before, after); + if (listener != null && before != after) { + listener.onStatusChange(this, before, after); } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobListener.java index 3042941d774..00f8d810972 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobListener.java @@ -18,12 +18,10 @@ package org.apache.zeppelin.scheduler; /** - * TODO(moon) : add description. + * Listener for job execution. */ public interface JobListener { void onProgressUpdate(Job job, int progress); - void beforeStatusChange(Job job, Job.Status before, Job.Status after); - - void afterStatusChange(Job job, Job.Status before, Job.Status after); + void onStatusChange(Job job, Job.Status before, Job.Status after); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 0888874fcb8..4a17f5d1ec5 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -2279,11 +2279,7 @@ public void onProgressUpdate(Job job, int progress) { } @Override - public void beforeStatusChange(Job job, Status before, Status after) { - } - - @Override - public void afterStatusChange(Job job, Status before, Status after) { + public void onStatusChange(Job job, Status before, Status after) { if (after == Status.ERROR) { if (job.getException() != null) { LOG.error("Error", job.getException()); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index fc70c706ab7..f2d77639976 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -912,23 +912,13 @@ public Map getInfo() { public void setInfo(Map info) { this.info = info; } - - @Override - public void beforeStatusChange(Job job, Status before, Status after) { - if (jobListenerFactory != null) { - ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); - if (listener != null) { - listener.beforeStatusChange(job, before, after); - } - } - } - + @Override - public void afterStatusChange(Job job, Status before, Status after) { + public void onStatusChange(Job job, Status before, Status after) { if (jobListenerFactory != null) { ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); if (listener != null) { - listener.afterStatusChange(job, before, after); + listener.onStatusChange(job, before, after); } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java index 982b84a979b..d6d1df70558 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java @@ -253,12 +253,10 @@ public synchronized Status getStatus() { if (status == Status.UNKNOWN) { // not found this job in the remote schedulers. // maybe not submitted, maybe already finished - //Status status = getLastStatus(); - listener.afterStatusChange(job, null, null); return job.getStatus(); } lastStatus = status; - listener.afterStatusChange(job, null, status); + listener.onStatusChange(job, null, status); return status; } } @@ -350,11 +348,7 @@ public void onProgressUpdate(Job job, int progress) { } @Override - public void beforeStatusChange(Job job, Status before, Status after) { - } - - @Override - public void afterStatusChange(Job job, Status before, Status after) { + public void onStatusChange(Job job, Status before, Status after) { // Update remoteStatus if (jobExecuted == false) { if (after == Status.FINISHED || after == Status.ABORT diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 03de533a2b6..af91819f759 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -322,12 +322,7 @@ public void onProgressUpdate(Job job, int progress) { } @Override - public void beforeStatusChange(Job job, Job.Status before, Job.Status after) { - - } - - @Override - public void afterStatusChange(Job job, Job.Status before, Job.Status after) { + public void onStatusChange(Job job, Job.Status before, Job.Status after) { } }; diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index cfb2e1660ce..490ac53c819 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -1460,11 +1460,7 @@ public void onProgressUpdate(Job job, int progress) { } @Override - public void beforeStatusChange(Job job, Status before, Status after) { - } - - @Override - public void afterStatusChange(Job job, Status before, Status after) { + public void onStatusChange(Job job, Status before, Status after) { if (afterStatusChangedListener != null) { afterStatusChangedListener.onStatusChanged(job, before, after); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 8904239310b..55712302e3f 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -428,11 +428,8 @@ public void onProgressUpdate(Job job, int progress) { } @Override - public void beforeStatusChange(Job job, Status before, Status after) { - } + public void onStatusChange(Job job, Status before, Status after) { - @Override - public void afterStatusChange(Job job, Status before, Status after) { } }; } From 8b2259cf5805f654b5da2725fa8207fa0b6d5f7e Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 6 Mar 2018 15:02:19 +0800 Subject: [PATCH 085/386] ZEPPELIN-2515. After 100 minutes R process quits silently and spark.r interpreter becomes unresponsive ### What is this PR for? This PR would 2 features: 1. Make timeout of sparkr backend configurable. 2. Detect R backend dead and display proper message to frontend. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2515 ### How should this be tested? * unit test is added ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2839 from zjffdu/ZEPPELIN-2515 and squashes the following commits: 62888c5 [Jeff Zhang] ZEPPELIN-2515. After 100 minutes R process quits silently and spark.r interpreter becomes unresponsive --- .../zeppelin/spark/SparkRInterpreter.java | 17 ++++++++-- .../org/apache/zeppelin/spark/ZeppelinR.java | 32 +++++++++++++++++-- .../src/main/resources/R/zeppelin_sparkr.R | 3 +- .../zeppelin/spark/SparkRInterpreterTest.java | 9 +++++- 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java index 5efff0e7be4..44f71b7926e 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java @@ -27,6 +27,7 @@ import org.apache.spark.api.java.JavaSparkContext; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.interpreter.util.InterpreterOutputStream; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; @@ -36,6 +37,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; /** * R and SparkR interpreter with visualization support. @@ -46,6 +49,7 @@ public class SparkRInterpreter extends Interpreter { private static String renderOptions; private SparkInterpreter sparkInterpreter; private ZeppelinR zeppelinR; + private AtomicBoolean rbackendDead = new AtomicBoolean(false); private SparkContext sc; private JavaSparkContext jsc; @@ -79,10 +83,11 @@ public void open() throws InterpreterException { } int port = SparkRBackend.port(); - this.sparkInterpreter = getSparkInterpreter(); this.sc = sparkInterpreter.getSparkContext(); this.jsc = sparkInterpreter.getJavaSparkContext(); + int timeout = this.sc.getConf().getInt("spark.r.backendConnectionTimeout", 6000); + SparkVersion sparkVersion = new SparkVersion(sc.version()); ZeppelinRContext.setSparkContext(sc); ZeppelinRContext.setJavaSparkContext(jsc); @@ -92,7 +97,7 @@ public void open() throws InterpreterException { ZeppelinRContext.setSqlContext(sparkInterpreter.getSQLContext()); ZeppelinRContext.setZeppelinContext(sparkInterpreter.getZeppelinContext()); - zeppelinR = new ZeppelinR(rCmdPath, sparkRLibPath, port, sparkVersion); + zeppelinR = new ZeppelinR(rCmdPath, sparkRLibPath, port, sparkVersion, timeout, this); try { zeppelinR.open(); } catch (IOException e) { @@ -159,6 +164,10 @@ public InterpreterResult interpret(String lines, InterpreterContext interpreterC try { // render output with knitr + if (rbackendDead.get()) { + return new InterpreterResult(InterpreterResult.Code.ERROR, + "sparkR backend is dead, please try to increase spark.r.backendConnectionTimeout"); + } if (useKnitr()) { zeppelinR.setInterpreterOutput(null); zeppelinR.set(".zcmd", "\n```{r " + renderOptions + "}\n" + lines + "\n```"); @@ -253,4 +262,8 @@ private boolean useKnitr() { return false; } } + + public AtomicBoolean getRbackendDead() { + return rbackendDead; + } } diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java index 130d849975b..e481dbecc34 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java @@ -36,9 +36,12 @@ * R repl interaction */ public class ZeppelinR implements ExecuteResultHandler { - Logger logger = LoggerFactory.getLogger(ZeppelinR.class); + private static Logger logger = LoggerFactory.getLogger(ZeppelinR.class); + + private final SparkRInterpreter sparkRInterpreter; private final String rCmdPath; private final SparkVersion sparkVersion; + private final int timeout; private DefaultExecutor executor; private InterpreterOutputStream outputStream; private PipedOutputStream input; @@ -108,11 +111,13 @@ public Object getValue() { * @param libPath sparkr library path */ public ZeppelinR(String rCmdPath, String libPath, int sparkRBackendPort, - SparkVersion sparkVersion) { + SparkVersion sparkVersion, int timeout, SparkRInterpreter sparkRInterpreter) { this.rCmdPath = rCmdPath; this.libPath = libPath; this.sparkVersion = sparkVersion; this.port = sparkRBackendPort; + this.timeout = timeout; + this.sparkRInterpreter = sparkRInterpreter; try { File scriptFile = File.createTempFile("zeppelin_sparkr-", ".R"); scriptPath = scriptFile.getAbsolutePath(); @@ -140,12 +145,13 @@ public void open() throws IOException, InterpreterException { cmd.addArgument(Integer.toString(port)); cmd.addArgument(libPath); cmd.addArgument(Integer.toString(sparkVersion.toNumber())); + cmd.addArgument(Integer.toString(timeout)); // dump out the R command to facilitate manually running it, e.g. for fault diagnosis purposes logger.debug(cmd.toString()); executor = new DefaultExecutor(); - outputStream = new InterpreterOutputStream(logger); + outputStream = new SparkRInterpreterOutputStream(logger, sparkRInterpreter); input = new PipedOutputStream(); PipedInputStream in = new PipedInputStream(input); @@ -391,4 +397,24 @@ public void onProcessFailed(ExecuteException e) { logger.error(e.getMessage(), e); rScriptRunning = false; } + + + public static class SparkRInterpreterOutputStream extends InterpreterOutputStream { + + private SparkRInterpreter sparkRInterpreter; + + public SparkRInterpreterOutputStream(Logger logger, SparkRInterpreter sparkRInterpreter) { + super(logger); + this.sparkRInterpreter = sparkRInterpreter; + } + + @Override + protected void processLine(String s, int i) { + super.processLine(s, i); + if (s.contains("Java SparkR backend might have failed") // spark 2.x + || s.contains("Execution halted")) { // spark 1.x + sparkRInterpreter.getRbackendDead().set(true); + } + } + } } diff --git a/spark/interpreter/src/main/resources/R/zeppelin_sparkr.R b/spark/interpreter/src/main/resources/R/zeppelin_sparkr.R index 525c6c5c40b..16b8415f659 100644 --- a/spark/interpreter/src/main/resources/R/zeppelin_sparkr.R +++ b/spark/interpreter/src/main/resources/R/zeppelin_sparkr.R @@ -22,6 +22,7 @@ hashCode <- as.integer(args[1]) port <- as.integer(args[2]) libPath <- args[3] version <- as.integer(args[4]) +timeout <- as.integer(args[5]) rm(args) print(paste("Port ", toString(port))) @@ -31,7 +32,7 @@ print(paste("LibPath ", libPath)) library(SparkR) -SparkR:::connectBackend("localhost", port, 6000) +SparkR:::connectBackend("localhost", port, timeout) # scStartTime is needed by R/pkg/R/sparkR.R assign(".scStartTime", as.integer(Sys.time()), envir = SparkR:::.sparkREnv) diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java index 0bd88d44d77..bcdd876e01f 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java @@ -47,7 +47,7 @@ public class SparkRInterpreterTest { private RemoteEventClient mockRemoteEventClient = mock(RemoteEventClient.class); @Test - public void testSparkRInterpreter() throws IOException, InterruptedException, InterpreterException { + public void testSparkRInterpreter() throws InterpreterException, InterruptedException { Properties properties = new Properties(); properties.setProperty("spark.master", "local"); properties.setProperty("spark.app.name", "test"); @@ -55,6 +55,7 @@ public void testSparkRInterpreter() throws IOException, InterruptedException, In properties.setProperty("zeppelin.spark.test", "true"); properties.setProperty("zeppelin.spark.useNew", "true"); properties.setProperty("zeppelin.R.knitr", "true"); + properties.setProperty("spark.r.backendConnectionTimeout", "10"); sparkRInterpreter = new SparkRInterpreter(properties); sparkInterpreter = new SparkInterpreter(properties); @@ -91,6 +92,12 @@ public void testSparkRInterpreter() throws IOException, InterruptedException, In // spark job url is sent verify(mockRemoteEventClient, atLeastOnce()).onParaInfosReceived(any(String.class), any(String.class), any(Map.class)); } + + // sparkr backend would be timeout after 10 seconds + Thread.sleep(15 * 1000); + result = sparkRInterpreter.interpret("1+1", getInterpreterContext()); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertTrue(result.message().get(0).getData().contains("sparkR backend is dead")); } private InterpreterContext getInterpreterContext() { From 6001b98de620c13d57c88a0b4ae328bf19f338ea Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Wed, 14 Mar 2018 15:34:48 +0530 Subject: [PATCH 086/386] [ZEPPELIN-3321] isRevisionSupported not working for importing notebooks. when importing note of from one zeppelin instance that don't support revision to another zeppelin instance where revision is supported, the imported note still don't support revision. [Bug Fix] * [ZEPPELIN-3321](https://issues.apache.org/jira/browse/ZEPPELIN-3321) * Does the licenses files need update? N/A * Is there breaking changes for older versions? N/A * Does this needs documentation? N/A Author: Prabhjyot Singh Closes #2862 from prabhjyotsingh/ZEPPELIN-3321 and squashes the following commits: eca6b59 [Prabhjyot Singh] add unit test for isRevisionSupported 552fdeb [Prabhjyot Singh] approach 2: set as isRevisionSupported zeppelin global config 18873d3 [Prabhjyot Singh] ZEPPELIN-3321: fix isRevisionSupported Change-Id: I49ec3350303b39a41b769bc47941c3397160234c --- .../zeppelin/socket/NotebookServer.java | 2 +- .../src/app/notebook/notebook-actionBar.html | 2 +- .../components/navbar/navbar.controller.js | 12 +++++++++ .../org/apache/zeppelin/notebook/Note.java | 11 -------- .../apache/zeppelin/notebook/Notebook.java | 11 +++++++- .../zeppelin/notebook/NotebookTest.java | 27 +++++++++++++++++++ 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 4a17f5d1ec5..87c757cfaa2 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -1908,7 +1908,7 @@ public boolean apply(String key) { .getVarName()); } }); - + configurations.put("isRevisionSupported", String.valueOf(notebook.isRevisionSupported())); conn.send(serializeMessage( new Message(OP.CONFIGURATIONS_INFO).put("configurations", configurations))); } diff --git a/zeppelin-web/src/app/notebook/notebook-actionBar.html b/zeppelin-web/src/app/notebook/notebook-actionBar.html index b4add9375a5..7559a871c38 100644 --- a/zeppelin-web/src/app/notebook/notebook-actionBar.html +++ b/zeppelin-web/src/app/notebook/notebook-actionBar.html @@ -100,7 +100,7 @@

    - +