From 226493a3a9c3c3ac8aad3b1d79138c3b13c05941 Mon Sep 17 00:00:00 2001 From: Renjith Kamath Date: Mon, 9 Nov 2015 22:39:22 +0530 Subject: [PATCH 1/6] ZEPPELIN-387 Allow enable/disable of specific paragraphs --- .../apache/zeppelin/socket/NotebookServer.java | 4 +++- zeppelin-web/src/app/home/home.css | 18 +++++++++++++++++- .../notebook/paragraph/paragraph.controller.js | 12 +++++++++++- .../src/app/notebook/paragraph/paragraph.html | 6 +++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index c6aa846d13e..cf4b94f3e42 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -610,7 +610,9 @@ private void runParagraph(NotebookSocket conn, Notebook notebook, note.persist(); try { - note.run(paragraphId); + if ((Boolean) config.get("enabled")) { + note.run(paragraphId); + } } catch (Exception ex) { LOG.error("Exception from run", ex); if (p != null) { diff --git a/zeppelin-web/src/app/home/home.css b/zeppelin-web/src/app/home/home.css index 42c317054ab..d0b3046789c 100644 --- a/zeppelin-web/src/app/home/home.css +++ b/zeppelin-web/src/app/home/home.css @@ -308,4 +308,20 @@ This part should be removed when new version of bootstrap handles this issue. .btn-group > .tooltip + .btn, .btn-group > .popover + .btn { margin-left: -1px; -} \ No newline at end of file +} + +.dropdown-menu > li > label { + display: block; + font-weight: normal; + padding: 3px 20px; + line-height: 1.42857143; + color: #333; + white-space: nowrap; + margin: 0; +} + +.dropdown-menu>li>label:hover { + color: #262626; + text-decoration: none; + background-color: #f5f5f5; +} diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 52fe339a343..58ee81f0626 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -127,6 +127,10 @@ angular.module('zeppelinWebApp') if (!config.graph.scatter) { config.graph.scatter = {}; } + + if (config.enabled === undefined) { + config.enabled = true; + } }; $scope.getIframeDimensions = function () { @@ -253,6 +257,12 @@ angular.module('zeppelinWebApp') $scope.dirtyText = undefined; }; + $scope.toggleEnableDisable = function () { + var newParams = angular.copy($scope.paragraph.settings.params); + var newConfig = angular.copy($scope.paragraph.config); + commitParagraph($scope.paragraph.title, $scope.paragraph.text, newConfig, newParams); + }; + $scope.moveUp = function() { $scope.$emit('moveParagraphUp', $scope.paragraph.id); }; @@ -633,7 +643,7 @@ angular.module('zeppelinWebApp') desc += ' (outdated)'; } return desc; - }; + }; $scope.isResultOutdated = function() { var pdata = $scope.paragraph; diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index 78c240f05f3..af4ef641a94 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -409,7 +409,7 @@ + ng-show="paragraph.status!='RUNNING' && paragraph.status!='PENDING' && paragraph.config.enabled"> @@ -436,6 +436,10 @@ +
  • + +
  • Move Up From 87aa4e82aa139471e0743c901aad9918325d0a2f Mon Sep 17 00:00:00 2001 From: Renjith Kamath Date: Tue, 10 Nov 2015 15:09:11 +0530 Subject: [PATCH 2/6] ZEPPELIN-387 remove checkbox, add icon and update the position --- zeppelin-web/src/app/home/home.css | 16 ---------------- .../notebook/paragraph/paragraph.controller.js | 1 + .../src/app/notebook/paragraph/paragraph.html | 9 ++++----- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/zeppelin-web/src/app/home/home.css b/zeppelin-web/src/app/home/home.css index d0b3046789c..8937cd7ef19 100644 --- a/zeppelin-web/src/app/home/home.css +++ b/zeppelin-web/src/app/home/home.css @@ -309,19 +309,3 @@ This part should be removed when new version of bootstrap handles this issue. .btn-group > .popover + .btn { margin-left: -1px; } - -.dropdown-menu > li > label { - display: block; - font-weight: normal; - padding: 3px 20px; - line-height: 1.42857143; - color: #333; - white-space: nowrap; - margin: 0; -} - -.dropdown-menu>li>label:hover { - color: #262626; - text-decoration: none; - background-color: #f5f5f5; -} diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 58ee81f0626..dc052e26d3e 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -258,6 +258,7 @@ angular.module('zeppelinWebApp') }; $scope.toggleEnableDisable = function () { + $scope.paragraph.config.enabled = $scope.paragraph.config.enabled ? false : true; var newParams = angular.copy($scope.paragraph.settings.params); var newConfig = angular.copy($scope.paragraph.config); commitParagraph($scope.paragraph.title, $scope.paragraph.text, newConfig, newParams); diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index af4ef641a94..4f55c6b01e8 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -436,10 +436,6 @@
    • -
  • - -
  • Move Up @@ -469,7 +465,10 @@ ng-click="showLineNumbers()" ng-show="!paragraph.config.lineNumbers"> Show line numbers
    • - +
  • + {{paragraph.config.enabled ? "Disable" : "Enable"}} paragraph +
  • Link this paragraph
    • From 3a7175ad38dbfa3b7a5436c0a35edc7c0ecd8ebf Mon Sep 17 00:00:00 2001 From: Renjith Kamath Date: Mon, 16 Nov 2015 11:50:01 +0530 Subject: [PATCH 3/6] ZEPPELIN-387 review updates --- zeppelin-web/src/app/notebook/paragraph/paragraph.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.html b/zeppelin-web/src/app/notebook/paragraph/paragraph.html index 4f55c6b01e8..601a95d8278 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.html @@ -466,8 +466,8 @@ ng-show="!paragraph.config.lineNumbers"> Show line numbers
  • - {{paragraph.config.enabled ? "Disable" : "Enable"}} paragraph + {{paragraph.config.enabled ? "Disable" : "Enable"}} run
  • Link this paragraph From 118d35814c2aaa0e0ee4953b2a9f0900104fffcd Mon Sep 17 00:00:00 2001 From: Renjith Kamath Date: Tue, 17 Nov 2015 21:48:14 +0530 Subject: [PATCH 4/6] ZEPPELIN-387 review fix --- .../main/java/org/apache/zeppelin/socket/NotebookServer.java | 4 +--- .../src/main/java/org/apache/zeppelin/notebook/Note.java | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index cf4b94f3e42..c6aa846d13e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -610,9 +610,7 @@ private void runParagraph(NotebookSocket conn, Notebook notebook, note.persist(); try { - if ((Boolean) config.get("enabled")) { - note.run(paragraphId); - } + note.run(paragraphId); } catch (Exception ex) { LOG.error("Exception from run", ex); if (p != null) { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index b72517722e2..8ba85d52637 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -283,7 +283,9 @@ public void run(String paragraphId) { if (intp == null) { throw new InterpreterException("Interpreter " + p.getRequiredReplName() + " not found"); } - intp.getScheduler().submit(p); + if ((Boolean) p.getConfig().get("enabled")) { + intp.getScheduler().submit(p); + } } public List completion(String paragraphId, String buffer, int cursor) { From 87ef87b4d35365fb987a952c1b9db6cff7f914ab Mon Sep 17 00:00:00 2001 From: Renjith Kamath Date: Fri, 20 Nov 2015 12:24:48 +0530 Subject: [PATCH 5/6] ZEPPELIN-387 fix tests --- .../zeppelin/rest/ZeppelinRestApiTest.java | 8 ++++++ .../rest/ZeppelinSparkClusterTest.java | 27 +++++++++++++++++++ .../zeppelin/notebook/NotebookTest.java | 19 +++++++++++-- .../notebook/repo/NotebookRepoSyncTest.java | 8 +++--- 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index bffc8886dce..6e54245cca5 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.rest; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -153,8 +154,11 @@ public void testInterpreterRestart() throws IOException, InterruptedException { Note note = ZeppelinServer.notebook.createNote(); note.addParagraph(); Paragraph p = note.getLastParagraph(); + Map config = p.getConfig(); + config.put("enabled", true); // run markdown paragraph + p.setConfig(config); p.setText("%md markdown"); note.run(p.getId()); while (p.getStatus() != Status.FINISHED) { @@ -173,6 +177,7 @@ public void testInterpreterRestart() throws IOException, InterruptedException { // run markdown paragraph, again p = note.addParagraph(); + p.setConfig(config); p.setText("%md markdown restarted"); note.run(p.getId()); while (p.getStatus() != Status.FINISHED) { @@ -257,6 +262,9 @@ public void testCloneNotebook() throws IOException, CloneNotSupportedException, assertNotNull("cant create new note", note); note.setName("source note for clone"); Paragraph paragraph = note.addParagraph(); + Map config = paragraph.getConfig(); + config.put("enabled", true); + paragraph.setConfig(config); paragraph.setText("%md This is my new paragraph in my new note"); note.persist(); String sourceNoteID = note.getId(); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index d5006eee957..7076a796e8b 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -20,7 +20,9 @@ import java.io.File; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.interpreter.InterpreterSetting; @@ -68,6 +70,9 @@ public void basicRDDTransformationAndActionTest() throws IOException { // run markdown paragraph, again Paragraph p = note.addParagraph(); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); p.setText("%spark print(sc.parallelize(1 to 10).reduce(_ + _))"); note.run(p.getId()); waitForFinish(p); @@ -84,6 +89,9 @@ public void pySparkTest() throws IOException { if (isPyspark() && sparkVersion >= 12) { // pyspark supported from 1.2.1 // run markdown paragraph, again Paragraph p = note.addParagraph(); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); p.setText("%pyspark print(sc.parallelize(range(1, 11)).reduce(lambda a, b: a + b))"); note.run(p.getId()); waitForFinish(p); @@ -102,6 +110,9 @@ public void pySparkAutoConvertOptionTest() throws IOException { if (isPyspark() && sparkVersion >= 14) { // auto_convert enabled from spark 1.4 // run markdown paragraph, again Paragraph p = note.addParagraph(); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); p.setText("%pyspark\nfrom pyspark.sql.functions import *\n" + "print(sqlContext.range(0, 10).withColumn('uniform', rand(seed=10) * 3.14).count())"); note.run(p.getId()); @@ -116,10 +127,19 @@ public void zRunTest() throws IOException { // create new note Note note = ZeppelinServer.notebook.createNote(); Paragraph p0 = note.addParagraph(); + Map config0 = p0.getConfig(); + config0.put("enabled", true); + p0.setConfig(config0); p0.setText("%spark z.run(1)"); Paragraph p1 = note.addParagraph(); + Map config1 = p1.getConfig(); + config1.put("enabled", true); + p1.setConfig(config1); p1.setText("%spark val a=10"); Paragraph p2 = note.addParagraph(); + Map config2 = p2.getConfig(); + config2.put("enabled", true); + p2.setConfig(config2); p2.setText("%spark print(a)"); note.run(p0.getId()); @@ -151,6 +171,9 @@ public void pySparkDepLoaderTest() throws IOException { // load dep Paragraph p0 = note.addParagraph(); + Map config = p0.getConfig(); + config.put("enabled", true); + p0.setConfig(config); p0.setText("%dep z.load(\"com.databricks:spark-csv_2.11:1.2.0\")"); note.run(p0.getId()); waitForFinish(p0); @@ -161,6 +184,7 @@ public void pySparkDepLoaderTest() throws IOException { // load data using libraries from dep loader Paragraph p1 = note.addParagraph(); + p1.setConfig(config); p1.setText("%pyspark\n" + "from pyspark.sql import SQLContext\n" + "print(sqlContext.read.format('com.databricks.spark.csv')" + @@ -178,6 +202,9 @@ public void pySparkDepLoaderTest() throws IOException { */ private int getSparkVersionNumber(Note note) { Paragraph p = note.addParagraph(); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); p.setText("%spark print(sc.version)"); note.run(p.getId()); waitForFinish(p); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 883dc64b714..e5b5afe5e52 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.IOException; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -93,6 +94,9 @@ public void testSelectingReplImplementation() throws IOException { // run with defatul repl Paragraph p1 = note.addParagraph(); + Map config = p1.getConfig(); + config.put("enabled", true); + p1.setConfig(config); p1.setText("hello world"); note.run(p1.getId()); while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); @@ -100,6 +104,7 @@ public void testSelectingReplImplementation() throws IOException { // run with specific repl Paragraph p2 = note.addParagraph(); + p2.setConfig(config); p2.setText("%mock2 hello world"); note.run(p2.getId()); while(p2.isTerminated()==false || p2.getResult()==null) Thread.yield(); @@ -153,6 +158,9 @@ public void testPersist() throws IOException, SchedulerException{ // run with default repl Paragraph p1 = note.addParagraph(); + Map config = p1.getConfig(); + config.put("enabled", true); + p1.setConfig(config); p1.setText("hello world"); note.persist(); @@ -164,10 +172,14 @@ public void testPersist() throws IOException, SchedulerException{ public void testRunAll() throws IOException { Note note = notebook.createNote(); note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); - Paragraph p1 = note.addParagraph(); + Map config = p1.getConfig(); + config.put("enabled", true); + p1.setConfig(config); p1.setText("p1"); Paragraph p2 = note.addParagraph(); + Map config1 = p2.getConfig(); + p2.setConfig(config1); p2.setText("p2"); assertEquals(null, p2.getResult()); note.runAll(); @@ -183,12 +195,15 @@ public void testSchedule() throws InterruptedException, IOException{ note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); Paragraph p = note.addParagraph(); + Map config = new HashMap(); + p.setConfig(config); p.setText("p1"); Date dateFinished = p.getDateFinished(); assertNull(dateFinished); // set cron scheduler, once a second - Map config = note.getConfig(); + config = note.getConfig(); + config.put("enabled", true); config.put("cron", "* * * * * ?"); note.setConfig(config); notebook.refreshCron(note.id()); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 981ed6e2451..b6eae0f8f4a 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -19,11 +19,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.File; import java.io.IOException; +import java.util.Map; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; @@ -33,10 +32,8 @@ import org.apache.zeppelin.interpreter.mock.MockInterpreter2; import org.apache.zeppelin.notebook.JobListenerFactory; import org.apache.zeppelin.notebook.Note; -import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.Paragraph; -import org.apache.zeppelin.notebook.repo.NotebookRepoSync; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; @@ -148,6 +145,9 @@ public void testSyncUpdateMain() throws IOException { /* create note */ Note note = notebookSync.createNote(); Paragraph p1 = note.addParagraph(); + Map config = p1.getConfig(); + config.put("enabled", true); + p1.setConfig(config); p1.setText("hello world"); /* new paragraph exists in note instance */ From 796575dd135f8a25ab696a605c8b2fcfcba8ec77 Mon Sep 17 00:00:00 2001 From: Renjith Kamath Date: Fri, 20 Nov 2015 16:30:07 +0530 Subject: [PATCH 6/6] ZEPPELIN-387 fix testClearParagraphOutput failure --- .../test/java/org/apache/zeppelin/notebook/NotebookTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 537cf57dd88..8ae0970868b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -174,6 +174,9 @@ public void testPersist() throws IOException, SchedulerException{ public void testClearParagraphOutput() throws IOException, SchedulerException{ Note note = notebook.createNote(); Paragraph p1 = note.addParagraph(); + Map config = p1.getConfig(); + config.put("enabled", true); + p1.setConfig(config); p1.setText("hello world"); note.run(p1.getId());