From 6d7e0630935b0f0df480ae9239a3c704a78011ae Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 30 Mar 2015 10:55:14 +0900 Subject: [PATCH 01/21] Add source file license header --- .../ScreenCaptureHtmlUnitDriver.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java b/zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java index ae83bee3885..91231f6d12a 100644 --- a/zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java +++ b/zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.webautomation; import java.io.ByteArrayOutputStream; From a7c77b87750d88748cc99ee38bfb53f684994569 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 30 Mar 2015 16:19:19 +0900 Subject: [PATCH 02/21] Update license of ScreenCaptureHtmlUnitDriver.java --- .../ScreenCaptureHtmlUnitDriver.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java b/zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java index 91231f6d12a..ae83bee3885 100644 --- a/zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java +++ b/zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java @@ -1,20 +1,3 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.webautomation; import java.io.ByteArrayOutputStream; From bb52d7bf39975646ed22fd9e627d6fae3c8e407a Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 3 Apr 2015 14:38:37 +0900 Subject: [PATCH 03/21] Add %angular display system --- .../interpreter/InterpreterResult.java | 1 + .../app/scripts/controllers/paragraph.js | 24 ++++++++++++++++++- zeppelin-web/app/views/paragraph.html | 5 ++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java index 0659a47f677..5d8d96f9064 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java @@ -48,6 +48,7 @@ public static enum Code { public static enum Type { TEXT, HTML, + ANGULAR, TABLE, IMG, SVG, diff --git a/zeppelin-web/app/scripts/controllers/paragraph.js b/zeppelin-web/app/scripts/controllers/paragraph.js index aac8a785677..d99a9faadd6 100644 --- a/zeppelin-web/app/scripts/controllers/paragraph.js +++ b/zeppelin-web/app/scripts/controllers/paragraph.js @@ -25,7 +25,7 @@ * @author anthonycorbacho */ angular.module('zeppelinWebApp') - .controller('ParagraphCtrl', function($scope, $rootScope, $route, $window, $element, $routeParams, $location, $timeout) { + .controller('ParagraphCtrl', function($scope, $rootScope, $route, $window, $element, $routeParams, $location, $timeout, $compile) { $scope.paragraph = null; $scope.editor = null; @@ -56,6 +56,8 @@ angular.module('zeppelinWebApp') $scope.setGraphMode($scope.getGraphMode(), false, false); } else if ($scope.getResultType() === 'HTML') { $scope.renderHtml(); + } else if ($scope.getResultType() === 'ANGULAR') { + $scope.renderAngular(); } }; @@ -77,6 +79,24 @@ angular.module('zeppelinWebApp') }; + $scope.renderAngular = function() { + var retryRenderer = function() { + if ($('#p'+$scope.paragraph.id+'_angular').length) { + try { + $('#p'+$scope.paragraph.id+'_angular').html($scope.paragraph.result.msg); + $compile($('#p'+$scope.paragraph.id+'_angular').contents())($scope); + } catch(err) { + console.log('ANGULAR rendering error %o', err); + } + } else { + $timeout(retryRenderer,10); + } + }; + $timeout(retryRenderer); + + }; + + var initializeDefault = function() { var config = $scope.paragraph.config; @@ -210,6 +230,8 @@ angular.module('zeppelinWebApp') } } else if (newType === 'HTML') { $scope.renderHtml(); + } else if (newType === 'ANGULAR') { + $scope.renderAngular(); } } }); diff --git a/zeppelin-web/app/views/paragraph.html b/zeppelin-web/app/views/paragraph.html index c77c85ae6ad..ef4daaaee1b 100644 --- a/zeppelin-web/app/views/paragraph.html +++ b/zeppelin-web/app/views/paragraph.html @@ -353,6 +353,11 @@ ng-Init="loadResultType(paragraph.result)"> +
+
+ Date: Sat, 4 Apr 2015 17:29:57 +0900 Subject: [PATCH 04/21] ZEPPELIN-25 impelemnet JS(angular) -JVM(scala) two-way binding --- .../zeppelin/spark/ZeppelinContext.java | 18 + .../zeppelin/spark/DepInterpreterTest.java | 4 +- .../zeppelin/spark/SparkInterpreterTest.java | 6 +- .../spark/SparkSqlInterpreterTest.java | 7 +- .../zeppelin/display/AngularObject.java | 107 ++ .../display/AngularObjectListener.java | 25 + .../display/AngularObjectRegistry.java | 86 + .../AngularObjectRegistryListener.java | 28 + .../remote/RemoteAngularObject.java | 44 + .../remote/RemoteAngularObjectRegistry.java | 66 + .../remote/RemoteInterpreterEventPoller.java | 113 ++ .../thrift/RemoteInterpreterEvent.java | 502 ++++++ .../thrift/RemoteInterpreterEventType.java | 51 + .../interpreter/InterpreterContext.java | 10 +- .../interpreter/InterpreterGroup.java | 16 + .../interpreter/remote/RemoteInterpreter.java | 2 +- .../remote/RemoteInterpreterProcess.java | 36 +- .../remote/RemoteInterpreterServer.java | 98 +- .../thrift/RemoteInterpreterService.java | 1462 +++++++++++++++++ .../thrift/RemoteInterpreterService.thrift | 15 + .../zeppelin/display/AngularObjectTest.java | 29 + .../remote/RemoteInterpreterProcessTest.java | 9 +- .../remote/RemoteInterpreterTest.java | 22 +- .../scheduler/RemoteSchedulerTest.java | 6 +- .../zeppelin/server/ZeppelinServer.java | 2 +- .../org/apache/zeppelin/socket/Message.java | 5 + .../zeppelin/socket/NotebookServer.java | 89 +- zeppelin-web/app/scripts/controllers/main.js | 5 +- .../app/scripts/controllers/notebook.js | 31 + .../app/scripts/controllers/paragraph.js | 4 +- .../interpreter/InterpreterFactory.java | 32 +- .../apache/zeppelin/notebook/Paragraph.java | 12 +- .../interpreter/InterpreterFactoryTest.java | 6 +- .../zeppelin/notebook/NotebookTest.java | 4 +- 34 files changed, 2915 insertions(+), 37 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectListener.java create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistryListener.java create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java create mode 100644 zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index 87cd18884da..517831a4d4c 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -251,4 +251,22 @@ public void setInterpreterContext(InterpreterContext interpreterContext) { this.interpreterContext = interpreterContext; } + +/* + public void angularBind(String name, sObject o) { + + } + + public void angularBind(String name, sObject o, sWatcher w) { + + } + + public void angularBind(String name, Function f) { + + } + + public void angularUnbind(sString name) { + + } + */ } diff --git a/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java index 8d24cc434b6..bdb7b2ca295 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.Properties; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -57,7 +58,8 @@ public void setUp() throws Exception { intpGroup.add(dep); dep.setInterpreterGroup(intpGroup); - context = new InterpreterContext("id", "title", "text", new HashMap(), new GUI()); + context = new InterpreterContext("id", "title", "text", new HashMap(), new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null)); } @After diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index 20f7fa41b4c..473793c204a 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -24,8 +24,10 @@ import java.util.HashMap; import java.util.Properties; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.spark.SparkInterpreter; @@ -55,7 +57,9 @@ public void setUp() throws Exception { repl.open(); } - context = new InterpreterContext("id", "title", "text", new HashMap(), new GUI()); + InterpreterGroup intpGroup = new InterpreterGroup(); + context = new InterpreterContext("id", "title", "text", new HashMap(), new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null)); } @After diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java index 71f088d282d..c85e2c72a95 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Properties; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -38,6 +39,7 @@ public class SparkSqlInterpreterTest { private SparkSqlInterpreter sql; private SparkInterpreter repl; private InterpreterContext context; + private InterpreterGroup intpGroup; @Before public void setUp() throws Exception { @@ -55,13 +57,14 @@ public void setUp() throws Exception { sql = new SparkSqlInterpreter(p); - InterpreterGroup intpGroup = new InterpreterGroup(); + intpGroup = new InterpreterGroup(); intpGroup.add(repl); intpGroup.add(sql); sql.setInterpreterGroup(intpGroup); sql.open(); } - context = new InterpreterContext("id", "title", "text", new HashMap(), new GUI()); + context = new InterpreterContext("id", "title", "text", new HashMap(), new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null)); } @After diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java new file mode 100644 index 00000000000..8d9f803e14a --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.display; + +import java.util.Map; + +/** + * + * + * @param + */ +public class AngularObject { + + /** + * + */ + public static enum AngularObjectType { + STRING, + MAP, + FUNCTION + }; + + private String name; + private T object; + private transient AngularObjectListener listener; + private AngularObjectType type; + + protected AngularObject(String name, T o, + AngularObjectListener listener) { + this.name = name; + this.listener = listener; + object = o; + type = checkType(); + } + + public AngularObjectType getType() { + return type; + } + + public AngularObjectType checkType() { + if (object == null) { + return AngularObjectType.STRING; + } else if (object instanceof Map) { + return AngularObjectType.MAP; + } else if (object instanceof String) { + return AngularObjectType.STRING; + } + return AngularObjectType.STRING; + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (o instanceof AngularObject) { + return name.equals(((AngularObject) o).name); + } else { + return false; + } + } + + public Object get() { + return object; + } + + public void emit(){ + if (listener != null) { + listener.updated(this); + } + } + + public void set(T o) { + set(o, true); + } + + public void set(T o, boolean emit) { + object = o; + if (emit) { + emit(); + } + } + + public void setListener(AngularObjectListener listener) { + this.listener = listener; + } + + public AngularObjectListener getListener() { + return listener; + } +} diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectListener.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectListener.java new file mode 100644 index 00000000000..84066003b9d --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectListener.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.display; + +/** + * + */ +public interface AngularObjectListener { + public void updated(AngularObject updatedObject); +} diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java new file mode 100644 index 00000000000..013cbaf8a49 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.display; + +import java.util.HashMap; +import java.util.Map; + +/** + * + * + */ +public class AngularObjectRegistry implements AngularObjectListener { + Map registry = new HashMap(); + private AngularObjectRegistryListener listener; + private String interpreterId; + + public AngularObjectRegistry(String interpreterId, + AngularObjectRegistryListener listener) { + this.interpreterId = interpreterId; + this.listener = listener; + + } + + public AngularObjectRegistryListener getListener() { + return listener; + } + + public AngularObject add(String name, Object o) { + AngularObject ao = createNewAngularObject(name, o); + + synchronized (registry) { + registry.put(name, ao); + if (listener != null) { + listener.onAdd(interpreterId, ao); + } + } + + return ao; + } + + protected AngularObject createNewAngularObject(String name, Object o) { + return new AngularObject(name, o, this); + } + + public AngularObject remove(String name) { + synchronized (registry) { + AngularObject o = registry.remove(name); + if (listener != null) { + listener.onRemove(interpreterId, o);; + } + return o; + } + } + + public AngularObject get(String name) { + synchronized (registry) { + return registry.get(name); + } + } + + @Override + public void updated(AngularObject updatedObject) { + if (listener != null) { + listener.onUpdate(interpreterId, updatedObject); + } + } + + public String getInterpreterGroupId() { + return interpreterId; + } +} diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistryListener.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistryListener.java new file mode 100644 index 00000000000..153d5821d7e --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistryListener.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.display; + +/** + * + * + */ +public interface AngularObjectRegistryListener { + public void onAdd(String interpreterGroupId, AngularObject object); + public void onUpdate(String interpreterGroupId, AngularObject object); + public void onRemove(String interpreterGroupId, AngularObject object); +} diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java new file mode 100644 index 00000000000..72007f8690d --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.interpreter.remote; + +import com.nflabs.zeppelin.display.AngularObject; +import com.nflabs.zeppelin.display.AngularObjectListener; + +/** + * + */ +public class RemoteAngularObject extends AngularObject { + + private transient RemoteInterpreterProcess remoteInterpreterProcess; + + RemoteAngularObject(String name, Object o, String interpreterGroupId, + AngularObjectListener listener, + RemoteInterpreterProcess remoteInterpreterProcess) { + super(name, o, listener); + this.remoteInterpreterProcess = remoteInterpreterProcess; + } + + @Override + public void set(Object o, boolean emit) { + super.set(o, emit); + + // send updated value to remote interpreter + remoteInterpreterProcess.updateRemoteAngularObject(getName(), o); + } +} diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java new file mode 100644 index 00000000000..352920fb29d --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.interpreter.remote; + +import com.nflabs.zeppelin.display.AngularObject; +import com.nflabs.zeppelin.display.AngularObjectRegistry; +import com.nflabs.zeppelin.display.AngularObjectRegistryListener; +import com.nflabs.zeppelin.interpreter.Interpreter; +import com.nflabs.zeppelin.interpreter.InterpreterGroup; +import com.nflabs.zeppelin.interpreter.WrappedInterpreter; + +/** + * + */ +public class RemoteAngularObjectRegistry extends AngularObjectRegistry { + + private InterpreterGroup interpreterGroup; + + public RemoteAngularObjectRegistry(String interpreterId, + AngularObjectRegistryListener listener, + InterpreterGroup interpreterGroup) { + super(interpreterId, listener); + this.interpreterGroup = interpreterGroup; + } + + private RemoteInterpreterProcess getRemoteInterpreterProcess() { + if (interpreterGroup.size() == 0) { + throw new RuntimeException("Can't get remoteInterpreterProcess"); + } + Interpreter p = interpreterGroup.get(0); + while (p instanceof WrappedInterpreter) { + p = ((WrappedInterpreter) p).getInnerInterpreter(); + } + + if (p instanceof RemoteInterpreter) { + return ((RemoteInterpreter) p).getInterpreterProcess(); + } else { + throw new RuntimeException("Can't get remoteInterpreterProcess"); + } + } + + @Override + protected AngularObject createNewAngularObject(String name, Object o) { + RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess(); + if (remoteInterpreterProcess == null) { + throw new RuntimeException("Remote Interpreter process not found"); + } + return new RemoteAngularObject(name, o, getInterpreterGroupId(), this, + getRemoteInterpreterProcess()); + } +} diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java new file mode 100644 index 00000000000..189ad085431 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.interpreter.remote; + +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; +import com.nflabs.zeppelin.display.AngularObject; +import com.nflabs.zeppelin.display.AngularObjectRegistry; +import com.nflabs.zeppelin.interpreter.InterpreterGroup; +import com.nflabs.zeppelin.interpreter.thrift.RemoteInterpreterEvent; +import com.nflabs.zeppelin.interpreter.thrift.RemoteInterpreterEventType; +import com.nflabs.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; + +/** + * + */ +public class RemoteInterpreterEventPoller extends Thread { + Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class); + private RemoteInterpreterProcess interpreterProcess; + boolean shutdown; + private InterpreterGroup interpreterGroup; + + public RemoteInterpreterEventPoller( + InterpreterGroup interpreterGroup, + RemoteInterpreterProcess interpreterProcess) { + this.interpreterGroup = interpreterGroup; + this.interpreterProcess = interpreterProcess; + shutdown = false; + } + + @Override + public void run() { + Client client = null; + + while (shutdown == false) { + try { + client = interpreterProcess.getClient(); + } catch (Exception e1) { + logger.error("Can't get RemoteInterpreterEvent", e1); + try { + synchronized (this) { + wait(1000); + } + } catch (InterruptedException e) { + } + continue; + } + + RemoteInterpreterEvent event = null; + try { + event = client.getEvent(); + } catch (TException e) { + logger.error("Can't get RemoteInterpreterEvent", e); + try { + synchronized (this) { + wait(1000); + } + } catch (InterruptedException e1) { + } + continue; + } + + interpreterProcess.releaseClient(client); + + Gson gson = new Gson(); + + AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry(); + + try { + if (event.getType() == RemoteInterpreterEventType.NO_OP) { + continue; + } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_ADD) { + AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class); + angularObjectRegistry.add(angularObject.getName(), angularObject.get()); + } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE) { + AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class); + AngularObject localAngularObject = angularObjectRegistry.get(angularObject.getName()); + localAngularObject.set(angularObject.get()); + } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) { + AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class); + angularObjectRegistry.remove(angularObject.getName()); + } + } catch (Exception e) { + logger.error("Can't handle event " + event, e); + } + } + } + + public void shutdown() { + shutdown = true; + synchronized (this) { + notify(); + } + } +} diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java new file mode 100644 index 00000000000..f9b55e0c2d0 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java @@ -0,0 +1,502 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.nflabs.zeppelin.interpreter.thrift; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RemoteInterpreterEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterEvent"); + + private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("type", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField DATA_FIELD_DESC = new org.apache.thrift.protocol.TField("data", org.apache.thrift.protocol.TType.STRING, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new RemoteInterpreterEventStandardSchemeFactory()); + schemes.put(TupleScheme.class, new RemoteInterpreterEventTupleSchemeFactory()); + } + + /** + * + * @see RemoteInterpreterEventType + */ + public RemoteInterpreterEventType type; // required + public String data; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + /** + * + * @see RemoteInterpreterEventType + */ + TYPE((short)1, "type"), + DATA((short)2, "data"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TYPE + return TYPE; + case 2: // DATA + return DATA; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TYPE, new org.apache.thrift.meta_data.FieldMetaData("type", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, RemoteInterpreterEventType.class))); + tmpMap.put(_Fields.DATA, new org.apache.thrift.meta_data.FieldMetaData("data", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RemoteInterpreterEvent.class, metaDataMap); + } + + public RemoteInterpreterEvent() { + } + + public RemoteInterpreterEvent( + RemoteInterpreterEventType type, + String data) + { + this(); + this.type = type; + this.data = data; + } + + /** + * Performs a deep copy on other. + */ + public RemoteInterpreterEvent(RemoteInterpreterEvent other) { + if (other.isSetType()) { + this.type = other.type; + } + if (other.isSetData()) { + this.data = other.data; + } + } + + public RemoteInterpreterEvent deepCopy() { + return new RemoteInterpreterEvent(this); + } + + @Override + public void clear() { + this.type = null; + this.data = null; + } + + /** + * + * @see RemoteInterpreterEventType + */ + public RemoteInterpreterEventType getType() { + return this.type; + } + + /** + * + * @see RemoteInterpreterEventType + */ + public RemoteInterpreterEvent setType(RemoteInterpreterEventType type) { + this.type = type; + return this; + } + + public void unsetType() { + this.type = null; + } + + /** Returns true if field type is set (has been assigned a value) and false otherwise */ + public boolean isSetType() { + return this.type != null; + } + + public void setTypeIsSet(boolean value) { + if (!value) { + this.type = null; + } + } + + public String getData() { + return this.data; + } + + public RemoteInterpreterEvent setData(String data) { + this.data = data; + return this; + } + + public void unsetData() { + this.data = null; + } + + /** Returns true if field data is set (has been assigned a value) and false otherwise */ + public boolean isSetData() { + return this.data != null; + } + + public void setDataIsSet(boolean value) { + if (!value) { + this.data = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TYPE: + if (value == null) { + unsetType(); + } else { + setType((RemoteInterpreterEventType)value); + } + break; + + case DATA: + if (value == null) { + unsetData(); + } else { + setData((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TYPE: + return getType(); + + case DATA: + return getData(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TYPE: + return isSetType(); + case DATA: + return isSetData(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof RemoteInterpreterEvent) + return this.equals((RemoteInterpreterEvent)that); + return false; + } + + public boolean equals(RemoteInterpreterEvent that) { + if (that == null) + return false; + + boolean this_present_type = true && this.isSetType(); + boolean that_present_type = true && that.isSetType(); + if (this_present_type || that_present_type) { + if (!(this_present_type && that_present_type)) + return false; + if (!this.type.equals(that.type)) + return false; + } + + boolean this_present_data = true && this.isSetData(); + boolean that_present_data = true && that.isSetData(); + if (this_present_data || that_present_data) { + if (!(this_present_data && that_present_data)) + return false; + if (!this.data.equals(that.data)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(RemoteInterpreterEvent other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + RemoteInterpreterEvent typedOther = (RemoteInterpreterEvent)other; + + lastComparison = Boolean.valueOf(isSetType()).compareTo(typedOther.isSetType()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetType()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.type, typedOther.type); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetData()).compareTo(typedOther.isSetData()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetData()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.data, typedOther.data); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("RemoteInterpreterEvent("); + boolean first = true; + + sb.append("type:"); + if (this.type == null) { + sb.append("null"); + } else { + sb.append(this.type); + } + first = false; + if (!first) sb.append(", "); + sb.append("data:"); + if (this.data == null) { + sb.append("null"); + } else { + sb.append(this.data); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class RemoteInterpreterEventStandardSchemeFactory implements SchemeFactory { + public RemoteInterpreterEventStandardScheme getScheme() { + return new RemoteInterpreterEventStandardScheme(); + } + } + + private static class RemoteInterpreterEventStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, RemoteInterpreterEvent struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TYPE + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.type = RemoteInterpreterEventType.findByValue(iprot.readI32()); + struct.setTypeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // DATA + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.data = iprot.readString(); + struct.setDataIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, RemoteInterpreterEvent struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.type != null) { + oprot.writeFieldBegin(TYPE_FIELD_DESC); + oprot.writeI32(struct.type.getValue()); + oprot.writeFieldEnd(); + } + if (struct.data != null) { + oprot.writeFieldBegin(DATA_FIELD_DESC); + oprot.writeString(struct.data); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class RemoteInterpreterEventTupleSchemeFactory implements SchemeFactory { + public RemoteInterpreterEventTupleScheme getScheme() { + return new RemoteInterpreterEventTupleScheme(); + } + } + + private static class RemoteInterpreterEventTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, RemoteInterpreterEvent struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetType()) { + optionals.set(0); + } + if (struct.isSetData()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetType()) { + oprot.writeI32(struct.type.getValue()); + } + if (struct.isSetData()) { + oprot.writeString(struct.data); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, RemoteInterpreterEvent struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.type = RemoteInterpreterEventType.findByValue(iprot.readI32()); + struct.setTypeIsSet(true); + } + if (incoming.get(1)) { + struct.data = iprot.readString(); + struct.setDataIsSet(true); + } + } + } + +} + diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java new file mode 100644 index 00000000000..d8342f3029a --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java @@ -0,0 +1,51 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package com.nflabs.zeppelin.interpreter.thrift; + + +import java.util.Map; +import java.util.HashMap; +import org.apache.thrift.TEnum; + +public enum RemoteInterpreterEventType implements org.apache.thrift.TEnum { + NO_OP(1), + ANGULAR_OBJECT_ADD(2), + ANGULAR_OBJECT_UPDATE(3), + ANGULAR_OBJECT_REMOVE(4); + + private final int value; + + private RemoteInterpreterEventType(int value) { + this.value = value; + } + + /** + * Get the integer value of this enum value, as defined in the Thrift IDL. + */ + public int getValue() { + return value; + } + + /** + * Find a the enum type by its integer value, as defined in the Thrift IDL. + * @return null if the value is not found. + */ + public static RemoteInterpreterEventType findByValue(int value) { + switch (value) { + case 1: + return NO_OP; + case 2: + return ANGULAR_OBJECT_ADD; + case 3: + return ANGULAR_OBJECT_UPDATE; + case 4: + return ANGULAR_OBJECT_REMOVE; + default: + return null; + } + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java index 2d70c8e02b4..f09980d7a91 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java @@ -19,6 +19,7 @@ import java.util.Map; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; /** @@ -30,19 +31,22 @@ public class InterpreterContext { private final String paragraphText; private final Map config; private GUI gui; + private AngularObjectRegistry angularObjectRegistry; public InterpreterContext(String paragraphId, String paragraphTitle, String paragraphText, Map config, - GUI gui + GUI gui, + AngularObjectRegistry angularObjectRegistry ) { this.paragraphId = paragraphId; this.paragraphTitle = paragraphTitle; this.paragraphText = paragraphText; this.config = config; this.gui = gui; + this.angularObjectRegistry = angularObjectRegistry; } public String getParagraphId() { @@ -65,4 +69,8 @@ public GUI getGui() { return gui; } + public AngularObjectRegistry getAngularObjectRegistry() { + return angularObjectRegistry; + } + } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java index 834630a6e17..3c882b18bfa 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java @@ -21,6 +21,8 @@ import java.util.Properties; import java.util.Random; +import com.nflabs.zeppelin.display.AngularObjectRegistry; + /** * InterpreterGroup is list of interpreters in the same group. * And unit of interpreter instantiate, restart, bind, unbind. @@ -28,6 +30,12 @@ public class InterpreterGroup extends LinkedList{ String id; + AngularObjectRegistry angularObjectRegistry; + + public InterpreterGroup() { + this.id = getId(); + } + private static String generateId() { return "InterpreterGroup_" + System.currentTimeMillis() + "_" + new Random().nextInt(); @@ -51,6 +59,14 @@ public Properties getProperty() { return p; } + public AngularObjectRegistry getAngularObjectRegistry() { + return angularObjectRegistry; + } + + public void setAngularObjectRegistry(AngularObjectRegistry angularObjectRegistry) { + this.angularObjectRegistry = angularObjectRegistry; + } + public void close() { for (Interpreter intp : this) { intp.close(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index e905d5f47b1..e32344874d8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -119,7 +119,7 @@ private synchronized void init() { } } - int rc = interpreterProcess.reference(); + int rc = interpreterProcess.reference(getInterpreterGroup()); synchronized (interpreterProcess) { // when first process created diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index a128cd7adf8..fb38cb46847 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -29,10 +29,14 @@ import org.apache.commons.exec.environment.EnvironmentUtils; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; +import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.gson.Gson; + /** * */ @@ -48,6 +52,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler { private GenericObjectPool clientPool; private Map env; + private RemoteInterpreterEventPoller remoteInterpreterEventPoller; public RemoteInterpreterProcess(String intpRunner, String intpDir, Map env) { this.interpreterRunner = intpRunner; @@ -60,7 +65,7 @@ public int getPort() { return port; } - public int reference() { + public int reference(InterpreterGroup interpreterGroup) { synchronized (referenceCount) { if (executor == null) { // start server process @@ -108,6 +113,9 @@ public int reference() { } clientPool = new GenericObjectPool(new ClientFactory("localhost", port)); + + remoteInterpreterEventPoller = new RemoteInterpreterEventPoller(interpreterGroup, this); + remoteInterpreterEventPoller.start(); } return referenceCount.incrementAndGet(); } @@ -126,6 +134,8 @@ public int dereference() { int r = referenceCount.decrementAndGet(); if (r == 0) { logger.info("shutdown interpreter process"); + remoteInterpreterEventPoller.shutdown(); + // first try shutdown try { Client client = getClient(); @@ -205,4 +215,28 @@ public int getNumIdleClient() { return clientPool.getNumIdle(); } } + + /** + * Called when angular object is updated in client side to propagate + * change to the remote process + * @param name + * @param o + */ + public void updateRemoteAngularObject(String name, Object o) { + Client client = null; + try { + client = getClient(); + } catch (Exception e) { + logger.error("Can't update angular object", e); + } + + try { + Gson gson = new Gson(); + client.angularObjectUpdate(name, gson.toJson(o)); + } catch (TException e) { + logger.error("Can't update angular object", e); + } finally { + releaseClient(client); + } + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index ee4aa2d6308..17def52ff17 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -21,6 +21,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.net.URL; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; @@ -29,6 +30,10 @@ import org.apache.thrift.server.TThreadPoolServer; import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TTransportException; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObject.AngularObjectType; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.ClassloaderInterpreter; import org.apache.zeppelin.interpreter.Interpreter; @@ -39,6 +44,8 @@ import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.scheduler.Job; @@ -52,16 +59,17 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; - /** * */ public class RemoteInterpreterServer extends Thread - implements RemoteInterpreterService.Iface { + implements RemoteInterpreterService.Iface, AngularObjectRegistryListener { Logger logger = LoggerFactory.getLogger(RemoteInterpreterServer.class); - InterpreterGroup interpreterGroup = new InterpreterGroup(); + InterpreterGroup interpreterGroup; + AngularObjectRegistry angularObjectRegistry; + Gson gson = new Gson(); RemoteInterpreterService.Processor processor; @@ -69,8 +77,14 @@ public class RemoteInterpreterServer private int port; private TThreadPoolServer server; + List eventQueue = new LinkedList(); + public RemoteInterpreterServer(int port) throws TTransportException { this.port = port; + interpreterGroup = new InterpreterGroup(); + angularObjectRegistry = new AngularObjectRegistry(interpreterGroup.getId(), this); + interpreterGroup.setAngularObjectRegistry(angularObjectRegistry); + processor = new RemoteInterpreterService.Processor(this); TServerSocket serverTransport = new TServerSocket(port); server = new TThreadPoolServer( @@ -306,7 +320,8 @@ private InterpreterContext convert(RemoteInterpreterContext ric) { ric.getParagraphText(), (Map) gson.fromJson(ric.getConfig(), new TypeToken>() {}.getType()), - gson.fromJson(ric.getGui(), GUI.class)); + gson.fromJson(ric.getGui(), GUI.class), + interpreterGroup.getAngularObjectRegistry()); } private RemoteInterpreterResult convert(InterpreterResult result, @@ -339,4 +354,79 @@ public String getStatus(String jobId) } return "Unknown"; } + + + + @Override + public void onAdd(String interpreterGroupId, AngularObject object) { + sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.ANGULAR_OBJECT_ADD, gson.toJson(object))); + } + + @Override + public void onUpdate(String interpreterGroupId, AngularObject object) { + sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE, gson.toJson(object))); + } + + @Override + public void onRemove(String interpreterGroupId, AngularObject object) { + sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE, gson.toJson(object))); + } + + private void sendEvent(RemoteInterpreterEvent event) { + synchronized (eventQueue) { + eventQueue.add(event); + eventQueue.notifyAll(); + } + } + + @Override + public RemoteInterpreterEvent getEvent() throws TException { + synchronized (eventQueue) { + if (eventQueue.isEmpty()) { + try { + eventQueue.wait(1000); + } catch (InterruptedException e) { + } + } + + if (eventQueue.isEmpty()) { + return new RemoteInterpreterEvent(RemoteInterpreterEventType.NO_OP, ""); + } else { + return eventQueue.remove(0); + } + } + } + + /** + * called when object is updated in client (web) side. + * @param className + * @param name + * @param object + * @throws TException + */ + @Override + public void angularObjectUpdate(String name, String object) + throws TException { + AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry(); + AngularObject ao = registry.get(name); + if (ao == null) { + logger.error("Angular object {} not exists", name); + return; + } + + if (ao.getType() == AngularObjectType.STRING) { + String value = gson.fromJson(object, String.class); + ao.set(value, false); + } else if (ao.getType() == AngularObjectType.MAP) { + Map value = gson.fromJson(object, + new TypeToken>() { + }.getType()); + ao.set(value, false); + } else { + logger.error("Update angular object type {} not supported", ao.getType()); + } + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java index a64395f1505..398cebf5aca 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java @@ -54,6 +54,10 @@ public interface Iface { public String getStatus(String jobId) throws org.apache.thrift.TException; + public RemoteInterpreterEvent getEvent() throws org.apache.thrift.TException; + + public void angularObjectUpdate(String name, String object) throws org.apache.thrift.TException; + } public interface AsyncIface { @@ -78,6 +82,10 @@ public interface AsyncIface { public void getStatus(String jobId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void getEvent(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void angularObjectUpdate(String name, String object, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + } public static class Client extends org.apache.thrift.TServiceClient implements Iface { @@ -321,6 +329,49 @@ public String recv_getStatus() throws org.apache.thrift.TException throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getStatus failed: unknown result"); } + public RemoteInterpreterEvent getEvent() throws org.apache.thrift.TException + { + send_getEvent(); + return recv_getEvent(); + } + + public void send_getEvent() throws org.apache.thrift.TException + { + getEvent_args args = new getEvent_args(); + sendBase("getEvent", args); + } + + public RemoteInterpreterEvent recv_getEvent() throws org.apache.thrift.TException + { + getEvent_result result = new getEvent_result(); + receiveBase(result, "getEvent"); + if (result.isSetSuccess()) { + return result.success; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getEvent failed: unknown result"); + } + + public void angularObjectUpdate(String name, String object) throws org.apache.thrift.TException + { + send_angularObjectUpdate(name, object); + recv_angularObjectUpdate(); + } + + public void send_angularObjectUpdate(String name, String object) throws org.apache.thrift.TException + { + angularObjectUpdate_args args = new angularObjectUpdate_args(); + args.setName(name); + args.setObject(object); + sendBase("angularObjectUpdate", args); + } + + public void recv_angularObjectUpdate() throws org.apache.thrift.TException + { + angularObjectUpdate_result result = new angularObjectUpdate_result(); + receiveBase(result, "angularObjectUpdate"); + return; + } + } public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { @@ -677,6 +728,70 @@ public String getResult() throws org.apache.thrift.TException { } } + public void getEvent(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + getEvent_call method_call = new getEvent_call(resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getEvent_call extends org.apache.thrift.async.TAsyncMethodCall { + public getEvent_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getEvent", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getEvent_args args = new getEvent_args(); + args.write(prot); + prot.writeMessageEnd(); + } + + public RemoteInterpreterEvent getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getEvent(); + } + } + + public void angularObjectUpdate(String name, String object, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + angularObjectUpdate_call method_call = new angularObjectUpdate_call(name, object, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class angularObjectUpdate_call extends org.apache.thrift.async.TAsyncMethodCall { + private String name; + private String object; + public angularObjectUpdate_call(String name, String object, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.name = name; + this.object = object; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("angularObjectUpdate", org.apache.thrift.protocol.TMessageType.CALL, 0)); + angularObjectUpdate_args args = new angularObjectUpdate_args(); + args.setName(name); + args.setObject(object); + args.write(prot); + prot.writeMessageEnd(); + } + + public void getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_angularObjectUpdate(); + } + } + } public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { @@ -700,6 +815,8 @@ protected Processor(I iface, Map extends org.apache.thrift.ProcessFunction { + public getEvent() { + super("getEvent"); + } + + public getEvent_args getEmptyArgsInstance() { + return new getEvent_args(); + } + + protected boolean isOneway() { + return false; + } + + public getEvent_result getResult(I iface, getEvent_args args) throws org.apache.thrift.TException { + getEvent_result result = new getEvent_result(); + result.success = iface.getEvent(); + return result; + } + } + + public static class angularObjectUpdate extends org.apache.thrift.ProcessFunction { + public angularObjectUpdate() { + super("angularObjectUpdate"); + } + + public angularObjectUpdate_args getEmptyArgsInstance() { + return new angularObjectUpdate_args(); + } + + protected boolean isOneway() { + return false; + } + + public angularObjectUpdate_result getResult(I iface, angularObjectUpdate_args args) throws org.apache.thrift.TException { + angularObjectUpdate_result result = new angularObjectUpdate_result(); + iface.angularObjectUpdate(args.name, args.object); + return result; + } + } + } public static class createInterpreter_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { @@ -8171,4 +8328,1309 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getStatus_result str } + public static class getEvent_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getEvent_args"); + + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getEvent_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getEvent_argsTupleSchemeFactory()); + } + + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { +; + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getEvent_args.class, metaDataMap); + } + + public getEvent_args() { + } + + /** + * Performs a deep copy on other. + */ + public getEvent_args(getEvent_args other) { + } + + public getEvent_args deepCopy() { + return new getEvent_args(this); + } + + @Override + public void clear() { + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getEvent_args) + return this.equals((getEvent_args)that); + return false; + } + + public boolean equals(getEvent_args that) { + if (that == null) + return false; + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getEvent_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getEvent_args typedOther = (getEvent_args)other; + + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getEvent_args("); + boolean first = true; + + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getEvent_argsStandardSchemeFactory implements SchemeFactory { + public getEvent_argsStandardScheme getScheme() { + return new getEvent_argsStandardScheme(); + } + } + + private static class getEvent_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getEvent_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getEvent_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getEvent_argsTupleSchemeFactory implements SchemeFactory { + public getEvent_argsTupleScheme getScheme() { + return new getEvent_argsTupleScheme(); + } + } + + private static class getEvent_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getEvent_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getEvent_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + } + } + + } + + public static class getEvent_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getEvent_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getEvent_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getEvent_resultTupleSchemeFactory()); + } + + public RemoteInterpreterEvent success; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RemoteInterpreterEvent.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getEvent_result.class, metaDataMap); + } + + public getEvent_result() { + } + + public getEvent_result( + RemoteInterpreterEvent success) + { + this(); + this.success = success; + } + + /** + * Performs a deep copy on other. + */ + public getEvent_result(getEvent_result other) { + if (other.isSetSuccess()) { + this.success = new RemoteInterpreterEvent(other.success); + } + } + + public getEvent_result deepCopy() { + return new getEvent_result(this); + } + + @Override + public void clear() { + this.success = null; + } + + public RemoteInterpreterEvent getSuccess() { + return this.success; + } + + public getEvent_result setSuccess(RemoteInterpreterEvent success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((RemoteInterpreterEvent)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getEvent_result) + return this.equals((getEvent_result)that); + return false; + } + + public boolean equals(getEvent_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(getEvent_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + getEvent_result typedOther = (getEvent_result)other; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getEvent_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (success != null) { + success.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getEvent_resultStandardSchemeFactory implements SchemeFactory { + public getEvent_resultStandardScheme getScheme() { + return new getEvent_resultStandardScheme(); + } + } + + private static class getEvent_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getEvent_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new RemoteInterpreterEvent(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getEvent_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getEvent_resultTupleSchemeFactory implements SchemeFactory { + public getEvent_resultTupleScheme getScheme() { + return new getEvent_resultTupleScheme(); + } + } + + private static class getEvent_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getEvent_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getEvent_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = new RemoteInterpreterEvent(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + } + } + + } + + public static class angularObjectUpdate_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectUpdate_args"); + + private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField OBJECT_FIELD_DESC = new org.apache.thrift.protocol.TField("object", org.apache.thrift.protocol.TType.STRING, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new angularObjectUpdate_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularObjectUpdate_argsTupleSchemeFactory()); + } + + public String name; // required + public String object; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + NAME((short)1, "name"), + OBJECT((short)2, "object"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // NAME + return NAME; + case 2: // OBJECT + return OBJECT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.OBJECT, new org.apache.thrift.meta_data.FieldMetaData("object", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectUpdate_args.class, metaDataMap); + } + + public angularObjectUpdate_args() { + } + + public angularObjectUpdate_args( + String name, + String object) + { + this(); + this.name = name; + this.object = object; + } + + /** + * Performs a deep copy on other. + */ + public angularObjectUpdate_args(angularObjectUpdate_args other) { + if (other.isSetName()) { + this.name = other.name; + } + if (other.isSetObject()) { + this.object = other.object; + } + } + + public angularObjectUpdate_args deepCopy() { + return new angularObjectUpdate_args(this); + } + + @Override + public void clear() { + this.name = null; + this.object = null; + } + + public String getName() { + return this.name; + } + + public angularObjectUpdate_args setName(String name) { + this.name = name; + return this; + } + + public void unsetName() { + this.name = null; + } + + /** Returns true if field name is set (has been assigned a value) and false otherwise */ + public boolean isSetName() { + return this.name != null; + } + + public void setNameIsSet(boolean value) { + if (!value) { + this.name = null; + } + } + + public String getObject() { + return this.object; + } + + public angularObjectUpdate_args setObject(String object) { + this.object = object; + return this; + } + + public void unsetObject() { + this.object = null; + } + + /** Returns true if field object is set (has been assigned a value) and false otherwise */ + public boolean isSetObject() { + return this.object != null; + } + + public void setObjectIsSet(boolean value) { + if (!value) { + this.object = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case NAME: + if (value == null) { + unsetName(); + } else { + setName((String)value); + } + break; + + case OBJECT: + if (value == null) { + unsetObject(); + } else { + setObject((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case NAME: + return getName(); + + case OBJECT: + return getObject(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case NAME: + return isSetName(); + case OBJECT: + return isSetObject(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof angularObjectUpdate_args) + return this.equals((angularObjectUpdate_args)that); + return false; + } + + public boolean equals(angularObjectUpdate_args that) { + if (that == null) + return false; + + boolean this_present_name = true && this.isSetName(); + boolean that_present_name = true && that.isSetName(); + if (this_present_name || that_present_name) { + if (!(this_present_name && that_present_name)) + return false; + if (!this.name.equals(that.name)) + return false; + } + + boolean this_present_object = true && this.isSetObject(); + boolean that_present_object = true && that.isSetObject(); + if (this_present_object || that_present_object) { + if (!(this_present_object && that_present_object)) + return false; + if (!this.object.equals(that.object)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(angularObjectUpdate_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + angularObjectUpdate_args typedOther = (angularObjectUpdate_args)other; + + lastComparison = Boolean.valueOf(isSetName()).compareTo(typedOther.isSetName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, typedOther.name); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetObject()).compareTo(typedOther.isSetObject()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetObject()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.object, typedOther.object); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("angularObjectUpdate_args("); + boolean first = true; + + sb.append("name:"); + if (this.name == null) { + sb.append("null"); + } else { + sb.append(this.name); + } + first = false; + if (!first) sb.append(", "); + sb.append("object:"); + if (this.object == null) { + sb.append("null"); + } else { + sb.append(this.object); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class angularObjectUpdate_argsStandardSchemeFactory implements SchemeFactory { + public angularObjectUpdate_argsStandardScheme getScheme() { + return new angularObjectUpdate_argsStandardScheme(); + } + } + + private static class angularObjectUpdate_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.name = iprot.readString(); + struct.setNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // OBJECT + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.object = iprot.readString(); + struct.setObjectIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.name != null) { + oprot.writeFieldBegin(NAME_FIELD_DESC); + oprot.writeString(struct.name); + oprot.writeFieldEnd(); + } + if (struct.object != null) { + oprot.writeFieldBegin(OBJECT_FIELD_DESC); + oprot.writeString(struct.object); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class angularObjectUpdate_argsTupleSchemeFactory implements SchemeFactory { + public angularObjectUpdate_argsTupleScheme getScheme() { + return new angularObjectUpdate_argsTupleScheme(); + } + } + + private static class angularObjectUpdate_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetName()) { + optionals.set(0); + } + if (struct.isSetObject()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetName()) { + oprot.writeString(struct.name); + } + if (struct.isSetObject()) { + oprot.writeString(struct.object); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.name = iprot.readString(); + struct.setNameIsSet(true); + } + if (incoming.get(1)) { + struct.object = iprot.readString(); + struct.setObjectIsSet(true); + } + } + } + + } + + public static class angularObjectUpdate_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectUpdate_result"); + + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new angularObjectUpdate_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularObjectUpdate_resultTupleSchemeFactory()); + } + + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { +; + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectUpdate_result.class, metaDataMap); + } + + public angularObjectUpdate_result() { + } + + /** + * Performs a deep copy on other. + */ + public angularObjectUpdate_result(angularObjectUpdate_result other) { + } + + public angularObjectUpdate_result deepCopy() { + return new angularObjectUpdate_result(this); + } + + @Override + public void clear() { + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof angularObjectUpdate_result) + return this.equals((angularObjectUpdate_result)that); + return false; + } + + public boolean equals(angularObjectUpdate_result that) { + if (that == null) + return false; + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + public int compareTo(angularObjectUpdate_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + angularObjectUpdate_result typedOther = (angularObjectUpdate_result)other; + + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("angularObjectUpdate_result("); + boolean first = true; + + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class angularObjectUpdate_resultStandardSchemeFactory implements SchemeFactory { + public angularObjectUpdate_resultStandardScheme getScheme() { + return new angularObjectUpdate_resultStandardScheme(); + } + } + + private static class angularObjectUpdate_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class angularObjectUpdate_resultTupleSchemeFactory implements SchemeFactory { + public angularObjectUpdate_resultTupleScheme getScheme() { + return new angularObjectUpdate_resultTupleScheme(); + } + } + + private static class angularObjectUpdate_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + } + } + + } + } diff --git a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift index 051730e08e3..fcd0832ed77 100644 --- a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift +++ b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift @@ -35,6 +35,18 @@ struct RemoteInterpreterResult { 5: string gui // json serialized gui } +enum RemoteInterpreterEventType { + NO_OP = 1, + ANGULAR_OBJECT_ADD = 2, + ANGULAR_OBJECT_UPDATE = 3, + ANGULAR_OBJECT_REMOVE = 4 +} + +struct RemoteInterpreterEvent { + 1: RemoteInterpreterEventType type, + 2: string data // json serialized data +} + service RemoteInterpreterService { void createInterpreter(1: string className, 2: map properties); @@ -48,4 +60,7 @@ service RemoteInterpreterService { void shutdown(); string getStatus(1:string jobId); + + RemoteInterpreterEvent getEvent(); + void angularObjectUpdate(1: string name, 2: string object); } \ No newline at end of file diff --git a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java new file mode 100644 index 00000000000..4d21b55ad5e --- /dev/null +++ b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.display; + +import org.junit.Test; + +public class AngularObjectTest { + + @Test + public void test() { + + } + +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index 02dc224c4e0..b9deef11dd3 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -22,6 +22,7 @@ import java.util.HashMap; +import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.junit.Test; @@ -30,11 +31,12 @@ public class RemoteInterpreterProcessTest { @Test public void testStartStop() { + InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap()); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); - assertEquals(1, rip.reference()); - assertEquals(2, rip.reference()); + assertEquals(1, rip.reference(intpGroup)); + assertEquals(2, rip.reference(intpGroup)); assertEquals(true, rip.isRunning()); assertEquals(1, rip.dereference()); assertEquals(true, rip.isRunning()); @@ -44,8 +46,9 @@ public void testStartStop() { @Test public void testClientFactory() throws Exception { + InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap()); - rip.reference(); + rip.reference(intpGroup); assertEquals(0, rip.getNumActiveClient()); assertEquals(0, rip.getNumIdleClient()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index 58299bb6d95..952b377569f 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -30,6 +30,7 @@ import java.util.Properties; import org.apache.thrift.transport.TTransportException; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -109,7 +110,8 @@ public void testRemoteInterperterCall() throws TTransportException, IOException "title", "text", new HashMap(), - new GUI())); + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null))); intpB.open(); assertEquals(2, process.referenceCount()); @@ -159,7 +161,8 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException "title", "text", new HashMap(), - new GUI())); + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null))); assertEquals("500", ret.message()); ret = intpB.interpret("500", @@ -168,7 +171,8 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException "title", "text", new HashMap(), - new GUI())); + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null))); assertEquals("1000", ret.message()); long end = System.currentTimeMillis(); assertTrue(end - start >= 1000); @@ -231,7 +235,8 @@ protected Object jobRun() throws Throwable { "title", "text", new HashMap(), - new GUI())); + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null))); } @Override @@ -262,7 +267,8 @@ protected Object jobRun() throws Throwable { "title", "text", new HashMap(), - new GUI())); + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null))); } @Override @@ -333,7 +339,8 @@ protected Object jobRun() throws Throwable { "title", "text", new HashMap(), - new GUI())); + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null))); synchronized (results) { results.add(ret.message()); @@ -413,7 +420,8 @@ protected Object jobRun() throws Throwable { "title", "text", new HashMap(), - new GUI())); + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null))); synchronized (results) { results.add(ret.message()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index 2c13ab296b2..8832c881df5 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Properties; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -53,7 +54,7 @@ public void tearDown(){ @Test public void test() throws Exception { Properties p = new Properties(); - InterpreterGroup intpGroup = new InterpreterGroup(); + final InterpreterGroup intpGroup = new InterpreterGroup(); Map env = new HashMap(); env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); @@ -93,7 +94,8 @@ protected Object jobRun() throws Throwable { "title", "text", new HashMap(), - new GUI())); + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null))); return "1000"; } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 1c9aa03aacf..35e068ce36e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -297,7 +297,7 @@ public ZeppelinServer() throws Exception { this.schedulerFactory = new SchedulerFactory(); - this.replFactory = new InterpreterFactory(conf); + this.replFactory = new InterpreterFactory(conf, notebookServer); notebook = new Notebook(conf, schedulerFactory, replFactory, notebookServer); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java index a7b8b668bc0..e4626bf7721 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java @@ -90,6 +90,11 @@ public static enum OP { // @param notes serialized List object PARAGRAPH_REMOVE, + + ANGULAR_OBJECT_UPDATE, // [s-c] add/update angular object + ANGULAR_OBJECT_REMOVE, // [s-c] add angular object del + + ANGULAR_OBJECT_UPDATED // [c-s] angular object value updated } public OP op; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index db5733e7605..646e76f9acc 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -25,7 +25,11 @@ import java.util.Map; import java.util.Set; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.JobListenerFactory; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Notebook; @@ -50,7 +54,8 @@ * * @author anthonycorbacho */ -public class NotebookServer extends WebSocketServer implements JobListenerFactory { +public class NotebookServer extends WebSocketServer implements + JobListenerFactory, AngularObjectRegistryListener { private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class); private static final int DEFAULT_PORT = 8282; @@ -130,6 +135,9 @@ public void onMessage(WebSocket conn, String msg) { case COMPLETION: completion(conn, notebook, messagereceived); break; + case ANGULAR_OBJECT_UPDATED: + angularObjectUpdated(conn, notebook, messagereceived); + break; default: broadcastNoteList(); break; @@ -380,6 +388,41 @@ private void completion(WebSocket conn, Notebook notebook, Message fromMessage) conn.send(serializeMessage(resp)); } + /** + * When angular object updated from client + * @param conn + * @param notebook + * @param fromMessage + */ + private void angularObjectUpdated(WebSocket conn, Notebook notebook, + Message fromMessage) { + String noteId = (String) fromMessage.get("noteId"); + String interpreterGroupId = (String) fromMessage.get("interpreterGroupId"); + String varName = (String) fromMessage.get("name"); + Object varValue = fromMessage.get("value"); + + Note note = notebook.getNote(noteId); + List settings = note.getNoteReplLoader().getInterpreterSettings(); + for (InterpreterSetting setting : settings) { + if (setting.getInterpreterGroup() == null) { + continue; + } + + if (interpreterGroupId.equals(setting.getInterpreterGroup().getId())) { + AngularObjectRegistry angularObjectRegistry = setting + .getInterpreterGroup().getAngularObjectRegistry(); + AngularObject ao = angularObjectRegistry.get(varName); + if (ao == null) { + LOG.warn("Object {} is not binded", varName); + } else { + // path from client -> server + ao.set(varValue, false); + } + } + } + } + + private void moveParagraph(WebSocket conn, Notebook notebook, Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); @@ -497,4 +540,48 @@ public void afterStatusChange(Job job, Status before, Status after) { public JobListener getParagraphJobListener(Note note) { return new ParagraphJobListener(this, note); } + + @Override + public void onAdd(String interpreterGroupId, AngularObject object) { + onUpdate(interpreterGroupId, object); + } + + @Override + public void onUpdate(String interpreterGroupId, AngularObject object) { + Notebook notebook = notebook(); + List notes = notebook.getAllNotes(); + for (Note note : notes) { + List intpSettings = note.getNoteReplLoader() + .getInterpreterSettings(); + + if (intpSettings.isEmpty()) continue; + + for (InterpreterSetting setting : intpSettings) { + + if (setting.getInterpreterGroup().getId().equals(interpreterGroupId)) { + broadcast(note.id(), new Message(OP.ANGULAR_OBJECT_UPDATE) + .put("angularObject", object) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", note.id())); + } + } + } + } + + @Override + public void onRemove(String interpreterGroupId, AngularObject object) { + Notebook notebook = notebook(); + List notes = notebook.getAllNotes(); + for (Note note : notes) { + List ids = note.getNoteReplLoader().getInterpreters(); + for (String id : ids) { + if (id.equals(interpreterGroupId)) { + broadcast( + note.id(), + new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", + object.getName())); + } + } + } + } } diff --git a/zeppelin-web/app/scripts/controllers/main.js b/zeppelin-web/app/scripts/controllers/main.js index 535cf7851d1..17c6e827a0a 100644 --- a/zeppelin-web/app/scripts/controllers/main.js +++ b/zeppelin-web/app/scripts/controllers/main.js @@ -24,7 +24,8 @@ */ angular.module('zeppelinWebApp') .controller('MainCtrl', function($scope, WebSocket, $rootScope, $window) { - + $rootScope.compiledScope = $scope.$new(true, $rootScope); + $rootScope.angularObjectRegistry = {}; $scope.WebSocketWaitingList = []; $scope.connected = false; $scope.looknfeel = 'default'; @@ -65,6 +66,8 @@ angular.module('zeppelinWebApp') $scope.$broadcast('updateProgress', data); } else if (op === 'COMPLETION_LIST') { $scope.$broadcast('completionList', data); + } else if (op === 'ANGULAR_OBJECT_UPDATE') { + $scope.$broadcast('angularObjectUpdate', data); } }); diff --git a/zeppelin-web/app/scripts/controllers/notebook.js b/zeppelin-web/app/scripts/controllers/notebook.js index cc295dcb9ce..4300acc463f 100644 --- a/zeppelin-web/app/scripts/controllers/notebook.js +++ b/zeppelin-web/app/scripts/controllers/notebook.js @@ -442,4 +442,35 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro return true; } }; + + $scope.$on('angularObjectUpdate', function(event, data) { + if (data.noteId === $scope.note.id) { + var scope = $rootScope.compiledScope; + var varName = data.angularObject.name; + + $rootScope.angularObjectRegistry[varName] = { + interpreterGroupId : data.interpreterGroupId + }; + scope[varName] = data.angularObject.object; + + scope.$watch(varName, function(newValue, oldValue) { + console.log("value updated to %o %o", varName, newValue); + $rootScope.$emit('sendNewEvent', { + op: 'ANGULAR_OBJECT_UPDATED', + data: { + noteId: $routeParams.noteId, + name:varName, + value:newValue, + interpreterGroupId:$rootScope.angularObjectRegistry[varName].interpreterGroupId + } + }); + }); + } + }); + + var isFunction = function(functionToCheck) { + var getType = {}; + return functionToCheck && getType.toString.call(functionToCheck) === '[object Function]'; + } + }); diff --git a/zeppelin-web/app/scripts/controllers/paragraph.js b/zeppelin-web/app/scripts/controllers/paragraph.js index d99a9faadd6..3d26b249e6b 100644 --- a/zeppelin-web/app/scripts/controllers/paragraph.js +++ b/zeppelin-web/app/scripts/controllers/paragraph.js @@ -84,7 +84,8 @@ angular.module('zeppelinWebApp') if ($('#p'+$scope.paragraph.id+'_angular').length) { try { $('#p'+$scope.paragraph.id+'_angular').html($scope.paragraph.result.msg); - $compile($('#p'+$scope.paragraph.id+'_angular').contents())($scope); + + $compile($('#p'+$scope.paragraph.id+'_angular').contents())($rootScope.compiledScope); } catch(err) { console.log('ANGULAR rendering error %o', err); } @@ -1615,5 +1616,4 @@ angular.module('zeppelinWebApp') var redirectToUrl = location.protocol + '//' + location.host + '/#/notebook/' + noteId + '/paragraph/' + $scope.paragraph.id+'?asIframe'; $window.open(redirectToUrl); }; - }); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 7c81e909d6f..92480be3920 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -43,7 +43,10 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; +import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,15 +76,21 @@ public class InterpreterFactory { private InterpreterOption defaultOption; - public InterpreterFactory(ZeppelinConfiguration conf) throws InterpreterException, IOException { - this(conf, new InterpreterOption(true)); + AngularObjectRegistryListener angularObjectRegistryListener; + + public InterpreterFactory(ZeppelinConfiguration conf, + AngularObjectRegistryListener angularObjectRegistryListener) + throws InterpreterException, IOException { + this(conf, new InterpreterOption(true), angularObjectRegistryListener); } - public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption) + public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption, + AngularObjectRegistryListener angularObjectRegistryListener) throws InterpreterException, IOException { this.conf = conf; this.defaultOption = defaultOption; + this.angularObjectRegistryListener = angularObjectRegistryListener; String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS); interpreterClassList = replsConf.split(","); @@ -338,7 +347,24 @@ private InterpreterGroup createInterpreterGroup(String groupName, InterpreterOption option, Properties properties) throws InterpreterException { + + AngularObjectRegistry angularObjectRegistry; + InterpreterGroup interpreterGroup = new InterpreterGroup(); + if (option.isRemote()) { + angularObjectRegistry = new RemoteAngularObjectRegistry( + interpreterGroup.getId(), + angularObjectRegistryListener, + interpreterGroup + ); + } else { + angularObjectRegistry = new AngularObjectRegistry( + interpreterGroup.getId(), + angularObjectRegistryListener); + } + + interpreterGroup.setAngularObjectRegistry(angularObjectRegistry); + for (String className : interpreterClassList) { Set keys = Interpreter.registeredInterpreters.keySet(); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index e0986bf9d74..e5599e00eb1 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -23,11 +23,13 @@ import java.util.Map; import java.util.Random; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.JobListener; @@ -207,11 +209,19 @@ protected boolean jobAbort() { } private InterpreterContext getInterpreterContext() { + AngularObjectRegistry registry = null; + + if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) { + InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0); + registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry(); + } + InterpreterContext interpreterContext = new InterpreterContext(getId(), this.getTitle(), this.getText(), this.getConfig(), - this.settings); + this.settings, + registry); return interpreterContext; } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 5199300bc2f..dc3ac3b62fd 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -54,8 +54,8 @@ public void setUp() throws Exception { System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); conf = new ZeppelinConfiguration(); - factory = new InterpreterFactory(conf, new InterpreterOption(false)); - context = new InterpreterContext("id", "title", "text", null, null); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null); + context = new InterpreterContext("id", "title", "text", null, null, null); } @@ -122,7 +122,7 @@ public void testSaveLoad() throws InterpreterException, IOException { factory.add("newsetting", "mock1", new InterpreterOption(false), new Properties()); assertEquals(3, factory.get().size()); - InterpreterFactory factory2 = new InterpreterFactory(conf); + InterpreterFactory factory2 = new InterpreterFactory(conf, null); assertEquals(3, factory2.get().size()); } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 88af541d9b5..8d2c65ad114 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -69,7 +69,7 @@ public void setUp() throws Exception { MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - factory = new InterpreterFactory(conf, new InterpreterOption(false)); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null); notebook = new Notebook(conf, schedulerFactory, factory, this); } @@ -108,7 +108,7 @@ public void testPersist() throws IOException, SchedulerException{ p1.setText("hello world"); note.persist(); - Notebook notebook2 = new Notebook(conf, schedulerFactory, new InterpreterFactory(conf), this); + Notebook notebook2 = new Notebook(conf, schedulerFactory, new InterpreterFactory(conf, null), this); assertEquals(1, notebook2.getAllNotes().size()); } From c2881982ede75a106287dc5eb116c84527604385 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 5 Apr 2015 01:29:43 +0900 Subject: [PATCH 05/21] ZEPPELIN-25 send scope variables when loading note --- .../display/AngularObjectRegistry.java | 10 ++++ .../zeppelin/socket/NotebookServer.java | 48 +++++++++++++++++++ .../app/scripts/controllers/notebook.js | 36 +++++++------- 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java index 013cbaf8a49..9ccfede749e 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java @@ -18,6 +18,8 @@ package com.nflabs.zeppelin.display; import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; /** @@ -73,6 +75,14 @@ public AngularObject get(String name) { } } + public List getAll() { + List all = new LinkedList(); + synchronized (registry) { + all.addAll(registry.values()); + } + return all; + } + @Override public void updated(AngularObject updatedObject) { if (listener != null) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 646e76f9acc..f8258e9fbcc 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -227,6 +227,19 @@ private String getOpenNoteId(WebSocket socket) { return id; } + private void broadcastToNoteBindedInterpreter(String interpreterGroupId, Message m) { + Notebook notebook = notebook(); + List notes = notebook.getAllNotes(); + for (Note note : notes) { + List ids = note.getNoteReplLoader().getInterpreters(); + for (String id : ids) { + if (id.equals(interpreterGroupId)) { + broadcast(note.id(), m); + } + } + } + } + private void broadcast(String noteId, Message m) { LOG.info("SEND >> " + m.op); synchronized (noteSocketMap) { @@ -271,9 +284,11 @@ private void sendNote(WebSocket conn, Notebook notebook, Message fromMessage) { return; } Note note = notebook.getNote(noteId); + if (note != null) { addConnectionToNote(note.id(), conn); conn.send(serializeMessage(new Message(OP.NOTE).put("note", note))); + sendAllAngularObjects(note, conn); } } @@ -417,6 +432,21 @@ private void angularObjectUpdated(WebSocket conn, Notebook notebook, } else { // path from client -> server ao.set(varValue, false); + + synchronized (noteSocketMap) { + List socketLists = noteSocketMap.get(noteId); + if (socketLists == null || socketLists.size() == 0) { + return; + } + for (WebSocket c : socketLists) { + if (c.equals(conn)) continue; + + c.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) + .put("angularObject", ao) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", noteId))); + } + } } } } @@ -541,6 +571,24 @@ public JobListener getParagraphJobListener(Note note) { return new ParagraphJobListener(this, note); } + private void sendAllAngularObjects(Note note, WebSocket conn) { + List settings = note.getNoteReplLoader().getInterpreterSettings(); + if (settings == null || settings.size() ==0) { + return; + } + + for (InterpreterSetting intpSetting : settings) { + AngularObjectRegistry registry = intpSetting.getInterpreterGroup().getAngularObjectRegistry(); + List objects = registry.getAll(); + for (AngularObject object : objects) { + conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) + .put("angularObject", object) + .put("interpreterGroupId", intpSetting.getInterpreterGroup().getId()) + .put("noteId", note.id()))); + } + } + } + @Override public void onAdd(String interpreterGroupId, AngularObject object) { onUpdate(interpreterGroupId, object); diff --git a/zeppelin-web/app/scripts/controllers/notebook.js b/zeppelin-web/app/scripts/controllers/notebook.js index 4300acc463f..7b965336301 100644 --- a/zeppelin-web/app/scripts/controllers/notebook.js +++ b/zeppelin-web/app/scripts/controllers/notebook.js @@ -448,24 +448,28 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro var scope = $rootScope.compiledScope; var varName = data.angularObject.name; - $rootScope.angularObjectRegistry[varName] = { - interpreterGroupId : data.interpreterGroupId - }; - scope[varName] = data.angularObject.object; - - scope.$watch(varName, function(newValue, oldValue) { - console.log("value updated to %o %o", varName, newValue); - $rootScope.$emit('sendNewEvent', { - op: 'ANGULAR_OBJECT_UPDATED', - data: { - noteId: $routeParams.noteId, - name:varName, - value:newValue, - interpreterGroupId:$rootScope.angularObjectRegistry[varName].interpreterGroupId - } + if (!$rootScope.angularObjectRegistry[varName]) { + var clearWatcher = scope.$watch(varName, function(newValue, oldValue) { + console.log("value updated to %o %o", varName, newValue); + $rootScope.$emit('sendNewEvent', { + op: 'ANGULAR_OBJECT_UPDATED', + data: { + noteId: $routeParams.noteId, + name:varName, + value:newValue, + interpreterGroupId:$rootScope.angularObjectRegistry[varName].interpreterGroupId + } + }); }); - }); + + $rootScope.angularObjectRegistry[varName] = { + interpreterGroupId : data.interpreterGroupId, + clearWatcher : clearWatcher + }; + } + scope[varName] = data.angularObject.object; } + }); var isFunction = function(functionToCheck) { From 5954e292bff5774392868f010bffe7af576e8fc4 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 5 Apr 2015 11:48:24 +0900 Subject: [PATCH 06/21] ZEPPELIN-25 save/restore angular object registry snapshot to the notebook file --- .../display/AngularObjectRegistry.java | 6 +- .../interpreter/InterpreterGroup.java | 7 +- .../remote/RemoteInterpreterProcess.java | 3 + .../zeppelin/socket/NotebookServer.java | 1 + .../interpreter/InterpreterFactory.java | 35 +++++---- .../interpreter/InterpreterSetting.java | 10 +-- .../org/apache/zeppelin/notebook/Note.java | 27 ++++++- .../apache/zeppelin/notebook/Notebook.java | 76 +++++++++++++++++++ 8 files changed, 141 insertions(+), 24 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java index 9ccfede749e..74ac6523949 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java @@ -43,11 +43,15 @@ public AngularObjectRegistryListener getListener() { } public AngularObject add(String name, Object o) { + return add(name, o, true); + } + + public AngularObject add(String name, Object o, boolean emit) { AngularObject ao = createNewAngularObject(name, o); synchronized (registry) { registry.put(name, ao); - if (listener != null) { + if (listener != null && emit) { listener.onAdd(interpreterId, ao); } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java index 3c882b18bfa..888f94e450e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java @@ -32,8 +32,12 @@ public class InterpreterGroup extends LinkedList{ AngularObjectRegistry angularObjectRegistry; + public InterpreterGroup(String id) { + this.id = id; + } + public InterpreterGroup() { - this.id = getId(); + getId(); } private static String generateId() { @@ -50,7 +54,6 @@ public String getId() { } } - public Properties getProperty() { Properties p = new Properties(); for (Interpreter intp : this) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index fb38cb46847..47b58f50694 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -226,6 +226,9 @@ public void updateRemoteAngularObject(String name, Object o) { Client client = null; try { client = getClient(); + } catch (NullPointerException e) { + // remote process not started + return; } catch (Exception e) { logger.error("Can't update angular object", e); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index f8258e9fbcc..e06cd9df3d8 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -597,6 +597,7 @@ public void onAdd(String interpreterGroupId, AngularObject object) { @Override public void onUpdate(String interpreterGroupId, AngularObject object) { Notebook notebook = notebook(); + List notes = notebook.getAllNotes(); for (Note note : notes) { List intpSettings = note.getNoteReplLoader() diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 92480be3920..c8fc485d281 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -226,17 +226,20 @@ private void loadFromFile() throws IOException { // previously created setting should turn this feature on here. setting.getOption().setRemote(true); - InterpreterGroup interpreterGroup = createInterpreterGroup( - setting.getGroup(), - setting.getOption(), - setting.getProperties()); + InterpreterSetting intpSetting = new InterpreterSetting( setting.id(), setting.getName(), setting.getGroup(), + setting.getOption()); + + InterpreterGroup interpreterGroup = createInterpreterGroup( + setting.id(), + setting.getGroup(), setting.getOption(), - interpreterGroup); + setting.getProperties()); + intpSetting.setInterpreterGroup(interpreterGroup); interpreterSettings.put(k, intpSetting); } @@ -329,37 +332,41 @@ public InterpreterGroup add(String name, String groupName, InterpreterOption option, Properties properties) throws InterpreterException, IOException { synchronized (interpreterSettings) { - InterpreterGroup interpreterGroup = createInterpreterGroup(groupName, option, properties); InterpreterSetting intpSetting = new InterpreterSetting( name, groupName, - option, - interpreterGroup); - interpreterSettings.put(intpSetting.id(), intpSetting); + option); + InterpreterGroup interpreterGroup = createInterpreterGroup( + intpSetting.id(), groupName, option, properties); + + intpSetting.setInterpreterGroup(interpreterGroup); + + interpreterSettings.put(intpSetting.id(), intpSetting); saveToFile(); return interpreterGroup; } } - private InterpreterGroup createInterpreterGroup(String groupName, + private InterpreterGroup createInterpreterGroup(String id, + String groupName, InterpreterOption option, Properties properties) throws InterpreterException { AngularObjectRegistry angularObjectRegistry; - InterpreterGroup interpreterGroup = new InterpreterGroup(); + InterpreterGroup interpreterGroup = new InterpreterGroup(id); if (option.isRemote()) { angularObjectRegistry = new RemoteAngularObjectRegistry( - interpreterGroup.getId(), + id, angularObjectRegistryListener, interpreterGroup ); } else { angularObjectRegistry = new AngularObjectRegistry( - interpreterGroup.getId(), + id, angularObjectRegistryListener); } @@ -506,6 +513,7 @@ public void setPropertyAndRestart(String id, InterpreterOption option, intpsetting.setOption(option); InterpreterGroup interpreterGroup = createInterpreterGroup( + intpsetting.id(), intpsetting.getGroup(), option, properties); intpsetting.setInterpreterGroup(interpreterGroup); saveToFile(); @@ -525,6 +533,7 @@ public void restart(String id) { intpsetting.getInterpreterGroup().destroy(); InterpreterGroup interpreterGroup = createInterpreterGroup( + intpsetting.id(), intpsetting.getGroup(), intpsetting.getOption(), intpsetting.getProperties()); intpsetting.setInterpreterGroup(interpreterGroup); } else { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 04785aa4eec..301ed238fb6 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -36,21 +36,17 @@ public class InterpreterSetting { public InterpreterSetting(String id, String name, String group, - InterpreterOption option, - InterpreterGroup interpreterGroup) { + InterpreterOption option) { this.id = id; this.name = name; this.group = group; - this.properties = interpreterGroup.getProperty(); this.option = option; - this.interpreterGroup = interpreterGroup; } public InterpreterSetting(String name, String group, - InterpreterOption option, - InterpreterGroup interpreterGroup) { - this(generateId(), name, group, option, interpreterGroup); + InterpreterOption option) { + this(generateId(), name, group, option); } public String id() { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 9204a07dcc2..fefd4cf00e2 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -33,7 +33,11 @@ import org.apache.commons.io.IOUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.utility.IdHashes; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.JobListener; @@ -54,6 +58,8 @@ public class Note implements Serializable, JobListener { private String name; private String id; + Map> angularObjects = new HashMap>(); + private transient NoteInterpreterLoader replLoader; private transient ZeppelinConfiguration conf; private transient JobListenerFactory jobListenerFactory; @@ -110,6 +116,10 @@ public void setZeppelinConfiguration(ZeppelinConfiguration conf) { this.conf = conf; } + public Map> getAngularObjects() { + return angularObjects; + } + /** * Add paragraph last. * @@ -268,6 +278,21 @@ public List getParagraphs() { } } + private void snapshotAngularObjectRegistry() { + angularObjects = new HashMap>(); + + List settings = replLoader.getInterpreterSettings(); + if (settings == null || settings.size() == 0) { + return; + } + + for (InterpreterSetting setting : settings) { + InterpreterGroup intpGroup = setting.getInterpreterGroup(); + AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry(); + angularObjects.put(intpGroup.getId(), registry.getAll()); + } + } + public void persist() throws IOException { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); @@ -283,6 +308,7 @@ public void persist() throws IOException { File file = new File(conf.getNotebookDir() + "/" + id + "/note.json"); logger().info("Persist note {} into {}", id, file.getAbsolutePath()); + snapshotAngularObjectRegistry(); String json = gson.toJson(this); FileOutputStream out = new FileOutputStream(file); out.write(json.getBytes(conf.getString(ConfVars.ZEPPELIN_ENCODING))); @@ -320,7 +346,6 @@ public static Note load(String id, ZeppelinConfiguration conf, NoteInterpreterLo p.setStatus(Status.ABORT); } } - return note; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 2d9ba36300e..de7e79633ac 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -22,6 +22,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -29,7 +31,10 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; @@ -160,6 +165,10 @@ private void loadAllNotes() throws IOException { if (dirs == null) { return; } + + Map angularObjectSnapshot = + new HashMap(); + for (File f : dirs) { boolean isHidden = f.getName().startsWith("."); if (f.isDirectory() && !isHidden) { @@ -174,12 +183,79 @@ private void loadAllNotes() throws IOException { jobListenerFactory, quartzSched); noteInterpreterLoader.setNoteId(note.id()); + // restore angular object -------------- + Date lastUpdatedDate = new Date(0); + for (Paragraph p : note.getParagraphs()) { + if (p.getDateFinished() != null && + lastUpdatedDate.before(p.getDateFinished())) { + lastUpdatedDate = p.getDateFinished(); + } + } + + Map> savedObjects = note.getAngularObjects(); + + if (savedObjects != null) { + for (String intpGroupName : savedObjects.keySet()) { + List objectList = savedObjects.get(intpGroupName); + + for (AngularObject savedObject : objectList) { + SnapshotAngularObject snapshot = angularObjectSnapshot.get(savedObject.getName()); + if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) { + angularObjectSnapshot.put( + savedObject.getName(), + new SnapshotAngularObject( + intpGroupName, + savedObject, + lastUpdatedDate)); + } + } + } + } + synchronized (notes) { notes.put(note.id(), note); refreshCron(note.id()); } } } + + for (String name : angularObjectSnapshot.keySet()) { + SnapshotAngularObject snapshot = angularObjectSnapshot.get(name); + List settings = replFactory.get(); + for (InterpreterSetting setting : settings) { + InterpreterGroup intpGroup = setting.getInterpreterGroup(); + if (intpGroup.getId().equals(snapshot.getIntpGroupId())) { + AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry(); + if (registry.get(name) == null) { + registry.add(name, snapshot.getAngularObject().get(), false); + } + } + } + } + } + + class SnapshotAngularObject { + String intpGroupId; + AngularObject angularObject; + Date lastUpdate; + + public SnapshotAngularObject(String intpGroupId, + AngularObject angularObject, Date lastUpdate) { + super(); + this.intpGroupId = intpGroupId; + this.angularObject = angularObject; + this.lastUpdate = lastUpdate; + } + + public String getIntpGroupId() { + return intpGroupId; + } + public AngularObject getAngularObject() { + return angularObject; + } + public Date getLastUpdate() { + return lastUpdate; + } } public List getAllNotes() { From 6df7f23e7c95102cac4cab618babbaa45d01af08 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 5 Apr 2015 12:16:27 +0900 Subject: [PATCH 07/21] ZEPPELIN-25 broadcast angular object change to related notes --- .../zeppelin/socket/NotebookServer.java | 48 ++++++++----------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index e06cd9df3d8..b0111868220 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -416,36 +416,26 @@ private void angularObjectUpdated(WebSocket conn, Notebook notebook, String varName = (String) fromMessage.get("name"); Object varValue = fromMessage.get("value"); - Note note = notebook.getNote(noteId); - List settings = note.getNoteReplLoader().getInterpreterSettings(); - for (InterpreterSetting setting : settings) { - if (setting.getInterpreterGroup() == null) { - continue; - } + for (Note note : notebook.getAllNotes()) { + List settings = note.getNoteReplLoader().getInterpreterSettings(); + for (InterpreterSetting setting : settings) { + if (setting.getInterpreterGroup() == null) { + continue; + } - if (interpreterGroupId.equals(setting.getInterpreterGroup().getId())) { - AngularObjectRegistry angularObjectRegistry = setting - .getInterpreterGroup().getAngularObjectRegistry(); - AngularObject ao = angularObjectRegistry.get(varName); - if (ao == null) { - LOG.warn("Object {} is not binded", varName); - } else { - // path from client -> server - ao.set(varValue, false); - - synchronized (noteSocketMap) { - List socketLists = noteSocketMap.get(noteId); - if (socketLists == null || socketLists.size() == 0) { - return; - } - for (WebSocket c : socketLists) { - if (c.equals(conn)) continue; - - c.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) - .put("angularObject", ao) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", noteId))); - } + if (interpreterGroupId.equals(setting.getInterpreterGroup().getId())) { + AngularObjectRegistry angularObjectRegistry = setting + .getInterpreterGroup().getAngularObjectRegistry(); + AngularObject ao = angularObjectRegistry.get(varName); + if (ao == null) { + LOG.warn("Object {} is not binded", varName); + } else { + // path from client -> server + ao.set(varValue, false); + this.broadcast(note.id(), new Message(OP.ANGULAR_OBJECT_UPDATE) + .put("angularObject", ao) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", note.id())); } } } From 6ce8f364d2e2a748667c0018657d5351c2e20c7d Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 5 Apr 2015 17:29:37 +0900 Subject: [PATCH 08/21] ZEPPELIN-25 implement watcher --- .../zeppelin/spark/SparkInterpreter.java | 10 +++ .../zeppelin/spark/ZeppelinContext.java | 55 ++++++++++-- .../zeppelin/display/AngularObject.java | 47 +++++++++++ .../display/AngularObjectWatcher.java | 25 ++++++ .../zeppelin/scheduler/ExecutorFactory.java | 83 +++++++++++++++++++ .../zeppelin/scheduler/SchedulerFactory.java | 9 +- .../zeppelin/socket/NotebookServer.java | 13 +-- .../apache/zeppelin/notebook/Notebook.java | 1 - 8 files changed, 223 insertions(+), 20 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectWatcher.java create mode 100644 zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/scheduler/ExecutorFactory.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 71c5ab5ef74..f7b891dd9a8 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -462,6 +462,16 @@ public void open() { } } } + + // add some implicit conversion + intp.interpret("implicit def watcherConversion(watcher: (Object, Object) => Unit):" + + "com.nflabs.zeppelin.display.AngularObjectWatcher = {" + + " new com.nflabs.zeppelin.display.AngularObjectWatcher() {" + + " def watch(before:Object, after:Object) = {" + + " watcher(before, after)" + + " }" + + " }" + + "}"); } private List currentClassPath() { diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index 517831a4d4c..ebafd6e3a9a 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -29,6 +29,9 @@ import org.apache.spark.SparkContext; import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.hive.HiveContext; +import com.nflabs.zeppelin.display.AngularObject; +import com.nflabs.zeppelin.display.AngularObjectRegistry; +import com.nflabs.zeppelin.display.AngularObjectWatcher; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.display.Input.ParamOption; import org.apache.zeppelin.interpreter.InterpreterContext; @@ -251,22 +254,58 @@ public void setInterpreterContext(InterpreterContext interpreterContext) { this.interpreterContext = interpreterContext; } - -/* - public void angularBind(String name, sObject o) { - + public Object angular(String name) { + AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry(); + AngularObject ao = registry.get(name); + if (ao == null) { + return null; + } else { + return ao.get(); + } } - public void angularBind(String name, sObject o, sWatcher w) { + public void angularBind(String name, Object o) { + AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry(); + if (registry.get(name) == null) { + registry.add(name, o); + } else { + registry.get(name).set(o); + } + } + public void angularBind(String name, Object o, AngularObjectWatcher w) { + AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry(); + if (registry.get(name) == null) { + registry.add(name, o); + } else { + registry.get(name).set(o); + } + angularWatch(name, w); } - public void angularBind(String name, Function f) { + public void angularWatch(String name, AngularObjectWatcher w) { + AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry(); + if (registry.get(name) != null) { + registry.get(name).addWatcher(w); + } + } + public void angularUnwatch(String name, AngularObjectWatcher w) { + AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry(); + if (registry.get(name) != null) { + registry.get(name).removeWatcher(w); + } } - public void angularUnbind(sString name) { + public void angularUnwatch(String name) { + AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry(); + if (registry.get(name) != null) { + registry.get(name).clearAllWatchers(); + } + } + public void angularUnbind(String name) { + AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry(); + registry.remove(name); } - */ } diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java index 8d9f803e14a..f5076cebba3 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java @@ -17,7 +17,15 @@ package com.nflabs.zeppelin.display; +import java.util.LinkedList; +import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.nflabs.zeppelin.scheduler.ExecutorFactory; /** * @@ -38,6 +46,8 @@ public static enum AngularObjectType { private String name; private T object; private transient AngularObjectListener listener; + private transient List watchers + = new LinkedList(); private AngularObjectType type; protected AngularObject(String name, T o, @@ -91,10 +101,28 @@ public void set(T o) { } public void set(T o, boolean emit) { + final T before = object; + final T after = o; object = o; if (emit) { emit(); } + + List ws = new LinkedList(); + synchronized (watchers) { + ws.addAll(watchers); + } + + ExecutorService executor = ExecutorFactory.singleton().createOrGet("angularObjectWatcher", 50); + for (final AngularObjectWatcher w : ws) { + Logger logger = LoggerFactory.getLogger(AngularObject.class); + executor.submit(new Runnable() { + @Override + public void run() { + w.watch(before, after); + } + }); + } } public void setListener(AngularObjectListener listener) { @@ -104,4 +132,23 @@ public void setListener(AngularObjectListener listener) { public AngularObjectListener getListener() { return listener; } + + public void addWatcher(AngularObjectWatcher watcher) { + synchronized (watchers) { + watchers.add(watcher); + } + } + + public void removeWatcher(AngularObjectWatcher watcher) { + synchronized (watchers) { + watchers.remove(watcher); + } + } + + public void clearAllWatchers() { + synchronized (watchers) { + watchers.clear(); + } + } + } diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectWatcher.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectWatcher.java new file mode 100644 index 00000000000..35a7808dad3 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectWatcher.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.display; + +/** + * + */ +public interface AngularObjectWatcher { + public void watch(Object oldObject, Object newObject); +} diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/scheduler/ExecutorFactory.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/scheduler/ExecutorFactory.java new file mode 100644 index 00000000000..f8f90c73f59 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/scheduler/ExecutorFactory.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nflabs.zeppelin.scheduler; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * + */ +public class ExecutorFactory { + private static ExecutorFactory _executor; + private static Long _executorLock = new Long(0); + + Map executor = new HashMap(); + + public ExecutorFactory() { + + } + + public static ExecutorFactory singleton() { + if (_executor == null) { + synchronized (_executorLock) { + if (_executor == null) { + _executor = new ExecutorFactory(); + } + } + } + return _executor; + } + + public ExecutorService getDefaultExecutor() { + return createOrGet("default"); + } + + public ExecutorService createOrGet(String name) { + return createOrGet(name, 100); + } + + public ExecutorService createOrGet(String name, int numThread) { + synchronized (executor) { + if (!executor.containsKey(name)) { + executor.put(name, Executors.newScheduledThreadPool(numThread)); + } + return executor.get(name); + } + } + + public void shutdown(String name) { + synchronized (executor) { + if (executor.containsKey(name)) { + ExecutorService e = executor.get(name); + e.shutdown(); + executor.remove(name); + } + } + } + + + public void shutdownAll() { + synchronized (executor) { + for (String name : executor.keySet()){ + shutdown(name); + } + } + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java index 2556a819d9c..71769b4365d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java @@ -22,8 +22,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ExecutorService; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.slf4j.Logger; @@ -37,7 +36,7 @@ */ public class SchedulerFactory implements SchedulerListener { private final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class); - ScheduledExecutorService executor; + ExecutorService executor; Map schedulers = new LinkedHashMap(); private static SchedulerFactory singleton; @@ -59,11 +58,11 @@ public static SchedulerFactory singleton() { } public SchedulerFactory() throws Exception { - executor = Executors.newScheduledThreadPool(100); + executor = ExecutorFactory.singleton().createOrGet("schedulerFactory", 100); } public void destroy() { - executor.shutdown(); + ExecutorFactory.singleton().shutdown("schedulerFactory"); } public Scheduler createOrGetFIFOScheduler(String name) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index b0111868220..e490b807f2d 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -241,12 +241,14 @@ private void broadcastToNoteBindedInterpreter(String interpreterGroupId, Message } private void broadcast(String noteId, Message m) { - LOG.info("SEND >> " + m.op); synchronized (noteSocketMap) { List socketLists = noteSocketMap.get(noteId); if (socketLists == null || socketLists.size() == 0) { return; } + + LOG.info("SEND >> " + m.op); + for (WebSocket conn : socketLists) { conn.send(serializeMessage(m)); } @@ -563,7 +565,7 @@ public JobListener getParagraphJobListener(Note note) { private void sendAllAngularObjects(Note note, WebSocket conn) { List settings = note.getNoteReplLoader().getInterpreterSettings(); - if (settings == null || settings.size() ==0) { + if (settings == null || settings.size() == 0) { return; } @@ -572,9 +574,9 @@ private void sendAllAngularObjects(Note note, WebSocket conn) { List objects = registry.getAll(); for (AngularObject object : objects) { conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) - .put("angularObject", object) - .put("interpreterGroupId", intpSetting.getInterpreterGroup().getId()) - .put("noteId", note.id()))); + .put("angularObject", object) + .put("interpreterGroupId", intpSetting.getInterpreterGroup().getId()) + .put("noteId", note.id()))); } } } @@ -596,7 +598,6 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { if (intpSettings.isEmpty()) continue; for (InterpreterSetting setting : intpSettings) { - if (setting.getInterpreterGroup().getId().equals(interpreterGroupId)) { broadcast(note.id(), new Message(OP.ANGULAR_OBJECT_UPDATE) .put("angularObject", object) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index de7e79633ac..844763f7576 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -261,7 +261,6 @@ public Date getLastUpdate() { public List getAllNotes() { synchronized (notes) { List noteList = new ArrayList(notes.values()); - logger.info("" + noteList.size()); Collections.sort(noteList, new Comparator() { @Override public int compare(Object one, Object two) { From d4d270e431d4a687bb7eff4b9daf6de8c964a77a Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 5 Apr 2015 21:36:28 +0900 Subject: [PATCH 09/21] ZEPPELIN-25 add unittest --- .../display/AngularObjectRegistryTest.java | 67 +++++++++ .../remote/RemoteAngularObjectTest.java | 142 ++++++++++++++++++ .../remote/mock/MockInterpreterAngular.java | 115 ++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectRegistryTest.java create mode 100644 zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectTest.java create mode 100644 zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java diff --git a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectRegistryTest.java b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectRegistryTest.java new file mode 100644 index 00000000000..85f951ce60b --- /dev/null +++ b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectRegistryTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.display; + +import static org.junit.Assert.assertEquals; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Test; + +public class AngularObjectRegistryTest { + + @Test + public void testBasic() { + final AtomicInteger onAdd = new AtomicInteger(0); + final AtomicInteger onUpdate = new AtomicInteger(0); + final AtomicInteger onRemove = new AtomicInteger(0); + + AngularObjectRegistry registry = new AngularObjectRegistry("intpId", + new AngularObjectRegistryListener() { + + @Override + public void onAdd(String interpreterGroupId, AngularObject object) { + onAdd.incrementAndGet(); + } + + @Override + public void onUpdate(String interpreterGroupId, AngularObject object) { + onUpdate.incrementAndGet(); + } + + @Override + public void onRemove(String interpreterGroupId, AngularObject object) { + onRemove.incrementAndGet(); + } + }); + + registry.add("name1", "value1"); + assertEquals(1, registry.getAll().size()); + assertEquals(1, onAdd.get()); + assertEquals(0, onUpdate.get()); + + registry.get("name1").set("newValue"); + assertEquals(1, onUpdate.get()); + + registry.remove("name1"); + assertEquals(0, registry.getAll().size()); + assertEquals(1, onRemove.get()); + + assertEquals(null, registry.get("name1")); + } +} diff --git a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectTest.java new file mode 100644 index 00000000000..f98a1c9f41d --- /dev/null +++ b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.interpreter.remote; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.util.HashMap; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.nflabs.zeppelin.display.AngularObject; +import com.nflabs.zeppelin.display.AngularObjectRegistry; +import com.nflabs.zeppelin.display.AngularObjectRegistryListener; +import com.nflabs.zeppelin.display.GUI; +import com.nflabs.zeppelin.interpreter.InterpreterContext; +import com.nflabs.zeppelin.interpreter.InterpreterGroup; +import com.nflabs.zeppelin.interpreter.InterpreterResult; +import com.nflabs.zeppelin.interpreter.remote.mock.MockInterpreterAngular; + +public class RemoteAngularObjectTest implements AngularObjectRegistryListener { + private InterpreterGroup intpGroup; + private HashMap env; + private RemoteInterpreter intp; + private InterpreterContext context; + private RemoteAngularObjectRegistry localRegistry; + + private AtomicInteger onAdd; + private AtomicInteger onUpdate; + private AtomicInteger onRemove; + + @Before + public void setUp() throws Exception { + onAdd = new AtomicInteger(0); + onUpdate = new AtomicInteger(0); + onRemove = new AtomicInteger(0); + + intpGroup = new InterpreterGroup("intpId"); + localRegistry = new RemoteAngularObjectRegistry("intpId", this, intpGroup); + intpGroup.setAngularObjectRegistry(localRegistry); + env = new HashMap(); + env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); + + Properties p = new Properties(); + + intp = new RemoteInterpreter( + p, + MockInterpreterAngular.class.getName(), + new File("../bin/interpreter.sh").getAbsolutePath(), + "fake", + env + ); + + intpGroup.add(intp); + intp.setInterpreterGroup(intpGroup); + + context = new InterpreterContext( + "id", + "title", + "text", + new HashMap(), + new GUI(), + new AngularObjectRegistry(intpGroup.getId(), null)); + + intp.open(); + } + + @After + public void tearDown() throws Exception { + intp.close(); + intpGroup.clone(); + intpGroup.destroy(); + } + + @Test + public void testAngularObjectCRUD() throws InterruptedException { + InterpreterResult ret = intp.interpret("get", context); + Thread.sleep(500); // waitFor eventpoller pool event + String[] result = ret.message().split(" "); + assertEquals("0", result[0]); // size of registry + assertEquals("0", result[1]); // num watcher called + + // create object + ret = intp.interpret("add n1 v1", context); + Thread.sleep(500); + result = ret.message().split(" "); + assertEquals("1", result[0]); // size of registry + assertEquals("0", result[1]); // num watcher called + assertEquals("v1", localRegistry.get("n1").get()); + + // update object + ret = intp.interpret("update n1 v11", context); + result = ret.message().split(" "); + Thread.sleep(500); + assertEquals("1", result[0]); // size of registry + assertEquals("1", result[1]); // num watcher called + assertEquals("v11", localRegistry.get("n1").get()); + + // remove object + ret = intp.interpret("remove n1", context); + result = ret.message().split(" "); + Thread.sleep(500); + assertEquals("0", result[0]); // size of registry + assertEquals("1", result[1]); // num watcher called + assertEquals(null, localRegistry.get("n1")); + } + + @Override + public void onAdd(String interpreterGroupId, AngularObject object) { + onAdd.incrementAndGet(); + } + + @Override + public void onUpdate(String interpreterGroupId, AngularObject object) { + onUpdate.incrementAndGet(); + } + + @Override + public void onRemove(String interpreterGroupId, AngularObject object) { + onRemove.incrementAndGet(); + } + +} diff --git a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java new file mode 100644 index 00000000000..ea5edf0ee56 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nflabs.zeppelin.interpreter.remote.mock; + +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import com.nflabs.zeppelin.display.AngularObjectRegistry; +import com.nflabs.zeppelin.display.AngularObjectWatcher; +import com.nflabs.zeppelin.interpreter.Interpreter; +import com.nflabs.zeppelin.interpreter.InterpreterContext; +import com.nflabs.zeppelin.interpreter.InterpreterPropertyBuilder; +import com.nflabs.zeppelin.interpreter.InterpreterResult; +import com.nflabs.zeppelin.interpreter.InterpreterResult.Code; + +public class MockInterpreterAngular extends Interpreter implements AngularObjectWatcher { + static { + Interpreter.register( + "angularTest", + "angular", + MockInterpreterA.class.getName(), + new InterpreterPropertyBuilder() + .add("p1", "v1", "property1").build()); + + } + + AtomicInteger numWatch = new AtomicInteger(0); + + public MockInterpreterAngular(Properties property) { + super(property); + } + + @Override + public void open() { + } + + @Override + public void close() { + + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + String[] stmt = st.split(" "); + String cmd = stmt[0]; + String name = null; + if (stmt.length >= 2) { + name = stmt[1]; + } + String value = null; + if (stmt.length == 3) { + value = stmt[2]; + } + + AngularObjectRegistry registry = context.getAngularObjectRegistry(); + + if (cmd.equals("add")) { + registry.add(name, value); + registry.get(name).addWatcher(this); + } else if (cmd.equalsIgnoreCase("update")) { + registry.get(name).set(value); + } else if (cmd.equals("remove")) { + registry.remove(name); + } + + try { + Thread.sleep(500); // wait for watcher executed + } catch (InterruptedException e) { + } + + String msg = registry.getAll().size() + " " + Integer.toString(numWatch.get()); + return new InterpreterResult(Code.SUCCESS, msg); + } + + @Override + public void cancel(InterpreterContext context) { + } + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public List completion(String buf, int cursor) { + return null; + } + + @Override + public void watch(Object oldObject, Object newObject) { + numWatch.incrementAndGet(); + } + +} \ No newline at end of file From 4d32d1971bc6548b34d31128be96ec30b01a4261 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 5 Apr 2015 21:36:46 +0900 Subject: [PATCH 10/21] ZEPPELIN-25 prevent watcher called multiple times --- .../zeppelin/display/AngularObject.java | 4 -- .../display/AngularObjectRegistry.java | 30 ++++++----- .../remote/RemoteAngularObject.java | 12 +++-- .../remote/RemoteAngularObjectRegistry.java | 3 +- .../remote/RemoteInterpreterEventPoller.java | 8 ++- .../zeppelin/display/AngularObjectTest.java | 50 ++++++++++++++++++- .../zeppelin/socket/NotebookServer.java | 28 +++++++++-- zeppelin-web/app/scripts/controllers/main.js | 3 +- .../app/scripts/controllers/notebook.js | 32 ++++++++---- 9 files changed, 133 insertions(+), 37 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java index f5076cebba3..84d1459d3ac 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java @@ -22,9 +22,6 @@ import java.util.Map; import java.util.concurrent.ExecutorService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.nflabs.zeppelin.scheduler.ExecutorFactory; /** @@ -115,7 +112,6 @@ public void set(T o, boolean emit) { ExecutorService executor = ExecutorFactory.singleton().createOrGet("angularObjectWatcher", 50); for (final AngularObjectWatcher w : ws) { - Logger logger = LoggerFactory.getLogger(AngularObject.class); executor.submit(new Runnable() { @Override public void run() { diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java index 74ac6523949..df6c4d28b6e 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java @@ -26,16 +26,25 @@ * * */ -public class AngularObjectRegistry implements AngularObjectListener { +public class AngularObjectRegistry { Map registry = new HashMap(); private AngularObjectRegistryListener listener; private String interpreterId; - public AngularObjectRegistry(String interpreterId, - AngularObjectRegistryListener listener) { + AngularObjectListener angularObjectListener; + + public AngularObjectRegistry(final String interpreterId, + final AngularObjectRegistryListener listener) { this.interpreterId = interpreterId; this.listener = listener; - + angularObjectListener = new AngularObjectListener() { + @Override + public void updated(AngularObject updatedObject) { + if (listener != null) { + listener.onUpdate(interpreterId, updatedObject); + } + } + }; } public AngularObjectRegistryListener getListener() { @@ -60,7 +69,11 @@ public AngularObject add(String name, Object o, boolean emit) { } protected AngularObject createNewAngularObject(String name, Object o) { - return new AngularObject(name, o, this); + return new AngularObject(name, o, angularObjectListener); + } + + protected AngularObjectListener getAngularObjectListener() { + return angularObjectListener; } public AngularObject remove(String name) { @@ -87,13 +100,6 @@ public List getAll() { return all; } - @Override - public void updated(AngularObject updatedObject) { - if (listener != null) { - listener.onUpdate(interpreterId, updatedObject); - } - } - public String getInterpreterGroupId() { return interpreterId; } diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java index 72007f8690d..7299e5423af 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java @@ -36,9 +36,15 @@ public class RemoteAngularObject extends AngularObject { @Override public void set(Object o, boolean emit) { - super.set(o, emit); + set(o, emit, true); + } + + public void set(Object o, boolean emitWeb, boolean emitRemoteProcess) { + super.set(o, emitWeb); - // send updated value to remote interpreter - remoteInterpreterProcess.updateRemoteAngularObject(getName(), o); + if (emitRemoteProcess) { + // send updated value to remote interpreter + remoteInterpreterProcess.updateRemoteAngularObject(getName(), o); + } } } diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java index 352920fb29d..10405a41d54 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java @@ -60,7 +60,8 @@ protected AngularObject createNewAngularObject(String name, Object o) { if (remoteInterpreterProcess == null) { throw new RuntimeException("Remote Interpreter process not found"); } - return new RemoteAngularObject(name, o, getInterpreterGroupId(), this, + return new RemoteAngularObject(name, o, getInterpreterGroupId(), + getAngularObjectListener(), getRemoteInterpreterProcess()); } } diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 189ad085431..46899bf5df5 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -93,7 +93,13 @@ public void run() { } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE) { AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class); AngularObject localAngularObject = angularObjectRegistry.get(angularObject.getName()); - localAngularObject.set(angularObject.get()); + if (localAngularObject instanceof RemoteAngularObject) { + // to avoid ping-pong loop + ((RemoteAngularObject) localAngularObject).set( + angularObject.get(), true, false); + } else { + localAngularObject.set(angularObject.get()); + } } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) { AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class); angularObjectRegistry.remove(angularObject.getName()); diff --git a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java index 4d21b55ad5e..31de8bf570c 100644 --- a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java @@ -17,13 +17,61 @@ package com.nflabs.zeppelin.display; +import static org.junit.Assert.assertEquals; + +import java.util.concurrent.atomic.AtomicInteger; + import org.junit.Test; public class AngularObjectTest { @Test - public void test() { + public void testListener() { + final AtomicInteger updated = new AtomicInteger(0); + AngularObject ao = new AngularObject("name", "value", new AngularObjectListener() { + + @Override + public void updated(AngularObject updatedObject) { + updated.incrementAndGet(); + } + + }); + + assertEquals(0, updated.get()); + ao.set("newValue"); + assertEquals(1, updated.get()); + assertEquals("newValue", ao.get()); + + ao.set("newValue"); + assertEquals(2, updated.get()); + ao.set("newnewValue", false); + assertEquals(2, updated.get()); + assertEquals("newnewValue", ao.get()); } + @Test + public void testWatcher() throws InterruptedException { + final AtomicInteger updated = new AtomicInteger(0); + final AtomicInteger onWatch = new AtomicInteger(0); + AngularObject ao = new AngularObject("name", "value", new AngularObjectListener() { + @Override + public void updated(AngularObject updatedObject) { + updated.incrementAndGet(); + } + }); + + ao.addWatcher(new AngularObjectWatcher() { + @Override + public void watch(Object oldObject, Object newObject) { + onWatch.incrementAndGet(); + } + }); + + assertEquals(0, onWatch.get()); + ao.set("newValue"); + + Thread.sleep(500); + assertEquals(1, onWatch.get()); + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index e490b807f2d..45371afbea1 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -418,7 +418,9 @@ private void angularObjectUpdated(WebSocket conn, Notebook notebook, String varName = (String) fromMessage.get("name"); Object varValue = fromMessage.get("value"); - for (Note note : notebook.getAllNotes()) { + // propagate change to (Remote) AngularObjectRegistry + Note note = notebook.getNote(noteId); + if (note != null) { List settings = note.getNoteReplLoader().getInterpreterSettings(); for (InterpreterSetting setting : settings) { if (setting.getInterpreterGroup() == null) { @@ -434,11 +436,29 @@ private void angularObjectUpdated(WebSocket conn, Notebook notebook, } else { // path from client -> server ao.set(varValue, false); - this.broadcast(note.id(), new Message(OP.ANGULAR_OBJECT_UPDATE) + } + + break; + } + } + } + + // broadcast change to all web session that uses related interpreter. + for (Note n : notebook.getAllNotes()) { + List settings = note.getNoteReplLoader().getInterpreterSettings(); + for (InterpreterSetting setting : settings) { + if (setting.getInterpreterGroup() == null) { + continue; + } + + if (interpreterGroupId.equals(setting.getInterpreterGroup().getId())) { + AngularObjectRegistry angularObjectRegistry = setting + .getInterpreterGroup().getAngularObjectRegistry(); + AngularObject ao = angularObjectRegistry.get(varName); + this.broadcast(n.id(), new Message(OP.ANGULAR_OBJECT_UPDATE) .put("angularObject", ao) .put("interpreterGroupId", interpreterGroupId) - .put("noteId", note.id())); - } + .put("noteId", n.id())); } } } diff --git a/zeppelin-web/app/scripts/controllers/main.js b/zeppelin-web/app/scripts/controllers/main.js index 17c6e827a0a..4948b49659b 100644 --- a/zeppelin-web/app/scripts/controllers/main.js +++ b/zeppelin-web/app/scripts/controllers/main.js @@ -24,8 +24,7 @@ */ angular.module('zeppelinWebApp') .controller('MainCtrl', function($scope, WebSocket, $rootScope, $window) { - $rootScope.compiledScope = $scope.$new(true, $rootScope); - $rootScope.angularObjectRegistry = {}; + $rootScope.compiledScope = $scope.$new(true, $rootScope); $scope.WebSocketWaitingList = []; $scope.connected = false; $scope.looknfeel = 'default'; diff --git a/zeppelin-web/app/scripts/controllers/notebook.js b/zeppelin-web/app/scripts/controllers/notebook.js index 7b965336301..9a9fcd7ce99 100644 --- a/zeppelin-web/app/scripts/controllers/notebook.js +++ b/zeppelin-web/app/scripts/controllers/notebook.js @@ -44,6 +44,8 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro $scope.interpreterSettings = []; $scope.interpreterBindings = []; + var angularObjectRegistry = {}; + $scope.getCronOptionNameFromValue = function(value) { if (!value) { return ''; @@ -448,24 +450,36 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro var scope = $rootScope.compiledScope; var varName = data.angularObject.name; - if (!$rootScope.angularObjectRegistry[varName]) { - var clearWatcher = scope.$watch(varName, function(newValue, oldValue) { - console.log("value updated to %o %o", varName, newValue); + if (angular.equals(data.angularObject.object, scope[varName])) { + // return when update has no change + return; + } + + if (!angularObjectRegistry[varName]) { + angularObjectRegistry[varName] = { + interpreterGroupId : data.interpreterGroupId, + } + } + + angularObjectRegistry[varName].skipEmit = true; + + if (!angularObjectRegistry[varName].clearWatcher) { + angularObjectRegistry[varName].clearWatcher = scope.$watch(varName, function(newValue, oldValue) { + if (angularObjectRegistry[varName].skipEmit) { + angularObjectRegistry[varName].skipEmit = false; + return; + } + $rootScope.$emit('sendNewEvent', { op: 'ANGULAR_OBJECT_UPDATED', data: { noteId: $routeParams.noteId, name:varName, value:newValue, - interpreterGroupId:$rootScope.angularObjectRegistry[varName].interpreterGroupId + interpreterGroupId:angularObjectRegistry[varName].interpreterGroupId } }); }); - - $rootScope.angularObjectRegistry[varName] = { - interpreterGroupId : data.interpreterGroupId, - clearWatcher : clearWatcher - }; } scope[varName] = data.angularObject.object; } From 42ee479fcd66c151b74208b0dc7398498fc4fccc Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 6 Apr 2015 02:04:54 +0900 Subject: [PATCH 11/21] com.nflabs -> org.apache --- .../zeppelin/spark/SparkInterpreter.java | 4 ++-- .../apache/zeppelin/spark/ZeppelinContext.java | 6 +++--- .../zeppelin/display/AngularObject.java | 4 ++-- .../display/AngularObjectListener.java | 2 +- .../display/AngularObjectRegistry.java | 2 +- .../display/AngularObjectRegistryListener.java | 2 +- .../zeppelin/display/AngularObjectWatcher.java | 2 +- .../zeppelin/interpreter/InterpreterGroup.java | 2 +- .../remote/RemoteAngularObject.java | 6 +++--- .../remote/RemoteAngularObjectRegistry.java | 14 +++++++------- .../remote/RemoteInterpreterEventPoller.java | 14 +++++++------- .../thrift/RemoteInterpreterEvent.java | 2 +- .../thrift/RemoteInterpreterEventType.java | 2 +- .../zeppelin/scheduler/ExecutorFactory.java | 2 +- .../display/AngularObjectRegistryTest.java | 2 +- .../zeppelin/display/AngularObjectTest.java | 2 +- .../remote/RemoteAngularObjectTest.java | 18 +++++++++--------- .../remote/mock/MockInterpreterAngular.java | 18 +++++++++--------- .../apache/zeppelin/socket/NotebookServer.java | 2 +- 19 files changed, 53 insertions(+), 53 deletions(-) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/display/AngularObject.java (97%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/display/AngularObjectListener.java (96%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/display/AngularObjectRegistry.java (98%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/display/AngularObjectRegistryListener.java (96%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/display/AngularObjectWatcher.java (96%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/interpreter/remote/RemoteAngularObject.java (90%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java (84%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java (90%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java (99%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java (95%) rename zeppelin-interpreter/src/main/java/{com/nflabs => org/apache}/zeppelin/scheduler/ExecutorFactory.java (98%) rename zeppelin-interpreter/src/test/java/{com/nflabs => org/apache}/zeppelin/display/AngularObjectRegistryTest.java (98%) rename zeppelin-interpreter/src/test/java/{com/nflabs => org/apache}/zeppelin/display/AngularObjectTest.java (98%) rename zeppelin-interpreter/src/test/java/{com/nflabs => org/apache}/zeppelin/interpreter/remote/RemoteAngularObjectTest.java (89%) rename zeppelin-interpreter/src/test/java/{com/nflabs => org/apache}/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java (85%) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index f7b891dd9a8..29e51f4c200 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -465,8 +465,8 @@ public void open() { // add some implicit conversion intp.interpret("implicit def watcherConversion(watcher: (Object, Object) => Unit):" - + "com.nflabs.zeppelin.display.AngularObjectWatcher = {" - + " new com.nflabs.zeppelin.display.AngularObjectWatcher() {" + + "org.apache.zeppelin.display.AngularObjectWatcher = {" + + " new org.apache.zeppelin.display.AngularObjectWatcher() {" + " def watch(before:Object, after:Object) = {" + " watcher(before, after)" + " }" diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index ebafd6e3a9a..e227116e646 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -29,9 +29,9 @@ import org.apache.spark.SparkContext; import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.hive.HiveContext; -import com.nflabs.zeppelin.display.AngularObject; -import com.nflabs.zeppelin.display.AngularObjectRegistry; -import com.nflabs.zeppelin.display.AngularObjectWatcher; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectWatcher; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.display.Input.ParamOption; import org.apache.zeppelin.interpreter.InterpreterContext; diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java similarity index 97% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java index 84d1459d3ac..b8fa9a185e9 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObject.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package com.nflabs.zeppelin.display; +package org.apache.zeppelin.display; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; -import com.nflabs.zeppelin.scheduler.ExecutorFactory; +import org.apache.zeppelin.scheduler.ExecutorFactory; /** * diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectListener.java similarity index 96% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectListener.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectListener.java index 84066003b9d..880e48732c8 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectListener.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package com.nflabs.zeppelin.display; +package org.apache.zeppelin.display; /** * diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java similarity index 98% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java index df6c4d28b6e..56eca22d895 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package com.nflabs.zeppelin.display; +package org.apache.zeppelin.display; import java.util.HashMap; import java.util.LinkedList; diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistryListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java similarity index 96% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistryListener.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java index 153d5821d7e..3f08efae4a2 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectRegistryListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package com.nflabs.zeppelin.display; +package org.apache.zeppelin.display; /** * diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectWatcher.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectWatcher.java similarity index 96% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectWatcher.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectWatcher.java index 35a7808dad3..a41fa33119d 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/display/AngularObjectWatcher.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectWatcher.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package com.nflabs.zeppelin.display; +package org.apache.zeppelin.display; /** * diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java index 888f94e450e..9baaef339d0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java @@ -21,7 +21,7 @@ import java.util.Properties; import java.util.Random; -import com.nflabs.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistry; /** * InterpreterGroup is list of interpreters in the same group. diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java similarity index 90% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java index 7299e5423af..3abd7644ef6 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObject.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package com.nflabs.zeppelin.interpreter.remote; +package org.apache.zeppelin.interpreter.remote; -import com.nflabs.zeppelin.display.AngularObject; -import com.nflabs.zeppelin.display.AngularObjectListener; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectListener; /** * diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java similarity index 84% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java index 10405a41d54..c711f69009f 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package com.nflabs.zeppelin.interpreter.remote; +package org.apache.zeppelin.interpreter.remote; -import com.nflabs.zeppelin.display.AngularObject; -import com.nflabs.zeppelin.display.AngularObjectRegistry; -import com.nflabs.zeppelin.display.AngularObjectRegistryListener; -import com.nflabs.zeppelin.interpreter.Interpreter; -import com.nflabs.zeppelin.interpreter.InterpreterGroup; -import com.nflabs.zeppelin.interpreter.WrappedInterpreter; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.WrappedInterpreter; /** * diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java similarity index 90% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 46899bf5df5..95ef2ca6576 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -15,19 +15,19 @@ * limitations under the License. */ -package com.nflabs.zeppelin.interpreter.remote; +package org.apache.zeppelin.interpreter.remote; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.Gson; -import com.nflabs.zeppelin.display.AngularObject; -import com.nflabs.zeppelin.display.AngularObjectRegistry; -import com.nflabs.zeppelin.interpreter.InterpreterGroup; -import com.nflabs.zeppelin.interpreter.thrift.RemoteInterpreterEvent; -import com.nflabs.zeppelin.interpreter.thrift.RemoteInterpreterEventType; -import com.nflabs.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; /** * diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java similarity index 99% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java index f9b55e0c2d0..b44ea963166 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java @@ -4,7 +4,7 @@ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated */ -package com.nflabs.zeppelin.interpreter.thrift; +package org.apache.zeppelin.interpreter.thrift; import org.apache.thrift.scheme.IScheme; import org.apache.thrift.scheme.SchemeFactory; diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java similarity index 95% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java index d8342f3029a..f84c44c8957 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java @@ -4,7 +4,7 @@ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated */ -package com.nflabs.zeppelin.interpreter.thrift; +package org.apache.zeppelin.interpreter.thrift; import java.util.Map; diff --git a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/scheduler/ExecutorFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ExecutorFactory.java similarity index 98% rename from zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/scheduler/ExecutorFactory.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ExecutorFactory.java index f8f90c73f59..31b534e11f7 100644 --- a/zeppelin-interpreter/src/main/java/com/nflabs/zeppelin/scheduler/ExecutorFactory.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ExecutorFactory.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.nflabs.zeppelin.scheduler; +package org.apache.zeppelin.scheduler; import java.util.HashMap; import java.util.Map; diff --git a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectRegistryTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java similarity index 98% rename from zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectRegistryTest.java rename to zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java index 85f951ce60b..b0ed45f3a94 100644 --- a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectRegistryTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package com.nflabs.zeppelin.display; +package org.apache.zeppelin.display; import static org.junit.Assert.assertEquals; diff --git a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java similarity index 98% rename from zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java rename to zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java index 31de8bf570c..7d6b000bece 100644 --- a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/display/AngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package com.nflabs.zeppelin.display; +package org.apache.zeppelin.display; import static org.junit.Assert.assertEquals; diff --git a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java similarity index 89% rename from zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectTest.java rename to zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index f98a1c9f41d..3af00fba635 100644 --- a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package com.nflabs.zeppelin.interpreter.remote; +package org.apache.zeppelin.interpreter.remote; import static org.junit.Assert.assertEquals; @@ -28,14 +28,14 @@ import org.junit.Before; import org.junit.Test; -import com.nflabs.zeppelin.display.AngularObject; -import com.nflabs.zeppelin.display.AngularObjectRegistry; -import com.nflabs.zeppelin.display.AngularObjectRegistryListener; -import com.nflabs.zeppelin.display.GUI; -import com.nflabs.zeppelin.interpreter.InterpreterContext; -import com.nflabs.zeppelin.interpreter.InterpreterGroup; -import com.nflabs.zeppelin.interpreter.InterpreterResult; -import com.nflabs.zeppelin.interpreter.remote.mock.MockInterpreterAngular; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterAngular; public class RemoteAngularObjectTest implements AngularObjectRegistryListener { private InterpreterGroup intpGroup; diff --git a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java similarity index 85% rename from zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java rename to zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java index ea5edf0ee56..c2468112e06 100644 --- a/zeppelin-interpreter/src/test/java/com/nflabs/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java @@ -15,19 +15,19 @@ * limitations under the License. */ -package com.nflabs.zeppelin.interpreter.remote.mock; +package org.apache.zeppelin.interpreter.remote.mock; import java.util.List; import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; -import com.nflabs.zeppelin.display.AngularObjectRegistry; -import com.nflabs.zeppelin.display.AngularObjectWatcher; -import com.nflabs.zeppelin.interpreter.Interpreter; -import com.nflabs.zeppelin.interpreter.InterpreterContext; -import com.nflabs.zeppelin.interpreter.InterpreterPropertyBuilder; -import com.nflabs.zeppelin.interpreter.InterpreterResult; -import com.nflabs.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectWatcher; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; public class MockInterpreterAngular extends Interpreter implements AngularObjectWatcher { static { @@ -112,4 +112,4 @@ public void watch(Object oldObject, Object newObject) { numWatch.incrementAndGet(); } -} \ No newline at end of file +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 45371afbea1..ae56d97fe55 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -455,7 +455,7 @@ private void angularObjectUpdated(WebSocket conn, Notebook notebook, AngularObjectRegistry angularObjectRegistry = setting .getInterpreterGroup().getAngularObjectRegistry(); AngularObject ao = angularObjectRegistry.get(varName); - this.broadcast(n.id(), new Message(OP.ANGULAR_OBJECT_UPDATE) + this.broadcast(n.id(), new Message(OP.ANGULAR_OBJECT_UPDATE) .put("angularObject", ao) .put("interpreterGroupId", interpreterGroupId) .put("noteId", n.id())); From 0033d326ea67ee9e5d6007c4e05d8f50a26feb3d Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 6 Apr 2015 12:33:33 +0900 Subject: [PATCH 12/21] Add angular interpreter --- angular/pom.xml | 127 ++++++++++++++++++ .../zeppelin/angular/AngularInterpreter.java | 81 +++++++++++ conf/zeppelin-site.xml.template | 2 +- pom.xml | 1 + .../zeppelin/conf/ZeppelinConfiguration.java | 1 + 5 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 angular/pom.xml create mode 100644 angular/src/main/java/org/apache/zeppelin/angular/AngularInterpreter.java diff --git a/angular/pom.xml b/angular/pom.xml new file mode 100644 index 00000000000..580b848e01f --- /dev/null +++ b/angular/pom.xml @@ -0,0 +1,127 @@ + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.5.0-SNAPSHOT + + + org.apache.zeppelin + zeppelin-angular + jar + 0.5.0-SNAPSHOT + Zeppelin: Angular interpreter + http://zeppelin.incubator.apache.org + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + provided + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + + + + maven-dependency-plugin + 2.8 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/../../interpreter/angular + false + false + true + runtime + + + + copy-artifact + package + + copy + + + ${project.build.directory}/../../interpreter/angular + false + false + true + runtime + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + + diff --git a/angular/src/main/java/org/apache/zeppelin/angular/AngularInterpreter.java b/angular/src/main/java/org/apache/zeppelin/angular/AngularInterpreter.java new file mode 100644 index 00000000000..c7a406d18d7 --- /dev/null +++ b/angular/src/main/java/org/apache/zeppelin/angular/AngularInterpreter.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.angular; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterResult.Type; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; + +/** + * + */ +public class AngularInterpreter extends Interpreter { + static { + Interpreter.register("angular", AngularInterpreter.class.getName()); + } + + public AngularInterpreter(Properties property) { + super(property); + } + + @Override + public void open() { + } + + @Override + public void close() { + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + return new InterpreterResult(Code.SUCCESS, Type.ANGULAR, st); + } + + @Override + public void cancel(InterpreterContext context) { + } + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public List completion(String buf, int cursor) { + return new LinkedList(); + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton().createOrGetFIFOScheduler( + AngularInterpreter.class.getName() + this.hashCode()); + } +} diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index c203179c55a..2e2cad80576 100644 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -48,7 +48,7 @@ zeppelin.interpreters - org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter + org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter Comma separated interpreter configurations. First interpreter become a default diff --git a/pom.xml b/pom.xml index 20a073e70bd..d2fc77c01b9 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,7 @@ zeppelin-zengine spark markdown + angular shell hive zeppelin-web diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 8495bb5341b..0f35f014d69 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -375,6 +375,7 @@ public static enum ConfVars { + "org.apache.zeppelin.spark.SparkSqlInterpreter," + "org.apache.zeppelin.spark.DepInterpreter," + "org.apache.zeppelin.markdown.Markdown," + + "org.apache.zeppelin.angular.AngularInterpreter," + "org.apache.zeppelin.shell.ShellInterpreter"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"), From 08990115dc5c75c31e6798550907fd527bf379aa Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 6 Apr 2015 20:05:18 +0900 Subject: [PATCH 13/21] Fix test --- .../test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 3e635031ead..a3bf2890e35 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -79,7 +79,7 @@ public void getAvailableInterpreters() throws IOException { assertThat(get, isAllowed()); Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>(){}.getType()); Map body = (Map) resp.get("body"); - assertEquals(6, body.size()); + assertEquals(7, body.size()); get.releaseConnection(); } From dac416d26c897a1c12faca6f52a787ed174bd2ee Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 7 Apr 2015 20:13:11 +0900 Subject: [PATCH 14/21] Implement z.run() --- .../zeppelin/spark/ZeppelinContext.java | 109 ++++++++++++++---- .../zeppelin/spark/DepInterpreterTest.java | 7 +- .../zeppelin/spark/SparkInterpreterTest.java | 6 +- .../spark/SparkSqlInterpreterTest.java | 7 +- .../interpreter/InterpreterContext.java | 11 +- .../interpreter/InterpreterContextRunner.java | 58 ++++++++++ .../remote/InterpreterContextRunnerPool.java | 88 ++++++++++++++ .../interpreter/remote/RemoteInterpreter.java | 18 ++- .../RemoteInterpreterContextRunner.java | 38 ++++++ .../remote/RemoteInterpreterEventPoller.java | 15 ++- .../remote/RemoteInterpreterProcess.java | 13 ++- .../remote/RemoteInterpreterServer.java | 36 +++++- .../thrift/RemoteInterpreterContext.java | 108 ++++++++++++++++- .../thrift/RemoteInterpreterEventType.java | 5 +- .../thrift/RemoteInterpreterService.thrift | 6 +- .../remote/RemoteAngularObjectTest.java | 12 +- .../remote/RemoteInterpreterProcessTest.java | 7 +- .../remote/RemoteInterpreterTest.java | 24 ++-- .../scheduler/RemoteSchedulerTest.java | 8 +- .../org/apache/zeppelin/notebook/Note.java | 10 +- .../apache/zeppelin/notebook/Paragraph.java | 39 ++++++- .../interpreter/InterpreterFactoryTest.java | 2 +- 22 files changed, 544 insertions(+), 83 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContextRunner.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index e227116e646..bbccf0ccce0 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -25,6 +25,8 @@ import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import org.apache.spark.SparkContext; import org.apache.spark.sql.SQLContext; @@ -35,6 +37,8 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.display.Input.ParamOption; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.spark.dep.DependencyResolver; import scala.Tuple2; @@ -66,12 +70,6 @@ public ZeppelinContext(SparkContext sc, SQLContext sql, public HiveContext hiveContext; private GUI gui; - /* spark-1.3 - public SchemaRDD sql(String sql) { - return sqlContext.sql(sql); - } - */ - /** * Load dependency for interpreter and runtime (driver). * And distribute them to spark cluster (sc.add()) @@ -224,25 +222,6 @@ public void setGui(GUI o) { this.gui = o; } - public void run(String lines) { - /* - String intpName = Paragraph.getRequiredReplName(lines); - String scriptBody = Paragraph.getScriptBody(lines); - Interpreter intp = interpreterContext.getParagraph().getRepl(intpName); - InterpreterResult ret = intp.interpret(scriptBody, interpreterContext); - if (ret.code() == InterpreterResult.Code.SUCCESS) { - out.println("%" + ret.type().toString().toLowerCase() + " " + ret.message()); - } else if (ret.code() == InterpreterResult.Code.ERROR) { - out.println("Error: " + ret.message()); - } else if (ret.code() == InterpreterResult.Code.INCOMPLETE) { - out.println("Incomplete"); - } else { - out.println("Unknown error"); - } - */ - throw new RuntimeException("Missing implementation"); - } - private void restartInterpreter() { } @@ -254,6 +233,86 @@ public void setInterpreterContext(InterpreterContext interpreterContext) { this.interpreterContext = interpreterContext; } + /** + * Run paragraph by id + * @param id + */ + public void run(String id) { + if (id.equals(interpreterContext.getParagraphId())) { + throw new InterpreterException("Can not run current Paragraph"); + } + + for (InterpreterContextRunner r : interpreterContext.getRunners()) { + if (id.equals(r.getParagraphId())) { + r.run(); + return; + } + } + + throw new InterpreterException("Paragraph " + id + " not found"); + } + + /** + * Run paragraph at index + * @param idx index starting from 0 + */ + public void run(int idx) { + if (idx >= interpreterContext.getRunners().size()) { + throw new InterpreterException("Index out of bound"); + } + + InterpreterContextRunner runner = interpreterContext.getRunners().get(idx); + if (runner.getParagraphId().equals(interpreterContext.getParagraphId())) { + throw new InterpreterException("Can not run current Paragraph"); + } + + runner.run(); + } + + /** + * Run paragraphs + * @param paragraphIdOrIdxs list of paragraph id or idx + */ + public void run(List paragraphIdOrIdx) { + for (Object idOrIdx : paragraphIdOrIdx) { + if (idOrIdx instanceof String) { + String id = (String) idOrIdx; + run(id); + } else if (idOrIdx instanceof Integer) { + Integer idx = (Integer) idOrIdx; + run(idx); + } else { + throw new InterpreterException("Paragraph " + idOrIdx + " not found"); + } + } + } + + + /** + * Run all paragraphs. except this. + */ + public void runAll() { + for (InterpreterContextRunner r : interpreterContext.getRunners()) { + if (r.getParagraphId().equals(interpreterContext.getParagraphId())) { + // skip itself + continue; + } + r.run(); + } + } + + public List listParagraphs() { + List paragraphs = new LinkedList(); + + for (InterpreterContextRunner r : interpreterContext.getRunners()) { + paragraphs.add(r.getParagraphId()); + } + + return paragraphs; + } + + + public Object angular(String name) { AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry(); AngularObject ao = registry.get(name); diff --git a/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java index bdb7b2ca295..2f8254d198b 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java @@ -21,16 +21,16 @@ import java.io.File; import java.util.HashMap; +import java.util.LinkedList; import java.util.Properties; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.spark.DepInterpreter; -import org.apache.zeppelin.spark.SparkInterpreter; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -59,7 +59,8 @@ public void setUp() throws Exception { dep.setInterpreterGroup(intpGroup); context = new InterpreterContext("id", "title", "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null)); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList()); } @After diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index 473793c204a..a5e0fe22760 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -22,15 +22,16 @@ import java.io.File; import java.util.HashMap; +import java.util.LinkedList; import java.util.Properties; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.spark.SparkInterpreter; import org.junit.After; import org.junit.Before; import org.junit.FixMethodOrder; @@ -59,7 +60,8 @@ public void setUp() throws Exception { InterpreterGroup intpGroup = new InterpreterGroup(); context = new InterpreterContext("id", "title", "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null)); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList()); } @After diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java index c85e2c72a95..19a760c16aa 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java @@ -20,16 +20,16 @@ import static org.junit.Assert.assertEquals; import java.util.HashMap; +import java.util.LinkedList; import java.util.Properties; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.apache.zeppelin.spark.SparkInterpreter; -import org.apache.zeppelin.spark.SparkSqlInterpreter; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -64,7 +64,8 @@ public void setUp() throws Exception { sql.open(); } context = new InterpreterContext("id", "title", "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null)); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList()); } @After diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java index f09980d7a91..2e4564e09ef 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.interpreter; +import java.util.List; import java.util.Map; import org.apache.zeppelin.display.AngularObjectRegistry; @@ -32,14 +33,15 @@ public class InterpreterContext { private final Map config; private GUI gui; private AngularObjectRegistry angularObjectRegistry; - + private List runners; public InterpreterContext(String paragraphId, String paragraphTitle, String paragraphText, Map config, GUI gui, - AngularObjectRegistry angularObjectRegistry + AngularObjectRegistry angularObjectRegistry, + List runners ) { this.paragraphId = paragraphId; this.paragraphTitle = paragraphTitle; @@ -47,6 +49,7 @@ public InterpreterContext(String paragraphId, this.config = config; this.gui = gui; this.angularObjectRegistry = angularObjectRegistry; + this.runners = runners; } public String getParagraphId() { @@ -73,4 +76,8 @@ public AngularObjectRegistry getAngularObjectRegistry() { return angularObjectRegistry; } + public List getRunners() { + return runners; + } + } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContextRunner.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContextRunner.java new file mode 100644 index 00000000000..7a2df100101 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContextRunner.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter; + +/** + */ +public abstract class InterpreterContextRunner implements Runnable { + String noteId; + private String paragraphId; + + public InterpreterContextRunner(String noteId, String paragraphId) { + this.noteId = noteId; + this.paragraphId = paragraphId; + } + + @Override + public boolean equals(Object o) { + if (o instanceof InterpreterContextRunner) { + InterpreterContextRunner io = ((InterpreterContextRunner) o); + if (io.getParagraphId().equals(paragraphId) && + io.getNoteId().equals(noteId)) { + return true; + } else { + return false; + } + + } else { + return false; + } + } + + @Override + public abstract void run(); + + public String getNoteId() { + return noteId; + } + + public String getParagraphId() { + return paragraphId; + } + +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java new file mode 100644 index 00000000000..ca2df124f6e --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.remote; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class InterpreterContextRunnerPool { + Logger logger = LoggerFactory.getLogger(InterpreterContextRunnerPool.class); + private Map> interpreterContextRunners; + + public InterpreterContextRunnerPool() { + interpreterContextRunners = new HashMap>(); + + } + + // add runner + public void add(String noteId, InterpreterContextRunner runner) { + synchronized (interpreterContextRunners) { + if (!interpreterContextRunners.containsKey(noteId)) { + interpreterContextRunners.put(noteId, new LinkedList()); + } + + interpreterContextRunners.get(noteId).add(runner); + } + } + + // replace all runners to noteId + public void addAll(String noteId, List runners) { + synchronized (interpreterContextRunners) { + if (!interpreterContextRunners.containsKey(noteId)) { + interpreterContextRunners.put(noteId, new LinkedList()); + } + + interpreterContextRunners.get(noteId).addAll(runners); + } + } + + public void clear(String noteId) { + synchronized (interpreterContextRunners) { + interpreterContextRunners.remove(noteId); + } + } + + + public void run(String noteId, String paragraphId) { + synchronized (interpreterContextRunners) { + List list = interpreterContextRunners.get(noteId); + if (list != null) { + for (InterpreterContextRunner r : list) { + if (noteId.equals(r.getNoteId()) && paragraphId.equals(r.getParagraphId())) { + logger.info("run paragraph {} on note {} from InterpreterContext", + r.getParagraphId(), r.getNoteId()); + r.run(); + return; + } + } + } + + throw new InterpreterException("Can not run paragraph " + paragraphId + " on " + noteId); + } + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index e32344874d8..3e6128f7775 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -26,6 +26,7 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; @@ -56,6 +57,8 @@ public class RemoteInterpreter extends Interpreter { static Map interpreterGroupReference = new HashMap(); + private InterpreterContextRunnerPool interpreterContextRunnerPool; + public RemoteInterpreter(Properties property, String className, String interpreterRunner, @@ -67,6 +70,7 @@ public RemoteInterpreter(Properties property, this.interpreterRunner = interpreterRunner; this.interpreterPath = interpreterPath; env = new HashMap(); + interpreterContextRunnerPool = new InterpreterContextRunnerPool(); } public RemoteInterpreter(Properties property, @@ -187,6 +191,15 @@ public InterpreterResult interpret(String st, InterpreterContext context) { throw new InterpreterException(e1); } + List runners = context.getRunners(); + if (runners != null && runners.size() != 0) { + // assume all runners in this InterpreterContext have the same note id + String noteId = runners.get(0).getNoteId(); + + interpreterContextRunnerPool.clear(noteId); + interpreterContextRunnerPool.addAll(noteId, runners); + } + try { GUI settings = context.getGui(); RemoteInterpreterResult remoteResult = client.interpret(className, st, convert(context)); @@ -316,7 +329,7 @@ public void setInterpreterGroup(InterpreterGroup interpreterGroup) { .containsKey(getInterpreterGroupKey(interpreterGroup))) { interpreterGroupReference.put(getInterpreterGroupKey(interpreterGroup), new RemoteInterpreterProcess(interpreterRunner, - interpreterPath, env)); + interpreterPath, env, interpreterContextRunnerPool)); logger.info("setInterpreterGroup = " + getInterpreterGroupKey(interpreterGroup) + " class=" + className @@ -335,7 +348,8 @@ private RemoteInterpreterContext convert(InterpreterContext ic) { ic.getParagraphTitle(), ic.getParagraphText(), gson.toJson(ic.getConfig()), - gson.toJson(ic.getGui())); + gson.toJson(ic.getGui()), + gson.toJson(ic.getRunners())); } private InterpreterResult convert(RemoteInterpreterResult result) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java new file mode 100644 index 00000000000..8d16ec52b14 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.remote; + +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; + +/** + * + */ +public class RemoteInterpreterContextRunner extends InterpreterContextRunner { + + public RemoteInterpreterContextRunner(String noteId, String paragraphId) { + super(noteId, paragraphId); + } + + @Override + public void run() { + // this class should be used only for gson deserialize abstract class + // code should not reach here + throw new InterpreterException("Assert"); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 95ef2ca6576..dc9ef0b7bb3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -18,16 +18,17 @@ package org.apache.zeppelin.interpreter.remote; import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.gson.Gson; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; /** * @@ -103,6 +104,12 @@ public void run() { } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) { AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class); angularObjectRegistry.remove(angularObject.getName()); + } else if (event.getType() == RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER) { + InterpreterContextRunner runnerFromRemote = gson.fromJson( + event.getData(), RemoteInterpreterContextRunner.class); + + interpreterProcess.getInterpreterContextRunnerPool().run( + runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId()); } } catch (Exception e) { logger.error("Can't handle event " + event, e); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 47b58f50694..dbfaa35f54a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -28,10 +28,10 @@ import org.apache.commons.exec.ExecuteWatchdog; import org.apache.commons.exec.environment.EnvironmentUtils; import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.thrift.TException; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; -import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,11 +53,16 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler { private GenericObjectPool clientPool; private Map env; private RemoteInterpreterEventPoller remoteInterpreterEventPoller; + private InterpreterContextRunnerPool interpreterContextRunnerPool; - public RemoteInterpreterProcess(String intpRunner, String intpDir, Map env) { + public RemoteInterpreterProcess(String intpRunner, + String intpDir, + Map env, + InterpreterContextRunnerPool interpreterContextRunnerPool) { this.interpreterRunner = intpRunner; this.interpreterDir = intpDir; this.env = env; + this.interpreterContextRunnerPool = interpreterContextRunnerPool; referenceCount = new AtomicInteger(0); } @@ -242,4 +247,8 @@ public void updateRemoteAngularObject(String name, Object o) { releaseClient(client); } } + + public InterpreterContextRunnerPool getInterpreterContextRunnerPool() { + return interpreterContextRunnerPool; + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 17def52ff17..8209696f1a3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -37,22 +37,23 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.ClassloaderInterpreter; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; -import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; import org.apache.zeppelin.scheduler.JobProgressPoller; import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.Job.Status; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,7 +70,6 @@ public class RemoteInterpreterServer InterpreterGroup interpreterGroup; AngularObjectRegistry angularObjectRegistry; - Gson gson = new Gson(); RemoteInterpreterService.Processor processor; @@ -314,6 +314,14 @@ public List completion(String className, String buf, int cursor) throws } private InterpreterContext convert(RemoteInterpreterContext ric) { + List contextRunners = new LinkedList(); + List runners = gson.fromJson(ric.getRunners(), + new TypeToken>(){}.getType()); + + for (InterpreterContextRunner r : runners) { + contextRunners.add(new ParagraphRunner(this, r.getNoteId(), r.getParagraphId())); + } + return new InterpreterContext( ric.getParagraphId(), ric.getParagraphTitle(), @@ -321,7 +329,27 @@ private InterpreterContext convert(RemoteInterpreterContext ric) { (Map) gson.fromJson(ric.getConfig(), new TypeToken>() {}.getType()), gson.fromJson(ric.getGui(), GUI.class), - interpreterGroup.getAngularObjectRegistry()); + interpreterGroup.getAngularObjectRegistry(), + contextRunners); + } + + + static class ParagraphRunner extends InterpreterContextRunner { + + private transient RemoteInterpreterServer server; + + public ParagraphRunner(RemoteInterpreterServer server, String noteId, String paragraphId) { + super(noteId, paragraphId); + this.server = server; + } + + @Override + public void run() { + Gson gson = new Gson(); + server.sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER, + gson.toJson(this))); + } } private RemoteInterpreterResult convert(InterpreterResult result, diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java index 4284cf12a1d..9827a45fdb0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java @@ -38,6 +38,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -50,6 +51,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase byName = new HashMap(); @@ -82,6 +85,8 @@ public static _Fields findByThriftId(int fieldId) { return CONFIG; case 5: // GUI return GUI; + case 6: // RUNNERS + return RUNNERS; default: return null; } @@ -135,6 +140,8 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.GUI, new org.apache.thrift.meta_data.FieldMetaData("gui", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.RUNNERS, new org.apache.thrift.meta_data.FieldMetaData("runners", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RemoteInterpreterContext.class, metaDataMap); } @@ -147,7 +154,8 @@ public RemoteInterpreterContext( String paragraphTitle, String paragraphText, String config, - String gui) + String gui, + String runners) { this(); this.paragraphId = paragraphId; @@ -155,6 +163,7 @@ public RemoteInterpreterContext( this.paragraphText = paragraphText; this.config = config; this.gui = gui; + this.runners = runners; } /** @@ -176,6 +185,9 @@ public RemoteInterpreterContext(RemoteInterpreterContext other) { if (other.isSetGui()) { this.gui = other.gui; } + if (other.isSetRunners()) { + this.runners = other.runners; + } } public RemoteInterpreterContext deepCopy() { @@ -189,6 +201,7 @@ public void clear() { this.paragraphText = null; this.config = null; this.gui = null; + this.runners = null; } public String getParagraphId() { @@ -311,6 +324,30 @@ public void setGuiIsSet(boolean value) { } } + public String getRunners() { + return this.runners; + } + + public RemoteInterpreterContext setRunners(String runners) { + this.runners = runners; + return this; + } + + public void unsetRunners() { + this.runners = null; + } + + /** Returns true if field runners is set (has been assigned a value) and false otherwise */ + public boolean isSetRunners() { + return this.runners != null; + } + + public void setRunnersIsSet(boolean value) { + if (!value) { + this.runners = null; + } + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case PARAGRAPH_ID: @@ -353,6 +390,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case RUNNERS: + if (value == null) { + unsetRunners(); + } else { + setRunners((String)value); + } + break; + } } @@ -373,6 +418,9 @@ public Object getFieldValue(_Fields field) { case GUI: return getGui(); + case RUNNERS: + return getRunners(); + } throw new IllegalStateException(); } @@ -394,6 +442,8 @@ public boolean isSet(_Fields field) { return isSetConfig(); case GUI: return isSetGui(); + case RUNNERS: + return isSetRunners(); } throw new IllegalStateException(); } @@ -456,6 +506,15 @@ public boolean equals(RemoteInterpreterContext that) { return false; } + boolean this_present_runners = true && this.isSetRunners(); + boolean that_present_runners = true && that.isSetRunners(); + if (this_present_runners || that_present_runners) { + if (!(this_present_runners && that_present_runners)) + return false; + if (!this.runners.equals(that.runners)) + return false; + } + return true; } @@ -522,6 +581,16 @@ public int compareTo(RemoteInterpreterContext other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetRunners()).compareTo(typedOther.isSetRunners()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRunners()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.runners, typedOther.runners); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -581,6 +650,14 @@ public String toString() { sb.append(this.gui); } first = false; + if (!first) sb.append(", "); + sb.append("runners:"); + if (this.runners == null) { + sb.append("null"); + } else { + sb.append(this.runners); + } + first = false; sb.append(")"); return sb.toString(); } @@ -664,6 +741,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, RemoteInterpreterCo org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 6: // RUNNERS + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.runners = iprot.readString(); + struct.setRunnersIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -704,6 +789,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, RemoteInterpreterC oprot.writeString(struct.gui); oprot.writeFieldEnd(); } + if (struct.runners != null) { + oprot.writeFieldBegin(RUNNERS_FIELD_DESC); + oprot.writeString(struct.runners); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -737,7 +827,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, RemoteInterpreterCo if (struct.isSetGui()) { optionals.set(4); } - oprot.writeBitSet(optionals, 5); + if (struct.isSetRunners()) { + optionals.set(5); + } + oprot.writeBitSet(optionals, 6); if (struct.isSetParagraphId()) { oprot.writeString(struct.paragraphId); } @@ -753,12 +846,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, RemoteInterpreterCo if (struct.isSetGui()) { oprot.writeString(struct.gui); } + if (struct.isSetRunners()) { + oprot.writeString(struct.runners); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, RemoteInterpreterContext struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(5); + BitSet incoming = iprot.readBitSet(6); if (incoming.get(0)) { struct.paragraphId = iprot.readString(); struct.setParagraphIdIsSet(true); @@ -779,6 +875,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, RemoteInterpreterCon struct.gui = iprot.readString(); struct.setGuiIsSet(true); } + if (incoming.get(5)) { + struct.runners = iprot.readString(); + struct.setRunnersIsSet(true); + } } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java index f84c44c8957..abbc68f364a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java @@ -15,7 +15,8 @@ public enum RemoteInterpreterEventType implements org.apache.thrift.TEnum { NO_OP(1), ANGULAR_OBJECT_ADD(2), ANGULAR_OBJECT_UPDATE(3), - ANGULAR_OBJECT_REMOVE(4); + ANGULAR_OBJECT_REMOVE(4), + RUN_INTERPRETER_CONTEXT_RUNNER(5); private final int value; @@ -44,6 +45,8 @@ public static RemoteInterpreterEventType findByValue(int value) { return ANGULAR_OBJECT_UPDATE; case 4: return ANGULAR_OBJECT_REMOVE; + case 5: + return RUN_INTERPRETER_CONTEXT_RUNNER; default: return null; } diff --git a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift index fcd0832ed77..f9cd181e17e 100644 --- a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift +++ b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift @@ -24,7 +24,8 @@ struct RemoteInterpreterContext { 2: string paragraphTitle, 3: string paragraphText, 4: string config, // json serialized config - 5: string gui // json serialized gui + 5: string gui, // json serialized gui + 6: string runners // json serialized runner } struct RemoteInterpreterResult { @@ -39,7 +40,8 @@ enum RemoteInterpreterEventType { NO_OP = 1, ANGULAR_OBJECT_ADD = 2, ANGULAR_OBJECT_UPDATE = 3, - ANGULAR_OBJECT_REMOVE = 4 + ANGULAR_OBJECT_REMOVE = 4, + RUN_INTERPRETER_CONTEXT_RUNNER = 5 } struct RemoteInterpreterEvent { diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index 3af00fba635..d4909e346ec 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -21,21 +21,22 @@ import java.io.File; import java.util.HashMap; +import java.util.LinkedList; import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterAngular; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; public class RemoteAngularObjectTest implements AngularObjectRegistryListener { private InterpreterGroup intpGroup; @@ -79,7 +80,8 @@ public void setUp() throws Exception { "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null)); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList()); intp.open(); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index b9deef11dd3..fcd684718cf 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -23,7 +23,6 @@ import java.util.HashMap; import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.junit.Test; @@ -32,7 +31,8 @@ public class RemoteInterpreterProcessTest { @Test public void testStartStop() { InterpreterGroup intpGroup = new InterpreterGroup(); - RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap()); + RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap(), + new InterpreterContextRunnerPool()); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); assertEquals(1, rip.reference(intpGroup)); @@ -47,7 +47,8 @@ public void testStartStop() { @Test public void testClientFactory() throws Exception { InterpreterGroup intpGroup = new InterpreterGroup(); - RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap()); + RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap(), + new InterpreterContextRunnerPool()); rip.reference(intpGroup); assertEquals(0, rip.getNumActiveClient()); assertEquals(0, rip.getNumIdleClient()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index 952b377569f..e743eabb96c 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -33,10 +33,9 @@ import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA; import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterB; import org.apache.zeppelin.scheduler.Job; @@ -111,7 +110,8 @@ public void testRemoteInterperterCall() throws TTransportException, IOException "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null))); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList())); intpB.open(); assertEquals(2, process.referenceCount()); @@ -162,7 +162,8 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null))); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList())); assertEquals("500", ret.message()); ret = intpB.interpret("500", @@ -172,7 +173,8 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null))); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList())); assertEquals("1000", ret.message()); long end = System.currentTimeMillis(); assertTrue(end - start >= 1000); @@ -236,7 +238,8 @@ protected Object jobRun() throws Throwable { "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null))); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList())); } @Override @@ -268,7 +271,8 @@ protected Object jobRun() throws Throwable { "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null))); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList())); } @Override @@ -340,7 +344,8 @@ protected Object jobRun() throws Throwable { "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null))); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList())); synchronized (results) { results.add(ret.message()); @@ -421,7 +426,8 @@ protected Object jobRun() throws Throwable { "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null))); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList())); synchronized (results) { results.add(ret.message()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index 8832c881df5..bb0fb8074f8 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -21,18 +21,17 @@ import java.io.File; import java.util.HashMap; +import java.util.LinkedList; import java.util.Map; import java.util.Properties; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA; -import org.apache.zeppelin.scheduler.Job; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -95,7 +94,8 @@ protected Object jobRun() throws Throwable { "text", new HashMap(), new GUI(), - new AngularObjectRegistry(intpGroup.getId(), null))); + new AngularObjectRegistry(intpGroup.getId(), null), + new LinkedList())); return "1000"; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index fefd4cf00e2..b5e68a42bb3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.Random; -import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; @@ -36,13 +35,14 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.utility.IdHashes; import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.Job.Status; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,7 +126,7 @@ public Map> getAngularObjects() { * @param p */ public Paragraph addParagraph() { - Paragraph p = new Paragraph(this, replLoader); + Paragraph p = new Paragraph(this, this, replLoader); synchronized (paragraphs) { paragraphs.add(p); } @@ -140,7 +140,7 @@ public Paragraph addParagraph() { * @param p */ public Paragraph insertParagraph(int index) { - Paragraph p = new Paragraph(this, replLoader); + Paragraph p = new Paragraph(this, this, replLoader); synchronized (paragraphs) { paragraphs.add(index, p); } @@ -342,6 +342,8 @@ public static Note load(String id, ZeppelinConfiguration conf, NoteInterpreterLo note.setReplLoader(replLoader); note.jobListenerFactory = jobListenerFactory; for (Paragraph p : note.paragraphs) { + p.setNote(note); + if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { p.setStatus(Status.ABORT); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index e5599e00eb1..79dfc3dc079 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -19,6 +19,7 @@ import java.io.Serializable; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; @@ -27,10 +28,11 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterSetting; -import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.JobListener; import org.slf4j.Logger; @@ -44,14 +46,16 @@ public class Paragraph extends Job implements Serializable { private static final transient long serialVersionUID = -6328572073497992016L; private transient NoteInterpreterLoader replLoader; + private transient Note note; String title; String text; private Map config; // paragraph configs like isOpen, colWidth, etc public final GUI settings; // form and parameter settings - public Paragraph(JobListener listener, NoteInterpreterLoader replLoader) { + public Paragraph(Note note, JobListener listener, NoteInterpreterLoader replLoader) { super(generateId(), listener); + this.note = note; this.replLoader = replLoader; title = null; text = null; @@ -81,6 +85,14 @@ public void setTitle(String title) { this.title = title; } + public void setNote(Note note) { + this.note = note; + } + + public Note getNote() { + return note; + } + public String getRequiredReplName() { return getRequiredReplName(text); } @@ -216,15 +228,36 @@ private InterpreterContext getInterpreterContext() { registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry(); } + List runners = new LinkedList(); + for (Paragraph p : note.getParagraphs()) { + runners.add(new ParagraphRunner(note, note.id(), p.getId())); + } + InterpreterContext interpreterContext = new InterpreterContext(getId(), this.getTitle(), this.getText(), this.getConfig(), this.settings, - registry); + registry, + runners); return interpreterContext; } + static class ParagraphRunner extends InterpreterContextRunner { + private Note note; + + public ParagraphRunner(Note note, String noteId, String paragraphId) { + super(noteId, paragraphId); + this.note = note; + } + + @Override + public void run() { + note.run(getParagraphId()); + } + } + + private Logger logger() { Logger logger = LoggerFactory.getLogger(Paragraph.class); return logger; diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index dc3ac3b62fd..63aef0df979 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -55,7 +55,7 @@ public void setUp() throws Exception { System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); conf = new ZeppelinConfiguration(); factory = new InterpreterFactory(conf, new InterpreterOption(false), null); - context = new InterpreterContext("id", "title", "text", null, null, null); + context = new InterpreterContext("id", "title", "text", null, null, null, null); } From ee29866881bd5eb3bbdea9ce7b2866e4b906f623 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 7 Apr 2015 21:09:17 +0900 Subject: [PATCH 15/21] ZEPPELIN-32 implement z.show() --- .../zeppelin/spark/SparkInterpreter.java | 6 +- .../zeppelin/spark/SparkSqlInterpreter.java | 84 +---------- .../zeppelin/spark/ZeppelinContext.java | 130 +++++++++++++++++- 3 files changed, 138 insertions(+), 82 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 29e51f4c200..df037432c1c 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -107,6 +107,7 @@ public class SparkInterpreter extends Interpreter { + "we should set this value") .add("zeppelin.spark.useHiveContext", "true", "Use HiveContext instead of SQLContext if it is true.") + .add("zeppelin.spark.maxResult", "1000", "Max number of SparkSQL result to display.") .add("args", "", "spark commandline args").build()); } @@ -400,7 +401,8 @@ public void open() { dep = getDependencyResolver(); - z = new ZeppelinContext(sc, sqlc, null, dep, printStream); + z = new ZeppelinContext(sc, sqlc, null, dep, printStream, + Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); try { if (sc.version().startsWith("1.1") || sc.version().startsWith("1.2")) { @@ -522,7 +524,7 @@ public Object getValue(String name) { } String getJobGroup(InterpreterContext context){ - return "zeppelin-" + this.hashCode() + "-" + context.getParagraphId(); + return "zeppelin-" + context.getParagraphId(); } /** diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index 2555988288f..e4eece81ba4 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -29,18 +29,15 @@ import org.apache.spark.scheduler.DAGScheduler; import org.apache.spark.scheduler.Stage; import org.apache.spark.sql.SQLContext; -import org.apache.spark.sql.SQLContext.QueryExecution; -import org.apache.spark.sql.catalyst.expressions.Attribute; import org.apache.spark.ui.jobs.JobProgressListener; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterUtils; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.WrappedInterpreter; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; @@ -76,7 +73,7 @@ public class SparkSqlInterpreter extends Interpreter { } private String getJobGroup(InterpreterContext context){ - return "zeppelin-" + this.hashCode() + "-" + context.getParagraphId(); + return "zeppelin-" + context.getParagraphId(); } private int maxResult; @@ -126,81 +123,10 @@ public InterpreterResult interpret(String st, InterpreterContext context) { sc.setLocalProperty("spark.scheduler.pool", null); } - sc.setJobGroup(getJobGroup(context), "Zeppelin", false); - - // SchemaRDD - spark 1.1, 1.2, DataFrame - spark 1.3 - Object rdd; - Object[] rows = null; - try { - rdd = sqlc.sql(st); - - Method take = rdd.getClass().getMethod("take", int.class); - rows = (Object[]) take.invoke(rdd, maxResult + 1); - } catch (Exception e) { - logger.error("Error", e); - sc.clearJobGroup(); - return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e)); - } + Object rdd = sqlc.sql(st); + String msg = ZeppelinContext.showRDD(sc, context, rdd, maxResult); - String msg = null; - - // get field names - Method queryExecution; - QueryExecution qe; - try { - queryExecution = rdd.getClass().getMethod("queryExecution"); - qe = (QueryExecution) queryExecution.invoke(rdd); - } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - throw new InterpreterException(e); - } - - List columns = - scala.collection.JavaConverters.asJavaListConverter( - qe.analyzed().output()).asJava(); - - for (Attribute col : columns) { - if (msg == null) { - msg = col.name(); - } else { - msg += "\t" + col.name(); - } - } - - msg += "\n"; - - // ArrayType, BinaryType, BooleanType, ByteType, DecimalType, DoubleType, DynamicType, - // FloatType, FractionalType, IntegerType, IntegralType, LongType, MapType, NativeType, - // NullType, NumericType, ShortType, StringType, StructType - - try { - for (int r = 0; r < maxResult && r < rows.length; r++) { - Object row = rows[r]; - Method isNullAt = row.getClass().getMethod("isNullAt", int.class); - Method apply = row.getClass().getMethod("apply", int.class); - - for (int i = 0; i < columns.size(); i++) { - if (!(Boolean) isNullAt.invoke(row, i)) { - msg += apply.invoke(row, i).toString(); - } else { - msg += "null"; - } - if (i != columns.size() - 1) { - msg += "\t"; - } - } - msg += "\n"; - } - } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - throw new InterpreterException(e); - } - - if (rows.length > maxResult) { - msg += "\nResults are limited by " + maxResult + "."; - } - InterpreterResult rett = new InterpreterResult(Code.SUCCESS, "%table " + msg); - sc.clearJobGroup(); + InterpreterResult rett = new InterpreterResult(Code.SUCCESS, msg); return rett; } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index bbccf0ccce0..e0e5d7acd83 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -22,6 +22,8 @@ import static scala.collection.JavaConversions.collectionAsScalaIterable; import java.io.PrintStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; @@ -30,6 +32,8 @@ import org.apache.spark.SparkContext; import org.apache.spark.sql.SQLContext; +import org.apache.spark.sql.SQLContext.QueryExecution; +import org.apache.spark.sql.catalyst.expressions.Attribute; import org.apache.spark.sql.hive.HiveContext; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; @@ -54,15 +58,18 @@ public class ZeppelinContext extends HashMap { private DependencyResolver dep; private PrintStream out; private InterpreterContext interpreterContext; + private int maxResult; public ZeppelinContext(SparkContext sc, SQLContext sql, InterpreterContext interpreterContext, - DependencyResolver dep, PrintStream printStream) { + DependencyResolver dep, PrintStream printStream, + int maxResult) { this.sc = sc; this.sqlContext = sql; this.interpreterContext = interpreterContext; this.dep = dep; this.out = printStream; + this.maxResult = maxResult; } public SparkContext sc; @@ -233,6 +240,127 @@ public void setInterpreterContext(InterpreterContext interpreterContext) { this.interpreterContext = interpreterContext; } + public void setMaxResult(int maxResult) { + this.maxResult = maxResult; + } + + /** + * show DataFrame or SchemaRDD + * @param o DataFrame or SchemaRDD object + */ + public void show(Object o) { + show(o, maxResult); + } + + /** + * show DataFrame or SchemaRDD + * @param o DataFrame or SchemaRDD object + * @param maxResult maximum number of rows to display + */ + public void show(Object o, int maxResult) { + Class cls = null; + try { + cls = this.getClass().forName("org.apache.spark.sql.DataFrame"); + } catch (ClassNotFoundException e) { + } + + if (cls == null) { + try { + cls = this.getClass().forName("org.apache.spark.sql.SchemaRDD"); + } catch (ClassNotFoundException e) { + } + } + + if (cls == null) { + throw new InterpreterException("Can not road DataFrame/SchemaRDD class"); + } + + if (cls.isInstance(o)) { + out.print(showRDD(sc, interpreterContext, o, maxResult)); + } else { + out.print(o.toString()); + } + } + + public static String showRDD(SparkContext sc, + InterpreterContext interpreterContext, + Object rdd, int maxResult) { + Object[] rows = null; + Method take; + String jobGroup = "zeppelin-" + interpreterContext.getParagraphId(); + sc.setJobGroup(jobGroup, "Zeppelin", false); + + try { + take = rdd.getClass().getMethod("take", int.class); + rows = (Object[]) take.invoke(rdd, maxResult + 1); + + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + sc.clearJobGroup(); + throw new InterpreterException(e); + } + + String msg = null; + + // get field names + Method queryExecution; + QueryExecution qe; + try { + queryExecution = rdd.getClass().getMethod("queryExecution"); + qe = (QueryExecution) queryExecution.invoke(rdd); + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + throw new InterpreterException(e); + } + + List columns = + scala.collection.JavaConverters.asJavaListConverter( + qe.analyzed().output()).asJava(); + + for (Attribute col : columns) { + if (msg == null) { + msg = col.name(); + } else { + msg += "\t" + col.name(); + } + } + + msg += "\n"; + + // ArrayType, BinaryType, BooleanType, ByteType, DecimalType, DoubleType, DynamicType, + // FloatType, FractionalType, IntegerType, IntegralType, LongType, MapType, NativeType, + // NullType, NumericType, ShortType, StringType, StructType + + try { + for (int r = 0; r < maxResult && r < rows.length; r++) { + Object row = rows[r]; + Method isNullAt = row.getClass().getMethod("isNullAt", int.class); + Method apply = row.getClass().getMethod("apply", int.class); + + for (int i = 0; i < columns.size(); i++) { + if (!(Boolean) isNullAt.invoke(row, i)) { + msg += apply.invoke(row, i).toString(); + } else { + msg += "null"; + } + if (i != columns.size() - 1) { + msg += "\t"; + } + } + msg += "\n"; + } + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + throw new InterpreterException(e); + } + + if (rows.length > maxResult) { + msg += "\nResults are limited by " + maxResult + "."; + } + sc.clearJobGroup(); + return "%table " + msg; + } + /** * Run paragraph by id * @param id From 2ebfa595b5647ff087f9a548cdce6fe9e88f35ce Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 8 Apr 2015 08:58:25 +0900 Subject: [PATCH 16/21] Let z.run optionally take InterpreterContext --- .../zeppelin/spark/ZeppelinContext.java | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index e0e5d7acd83..63355a72cbe 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -366,11 +366,20 @@ public static String showRDD(SparkContext sc, * @param id */ public void run(String id) { - if (id.equals(interpreterContext.getParagraphId())) { + run(id, interpreterContext); + } + + /** + * Run paragraph by id + * @param id + * @param context + */ + public void run(String id, InterpreterContext context) { + if (id.equals(context.getParagraphId())) { throw new InterpreterException("Can not run current Paragraph"); } - for (InterpreterContextRunner r : interpreterContext.getRunners()) { + for (InterpreterContextRunner r : context.getRunners()) { if (id.equals(r.getParagraphId())) { r.run(); return; @@ -380,48 +389,64 @@ public void run(String id) { throw new InterpreterException("Paragraph " + id + " not found"); } + /** + * Run paragraph at idx + * @param idx + */ + public void run(int idx) { + run(idx, interpreterContext); + } + /** * Run paragraph at index * @param idx index starting from 0 + * @param context interpreter context */ - public void run(int idx) { - if (idx >= interpreterContext.getRunners().size()) { + public void run(int idx, InterpreterContext context) { + if (idx >= context.getRunners().size()) { throw new InterpreterException("Index out of bound"); } - InterpreterContextRunner runner = interpreterContext.getRunners().get(idx); - if (runner.getParagraphId().equals(interpreterContext.getParagraphId())) { + InterpreterContextRunner runner = context.getRunners().get(idx); + if (runner.getParagraphId().equals(context.getParagraphId())) { throw new InterpreterException("Can not run current Paragraph"); } runner.run(); } + public void run(List paragraphIdOrIdx) { + run(paragraphIdOrIdx, interpreterContext); + } + /** * Run paragraphs * @param paragraphIdOrIdxs list of paragraph id or idx */ - public void run(List paragraphIdOrIdx) { + public void run(List paragraphIdOrIdx, InterpreterContext context) { for (Object idOrIdx : paragraphIdOrIdx) { if (idOrIdx instanceof String) { String id = (String) idOrIdx; - run(id); + run(id, context); } else if (idOrIdx instanceof Integer) { Integer idx = (Integer) idOrIdx; - run(idx); + run(idx, context); } else { throw new InterpreterException("Paragraph " + idOrIdx + " not found"); } } } + public void runAll() { + runAll(interpreterContext); + } /** * Run all paragraphs. except this. */ - public void runAll() { - for (InterpreterContextRunner r : interpreterContext.getRunners()) { - if (r.getParagraphId().equals(interpreterContext.getParagraphId())) { + public void runAll(InterpreterContext context) { + for (InterpreterContextRunner r : context.getRunners()) { + if (r.getParagraphId().equals(context.getParagraphId())) { // skip itself continue; } From 46dba2f355723fab775cd4b0b6e7a610fb74c761 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 8 Apr 2015 09:20:51 +0900 Subject: [PATCH 17/21] Catch sql syntax error --- .../apache/zeppelin/spark/SparkSqlInterpreter.java | 12 +++++++----- .../zeppelin/spark/SparkSqlInterpreterTest.java | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index e4eece81ba4..618579d0eaa 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -123,11 +123,13 @@ public InterpreterResult interpret(String st, InterpreterContext context) { sc.setLocalProperty("spark.scheduler.pool", null); } - Object rdd = sqlc.sql(st); - String msg = ZeppelinContext.showRDD(sc, context, rdd, maxResult); - - InterpreterResult rett = new InterpreterResult(Code.SUCCESS, msg); - return rett; + try { + Object rdd = sqlc.sql(st); + String msg = ZeppelinContext.showRDD(sc, context, rdd, maxResult); + return new InterpreterResult(Code.SUCCESS, msg); + } catch (Exception e) { + return new InterpreterResult(Code.ERROR, e.getMessage()); + } } @Override diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java index 19a760c16aa..27198b37aa0 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java @@ -83,7 +83,7 @@ public void test() { assertEquals(Type.TABLE, ret.type()); assertEquals("name\tage\nmoon\t33\npark\t34\n", ret.message()); - assertEquals(InterpreterResult.Code.ERROR, sql.interpret("select wrong syntax", context).code()); + assertEquals(InterpreterResult.Code.ERROR, sql.interpret("select wrong syntax", context).code()); assertEquals(InterpreterResult.Code.SUCCESS, sql.interpret("select case when name==\"aa\" then name else name end from people", context).code()); } From 88fd6357db0698f052ee2bdbe7213a9357e853ae Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 8 Apr 2015 09:21:05 +0900 Subject: [PATCH 18/21] catch and print watcher user provided routine exception --- .../java/org/apache/zeppelin/display/AngularObject.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java index b8fa9a185e9..41c573d04e9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java @@ -23,6 +23,8 @@ import java.util.concurrent.ExecutorService; import org.apache.zeppelin.scheduler.ExecutorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @@ -105,6 +107,7 @@ public void set(T o, boolean emit) { emit(); } + final Logger logger = LoggerFactory.getLogger(AngularObject.class); List ws = new LinkedList(); synchronized (watchers) { ws.addAll(watchers); @@ -115,7 +118,11 @@ public void set(T o, boolean emit) { executor.submit(new Runnable() { @Override public void run() { - w.watch(before, after); + try { + w.watch(before, after); + } catch (Exception e) { + logger.error("Exception on watch", e); + } } }); } From 80760983617af6dbf79229a13d6d5285d02be58d Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 8 Apr 2015 12:06:06 +0900 Subject: [PATCH 19/21] Remove unnecessary type information --- .../zeppelin/display/AngularObject.java | 28 ---------------- .../remote/RemoteInterpreterServer.java | 32 ++++++++++++------- 2 files changed, 21 insertions(+), 39 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java index 41c573d04e9..bbfcd1b184d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java @@ -19,7 +19,6 @@ import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.concurrent.ExecutorService; import org.apache.zeppelin.scheduler.ExecutorFactory; @@ -32,44 +31,17 @@ * @param */ public class AngularObject { - - /** - * - */ - public static enum AngularObjectType { - STRING, - MAP, - FUNCTION - }; - private String name; private T object; private transient AngularObjectListener listener; private transient List watchers = new LinkedList(); - private AngularObjectType type; protected AngularObject(String name, T o, AngularObjectListener listener) { this.name = name; this.listener = listener; object = o; - type = checkType(); - } - - public AngularObjectType getType() { - return type; - } - - public AngularObjectType checkType() { - if (object == null) { - return AngularObjectType.STRING; - } else if (object instanceof Map) { - return AngularObjectType.MAP; - } else if (object instanceof String) { - return AngularObjectType.STRING; - } - return AngularObjectType.STRING; } public String getName() { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 8209696f1a3..8b4b2363027 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -31,7 +31,6 @@ import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TTransportException; import org.apache.zeppelin.display.AngularObject; -import org.apache.zeppelin.display.AngularObject.AngularObjectType; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.display.GUI; @@ -445,16 +444,27 @@ public void angularObjectUpdate(String name, String object) return; } - if (ao.getType() == AngularObjectType.STRING) { - String value = gson.fromJson(object, String.class); - ao.set(value, false); - } else if (ao.getType() == AngularObjectType.MAP) { - Map value = gson.fromJson(object, - new TypeToken>() { - }.getType()); - ao.set(value, false); - } else { - logger.error("Update angular object type {} not supported", ao.getType()); + if (object == null) { + ao.set(null, false); + return; + } + + Object oldObject = ao.get(); + if (oldObject != null) { // first try with previous object's type + Object value; + try { + value = gson.fromJson(object, oldObject.getClass()); + ao.set(value, false); + return; + } catch (Exception e) { + // no luck + } } + + // Generic java object type for json. + Map value = gson.fromJson(object, + new TypeToken>() { + }.getType()); + ao.set(value, false); } } From 34fa2981518cde0e623ad5135dce0f51a5e90229 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 10 Apr 2015 07:24:33 +0900 Subject: [PATCH 20/21] jquery to angular --- zeppelin-web/app/scripts/controllers/paragraph.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zeppelin-web/app/scripts/controllers/paragraph.js b/zeppelin-web/app/scripts/controllers/paragraph.js index 3d26b249e6b..3b1d05f8804 100644 --- a/zeppelin-web/app/scripts/controllers/paragraph.js +++ b/zeppelin-web/app/scripts/controllers/paragraph.js @@ -81,11 +81,11 @@ angular.module('zeppelinWebApp') $scope.renderAngular = function() { var retryRenderer = function() { - if ($('#p'+$scope.paragraph.id+'_angular').length) { + if (angular.element('#p'+$scope.paragraph.id+'_angular').length) { try { - $('#p'+$scope.paragraph.id+'_angular').html($scope.paragraph.result.msg); + angular.element('#p'+$scope.paragraph.id+'_angular').html($scope.paragraph.result.msg); - $compile($('#p'+$scope.paragraph.id+'_angular').contents())($rootScope.compiledScope); + $compile(angular.element('#p'+$scope.paragraph.id+'_angular').contents())($rootScope.compiledScope); } catch(err) { console.log('ANGULAR rendering error %o', err); } From 04d717596889a78665b51b0474031f9214d043bc Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 11 Apr 2015 11:15:05 +0900 Subject: [PATCH 21/21] Remove implicit conversion because of side effect --- .../zeppelin/spark/SparkInterpreter.java | 10 ------- .../zeppelin/spark/ZeppelinContext.java | 27 +++++++++++++++++++ .../display/AngularObjectWatcher.java | 16 +++++++++-- .../zeppelin/display/AngularObjectTest.java | 5 ++-- .../remote/mock/MockInterpreterAngular.java | 18 +++++++------ 5 files changed, 54 insertions(+), 22 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index df037432c1c..1b0b8201e0a 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -464,16 +464,6 @@ public void open() { } } } - - // add some implicit conversion - intp.interpret("implicit def watcherConversion(watcher: (Object, Object) => Unit):" - + "org.apache.zeppelin.display.AngularObjectWatcher = {" - + " new org.apache.zeppelin.display.AngularObjectWatcher() {" - + " def watch(before:Object, after:Object) = {" - + " watcher(before, after)" - + " }" - + " }" - + "}"); } private List currentClassPath() { diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index 63355a72cbe..2c03f1cd91b 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -46,6 +46,7 @@ import org.apache.zeppelin.spark.dep.DependencyResolver; import scala.Tuple2; +import scala.Unit; import scala.collection.Iterable; /** @@ -502,6 +503,32 @@ public void angularWatch(String name, AngularObjectWatcher w) { } } + + public void angularWatch(String name, + final scala.Function2 func) { + AngularObjectWatcher w = new AngularObjectWatcher(getInterpreterContext()) { + @Override + public void watch(Object oldObject, Object newObject, + InterpreterContext context) { + func.apply(newObject, newObject); + } + }; + angularWatch(name, w); + } + + public void angularWatch( + String name, + final scala.Function3 func) { + AngularObjectWatcher w = new AngularObjectWatcher(getInterpreterContext()) { + @Override + public void watch(Object oldObject, Object newObject, + InterpreterContext context) { + func.apply(oldObject, newObject, context); + } + }; + angularWatch(name, w); + } + public void angularUnwatch(String name, AngularObjectWatcher w) { AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry(); if (registry.get(name) != null) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectWatcher.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectWatcher.java index a41fa33119d..c5bd5e27c18 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectWatcher.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectWatcher.java @@ -17,9 +17,21 @@ package org.apache.zeppelin.display; +import org.apache.zeppelin.interpreter.InterpreterContext; + /** * */ -public interface AngularObjectWatcher { - public void watch(Object oldObject, Object newObject); +public abstract class AngularObjectWatcher { + private InterpreterContext context; + + public AngularObjectWatcher(InterpreterContext context) { + this.context = context; + } + + void watch(Object oldObject, Object newObject) { + watch(oldObject, newObject, context); + } + + public abstract void watch(Object oldObject, Object newObject, InterpreterContext context); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java index 7d6b000bece..7ccc934c170 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger; +import org.apache.zeppelin.interpreter.InterpreterContext; import org.junit.Test; public class AngularObjectTest { @@ -61,9 +62,9 @@ public void updated(AngularObject updatedObject) { } }); - ao.addWatcher(new AngularObjectWatcher() { + ao.addWatcher(new AngularObjectWatcher(null) { @Override - public void watch(Object oldObject, Object newObject) { + public void watch(Object oldObject, Object newObject, InterpreterContext context) { onWatch.incrementAndGet(); } }); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java index c2468112e06..ff1b8edeea0 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java @@ -29,7 +29,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -public class MockInterpreterAngular extends Interpreter implements AngularObjectWatcher { +public class MockInterpreterAngular extends Interpreter { static { Interpreter.register( "angularTest", @@ -72,7 +72,15 @@ public InterpreterResult interpret(String st, InterpreterContext context) { if (cmd.equals("add")) { registry.add(name, value); - registry.get(name).addWatcher(this); + registry.get(name).addWatcher(new AngularObjectWatcher(null) { + + @Override + public void watch(Object oldObject, Object newObject, + InterpreterContext context) { + numWatch.incrementAndGet(); + } + + }); } else if (cmd.equalsIgnoreCase("update")) { registry.get(name).set(value); } else if (cmd.equals("remove")) { @@ -106,10 +114,4 @@ public int getProgress(InterpreterContext context) { public List completion(String buf, int cursor) { return null; } - - @Override - public void watch(Object oldObject, Object newObject) { - numWatch.incrementAndGet(); - } - }