From 1aa1d68da0ca6af207c5ffec75cb29f895acf637 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 22 Jul 2020 17:30:32 +0800 Subject: [PATCH 01/19] [ZEPPELIN-4981]. Zeppelin Client API (Zeppelin SDK) --- pom.xml | 1 + spark/pom.xml | 2 + zeppelin-client/pom.xml | 113 +++ .../client/AbstractMessageHandler.java | 75 ++ .../apache/zeppelin/client/ClientConfig.java | 54 ++ .../apache/zeppelin/client/ExecuteResult.java | 72 ++ .../zeppelin/client/FlinkMessageHandler.java | 45 + .../zeppelin/client/MessageHandler.java | 25 + .../apache/zeppelin/client/NoteResult.java | 45 + .../zeppelin/client/ParagraphResult.java | 111 +++ .../client/ParagraphResultCollector.java | 64 ++ .../org/apache/zeppelin/client/Result.java | 56 ++ .../zeppelin/client/SimpleMessageHandler.java | 36 + .../org/apache/zeppelin/client/Status.java | 50 + .../org/apache/zeppelin/client/ZSession.java | 314 +++++++ .../zeppelin/client/ZeppelinClient.java | 753 +++++++++++++++ .../client/ZeppelinWebSocketClient.java | 139 +++ .../src/main/resources/log4j.properties | 28 + zeppelin-interpreter-integration/pom.xml | 12 + .../integration/ZSessionIntegrationTest.java | 516 ++++++++++ .../ZeppelinClientIntegrationTest.java | 384 ++++++++ ...ZeppelinClientWithAuthIntegrationTest.java | 105 +++ .../src/test/resources/log4j.properties | 6 +- .../interpreter/ExecutionContext.java | 9 +- .../interpreter/ExecutionContextBuilder.java | 9 +- .../zeppelin/rest/InterpreterRestApi.java | 2 + .../apache/zeppelin/rest/NotebookRestApi.java | 879 ++++++++++-------- .../apache/zeppelin/rest/SessionManager.java | 52 ++ .../apache/zeppelin/rest/SessionRestApi.java | 114 +++ .../zeppelin/service/NotebookService.java | 52 +- .../zeppelin/socket/NotebookServer.java | 4 +- .../socket/NotebookWebSocketCreator.java | 1 + .../src/main/resources/log4j.properties | 28 + .../zeppelin/rest/AbstractTestRestApi.java | 1 + .../zeppelin/rest/NotebookRestApiTest.java | 11 +- .../rest/NotebookSecurityRestApiTest.java | 8 +- .../zeppelin/service/NotebookServiceTest.java | 20 +- .../src/test/resources/zeppelin-site.xml | 2 +- .../interpreter/InterpreterSetting.java | 8 + .../RemoteInterpreterEventServer.java | 3 +- .../org/apache/zeppelin/notebook/Note.java | 14 +- .../apache/zeppelin/notebook/Paragraph.java | 31 +- .../zeppelin/notebook/NotebookTest.java | 2 +- 43 files changed, 3840 insertions(+), 416 deletions(-) create mode 100644 zeppelin-client/pom.xml create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/ClientConfig.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/FlinkMessageHandler.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResultCollector.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/Status.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java create mode 100644 zeppelin-client/src/main/resources/log4j.properties create mode 100644 zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java create mode 100644 zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientIntegrationTest.java create mode 100644 zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java create mode 100644 zeppelin-server/src/main/resources/log4j.properties diff --git a/pom.xml b/pom.xml index 47e6e02a00c..2cbc42a6ec5 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,7 @@ geode ksql sparql + zeppelin-client zeppelin-web zeppelin-server zeppelin-jupyter diff --git a/spark/pom.xml b/spark/pom.xml index 25301b29975..022c3cd9d98 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -42,6 +42,8 @@ 2.4.5 + 2.5.0 + 0.10.7 2.11.12 2.11 diff --git a/zeppelin-client/pom.xml b/zeppelin-client/pom.xml new file mode 100644 index 00000000000..b99f8581dd2 --- /dev/null +++ b/zeppelin-client/pom.xml @@ -0,0 +1,113 @@ + + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../pom.xml + + + org.apache.zeppelin + zeppelin-client + jar + 0.9.0-SNAPSHOT + Zeppelin: Client + Zeppelin Client + + + 3.7.04 + 1.8 + + + + + + org.apache.zeppelin + zeppelin-zengine + ${project.version} + + + + com.konghq + unirest-java + ${unirest.version} + standalone + + + + org.apache.commons + commons-lang3 + + + + org.eclipse.jetty.websocket + websocket-client + ${jetty.version} + + + + org.apache.commons + commons-text + ${commons-text.version} + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + junit + junit + test + + + + org.mockito + mockito-all + test + + + + + + + src/main/resources + true + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + + diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java new file mode 100644 index 00000000000..93e47767fa5 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client; + +import com.google.gson.Gson; +import org.apache.zeppelin.notebook.socket.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class AbstractMessageHandler implements MessageHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMessageHandler.class); + private static final Gson GSON = new Gson(); + + @Override + public void onMessage(String msg) { + try { + Message messageReceived = GSON.fromJson(msg, Message.class); + if (messageReceived.op != Message.OP.PING) { + LOGGER.debug("RECEIVE: " + messageReceived.op + + ", RECEIVE PRINCIPAL: " + messageReceived.principal + + ", RECEIVE TICKET: " + messageReceived.ticket + + ", RECEIVE ROLES: " + messageReceived.roles + + ", RECEIVE DATA: " + messageReceived.data); + } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("RECEIVE MSG = " + messageReceived); + } + + // Lets be elegant here + switch (messageReceived.op) { + case PARAGRAPH_UPDATE_OUTPUT: + String noteId = (String) messageReceived.data.get("noteId"); + String paragraphId = (String) messageReceived.data.get("paragraphId"); + int index = (int) Double.parseDouble(messageReceived.data.get("index").toString()); + String type = (String) messageReceived.data.get("type"); + String output = (String) messageReceived.data.get("data"); + onStatementUpdateOutput(paragraphId, index, type, output); + break; + case PARAGRAPH_APPEND_OUTPUT: + noteId = (String) messageReceived.data.get("noteId"); + paragraphId = (String) messageReceived.data.get("paragraphId"); + index = (int) Double.parseDouble(messageReceived.data.get("index").toString()); + output = (String) messageReceived.data.get("data"); + onStatementAppendOutput(paragraphId, index, output); + break; + default: + break; + } + } catch (Exception e) { + LOGGER.error("Can't handle message: " + msg, e); + } + + } + + public abstract void onStatementAppendOutput(String statementId, int index, String output); + + public abstract void onStatementUpdateOutput(String statementId, int index, String type, String output); + +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ClientConfig.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ClientConfig.java new file mode 100644 index 00000000000..aaf84fd9747 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ClientConfig.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client; + +/** + * Configuration of Zeppelin client, such as zeppelin server rest url and + * query interval of polling note/paragraph result. + */ +public class ClientConfig { + private String zeppelinRestUrl; + private long queryInterval ; + private boolean useKnox = false; + + public ClientConfig(String zeppelinRestUrl) { + this(zeppelinRestUrl, 1000); + } + + public ClientConfig(String zeppelinRestUrl, long queryInterval) { + this(zeppelinRestUrl, queryInterval, false); + } + + public ClientConfig(String zeppelinRestUrl, long queryInterval, boolean useKnox) { + this.zeppelinRestUrl = zeppelinRestUrl; + this.queryInterval = queryInterval; + this.useKnox = useKnox; + } + + public String getZeppelinRestUrl() { + return zeppelinRestUrl; + } + + public long getQueryInterval() { + return queryInterval; + } + + public boolean isUseKnox() { + return useKnox; + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java new file mode 100644 index 00000000000..2ebf11aaa5b --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.client; + +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; + +/** + * Execution result of each statement. + * + */ +public class ExecuteResult { + + private String statementId; + private Status status; + private List results; + private List jobUrls; + private List bufferedResults; + private int progress; + + public ExecuteResult(ParagraphResult paragraphResult) { + this.statementId = paragraphResult.getParagraphId(); + this.status = paragraphResult.getStatus(); + this.progress = paragraphResult.getProgress(); + this.results = paragraphResult.getResults(); + this.jobUrls = paragraphResult.getJobUrls(); + } + + public String getStatementId() { + return statementId; + } + + public Status getStatus() { + return status; + } + + public List getResults() { + return results; + } + + public List getJobUrls() { + return jobUrls; + } + + @Override + public String toString() { + return "ExecuteResult{" + + "status=" + status + + ", progress=" + progress + + ", results=" + StringUtils.join(results, ", ") + + ", jobUrls=" + jobUrls + + '}'; + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/FlinkMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/FlinkMessageHandler.java new file mode 100644 index 00000000000..bd1f105eadc --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/FlinkMessageHandler.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.client; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +public class FlinkMessageHandler extends AbstractMessageHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(FlinkMessageHandler.class); + + private Map resultMap = new HashMap<>(); + + @Override + public void onStatementAppendOutput(String statementId, int index, String output) { + + } + + @Override + public void onStatementUpdateOutput(String statementId, int index, String type, String output) { + if (resultMap.containsKey(statementId)) { +// LOGGER.info("result") + } + resultMap.put(statementId, new Result(type, output)); + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java new file mode 100644 index 00000000000..b52737e80f7 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.client; + +public interface MessageHandler { + + void onMessage(String msg); + +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java new file mode 100644 index 00000000000..e6a5a363af3 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.client; + +import java.util.List; + +public class NoteResult { + private String noteId; + private boolean isRunning; + private List paragraphResultList; + + public NoteResult(String noteId, boolean isRunning, List paragraphResultList) { + this.noteId = noteId; + this.isRunning = isRunning; + this.paragraphResultList = paragraphResultList; + } + + public String getNoteId() { + return noteId; + } + + public boolean isRunning() { + return isRunning; + } + + public List getParagraphResultList() { + return paragraphResultList; + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java new file mode 100644 index 00000000000..95ee88eceda --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.client; + +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +public class ParagraphResult { + private String paragraphId; + private Status status; + private int progress; + private List results; + private List jobUrls; + + public ParagraphResult(JSONObject paragraphJson) { + this.paragraphId = paragraphJson.getString("id"); + this.status = Status.valueOf(paragraphJson.getString("status")); + this.progress = paragraphJson.getInt("progress"); + this.results = new ArrayList<>(); + if (paragraphJson.has("results")) { + JSONObject resultJson = paragraphJson.getJSONObject("results"); + JSONArray msgArray = resultJson.getJSONArray("msg"); + for (int i = 0; i < msgArray.length(); ++i) { + JSONObject resultObject = msgArray.getJSONObject(i); + results.add(new Result(resultObject)); + } + } + + this.jobUrls = new ArrayList<>(); + if (paragraphJson.has("runtimeInfos")) { + JSONObject runtimeInfosJson = paragraphJson.getJSONObject("runtimeInfos"); + if (runtimeInfosJson.has("jobUrl")) { + JSONObject jobUrlJson = runtimeInfosJson.getJSONObject("jobUrl"); + if (jobUrlJson.has("values")) { + JSONArray valuesArray = jobUrlJson.getJSONArray("values"); + for (int i=0;i< valuesArray.length(); ++i) { + JSONObject object = valuesArray.getJSONObject(i); + if (object.has("jobUrl")) { + jobUrls.add(object.getString("jobUrl")); + } + } + } + } + } + } + + public String getParagraphId() { + return paragraphId; + } + + public Status getStatus() { + return status; + } + + public int getProgress() { + return progress; + } + + public List getResults() { + return results; + } + + public List getJobUrls() { + return jobUrls; + } + + /** + * + * @return + */ + public String getMessage() { + StringBuilder builder = new StringBuilder(); + if (results != null) { + for (Result result : results) { + builder.append(result.getData() + "\n"); + } + } + return builder.toString(); + } + + @Override + public String toString() { + return "ParagraphResult{" + + "paragraphId='" + paragraphId + '\'' + + ", status=" + status + + ", results=" + StringUtils.join(results, ", ") + + ", progress=" + progress + + '}'; + } + +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResultCollector.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResultCollector.java new file mode 100644 index 00000000000..4238a3e5f8b --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResultCollector.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class ParagraphResultCollector { + + private static final Logger LOGGER = LoggerFactory.getLogger(ParagraphResultCollector.class); + + private String noteId; + private String paragraphId; + private List results = new ArrayList<>(); + + public void onOutputAppend(String noteId, + String paragraphId, + int index, + String output) { + if (index > (results.size() - 1)) { + LOGGER.warn("Get output append for index: " + index + + ", but currently there's only " + results.size() + " results"); + } + results.get(index).appendData(output); + } + + + public void onOutputUpdated(String noteId, + String paragraphId, + int index, + String type, + String output) { + Result result = new Result(type, output); + if (index < results.size()) { + results.set(index, result); + } else { + results.add(result); + } + } + + public void onOutputClear(String noteId, + String paragraphId) { + + } + +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java new file mode 100644 index 00000000000..d965fb5a0b6 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.client; + +import kong.unirest.json.JSONObject; + +public class Result { + private String type; + private String data; + + public Result(JSONObject jsonObject) { + this.type = jsonObject.getString("type"); + this.data = jsonObject.getString("data"); + } + + public Result(String type, String data) { + this.type = type; + this.data = data; + } + + public String getType() { + return type; + } + + public String getData() { + return data; + } + + public void appendData(String newData) { + this.data = this.data + newData; + } + + @Override + public String toString() { + return "Result{" + + "type='" + type + '\'' + + ", data='" + data + '\'' + + '}'; + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java new file mode 100644 index 00000000000..258f869c40e --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SimpleMessageHandler extends AbstractMessageHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(SimpleMessageHandler.class); + + @Override + public void onStatementAppendOutput(String statementId, int index, String output) { + LOGGER.info("Append output, data: {}", output); + } + + @Override + public void onStatementUpdateOutput(String statementId, int index, String type, String output) { + LOGGER.info("Update output, type: {}, data: {}", type, output); + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Status.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Status.java new file mode 100644 index 00000000000..a22a02664bf --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Status.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.client; + +/** + * Job status. + * + * UNKNOWN - Job is not found in remote + * READY - Job is not running, ready to run. + * PENDING - Job is submitted to scheduler. but not running yet + * RUNNING - Job is running. + * FINISHED - Job finished run. with success + * ERROR - Job finished run. with error + * ABORT - Job finished by abort + */ +public enum Status { + UNKNOWN, READY, PENDING, RUNNING, FINISHED, ERROR, ABORT; + + public boolean isReady() { + return this == READY; + } + + public boolean isRunning() { + return this == RUNNING; + } + + public boolean isPending() { + return this == PENDING; + } + + public boolean isCompleted() { + return this == FINISHED || this == ERROR || this == ABORT; + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java new file mode 100644 index 00000000000..d9b3027aa88 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client; + +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.notebook.socket.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * Each ZSession represent one interpreter process, you can start/stop it, and execute/submit/cancel code. + * There's no Zeppelin concept(like note/paragraph) in ZSession. + * + */ +public class ZSession { + private static final Logger LOGGER = LoggerFactory.getLogger(ZSession.class); + + private ZeppelinClient zeppelinClient; + private String interpreter; + private Map intpProperties; + private int maxStatement; + + private String sessionId; + private String noteId; + private String weburl; + + private ZeppelinWebSocketClient webSocketClient; + + public ZSession(ClientConfig clientConfig, + String interpreter) throws Exception { + this(clientConfig, interpreter, new HashMap<>(), 100); + } + + public ZSession(ClientConfig clientConfig, + String interpreter, + Map intpProperties, + int maxStatement) throws Exception { + this.zeppelinClient = new ZeppelinClient(clientConfig); + this.interpreter = interpreter; + this.intpProperties = intpProperties; + this.maxStatement = maxStatement; + } + + public void start() throws Exception { + start(null); + } + + /** + * Start this ZSession, underneath it would create a note for this ZSession and + * start a dedicated interpreter group. + * + * @throws Exception + */ + public void start(MessageHandler messageHandler) throws Exception { + this.sessionId = zeppelinClient.newSession(interpreter); + + this.noteId = zeppelinClient.createNote("/_ZSession/" + interpreter + "/" + sessionId); + // inline configuration + StringBuilder builder = new StringBuilder("%" + interpreter + ".conf\n"); + if (intpProperties != null) { + for (Map.Entry entry : intpProperties.entrySet()) { + builder.append(entry.getKey() + " " + entry.getValue() + "\n"); + } + } + String paragraphId = zeppelinClient.addParagraph(noteId, "Session Configuration", builder.toString()); + ParagraphResult paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId, sessionId); + if (paragraphResult.getStatus() != Status.FINISHED) { + throw new Exception("Fail to configure session, " + paragraphResult.getMessage()); + } + + // start session + paragraphId = zeppelinClient.addParagraph(noteId, "Session Init", "%" + interpreter); + paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId, sessionId); + if (paragraphResult.getStatus() != Status.FINISHED) { + throw new Exception("Fail to init session, " + paragraphResult.getMessage()); + } + this.weburl = zeppelinClient.getSessionWebUrl(sessionId); + + if (messageHandler != null) { + this.webSocketClient = new ZeppelinWebSocketClient(messageHandler); + this.webSocketClient.connect(zeppelinClient.getClientConfig().getZeppelinRestUrl() + .replace("https", "ws").replace("http", "ws") + "/ws"); + + // call GET_NOTE to establish websocket connection between this session and zeppelin-server + Message msg = new Message(Message.OP.GET_NOTE); + msg.put("id", this.noteId); + this.webSocketClient.send(msg); + } + } + + /** + * Stop this underlying interpreter process. + * + * @throws Exception + */ + public void stop() throws Exception { + if (sessionId != null) { + zeppelinClient.stopSession(interpreter, sessionId); + } + } + + /** + * + * @param code + * @return + * @throws Exception + */ + public ExecuteResult execute(String code) throws Exception { + return execute("", code); + } + + /** + * + * @param subInterpreter + * @param code + * @return + * @throws Exception + */ + public ExecuteResult execute(String subInterpreter, String code) throws Exception { + return execute(subInterpreter, new HashMap<>(), code); + } + + /** + * + * @param subInterpreter + * @param localProperties + * @param code + * @return + * @throws Exception + */ + public ExecuteResult execute(String subInterpreter, Map localProperties, String code) throws Exception { + StringBuilder builder = new StringBuilder("%" + interpreter); + if (!StringUtils.isBlank(subInterpreter)) { + builder.append("." + subInterpreter); + } + if (localProperties != null && !localProperties.isEmpty()) { + builder.append("("); + for (Map.Entry entry : localProperties.entrySet()) { + builder.append(entry.getKey() + "=\"" + entry.getValue() + "\""); + } + builder.append(")"); + } + builder.append("\n" + code); + + String text = builder.toString(); + + String nextParagraphId = zeppelinClient.nextSessionParagraph(noteId, maxStatement); + zeppelinClient.updateParagraph(noteId, nextParagraphId, "", text); + ParagraphResult paragraphResult = zeppelinClient.executeParagraph(noteId, nextParagraphId, sessionId); + return new ExecuteResult(paragraphResult); + } + + /** + * + * @param code + * @return + * @throws Exception + */ + public ExecuteResult submit(String code) throws Exception { + return submit("", code); + } + + /** + * + * @param subInterpreter + * @param code + * @return + * @throws Exception + */ + public ExecuteResult submit(String subInterpreter, String code) throws Exception { + return submit(subInterpreter, new HashMap<>(), code); + } + + /** + * + * @param subInterpreter + * @param code + * @return + * @throws Exception + */ + public ExecuteResult submit(String subInterpreter, Map localProperties, String code) throws Exception { + StringBuilder builder = new StringBuilder("%" + interpreter); + if (!StringUtils.isBlank(subInterpreter)) { + builder.append("." + subInterpreter); + } + if (localProperties != null && !localProperties.isEmpty()) { + builder.append("("); + for (Map.Entry entry : localProperties.entrySet()) { + builder.append(entry.getKey() + "=\"" + entry.getValue() + "\""); + } + builder.append(")"); + } + builder.append("\n" + code); + + String text = builder.toString(); + String nextParagraphId = zeppelinClient.nextSessionParagraph(noteId, maxStatement); + zeppelinClient.updateParagraph(noteId, nextParagraphId, "", text); + ParagraphResult paragraphResult = zeppelinClient.submitParagraph(noteId, nextParagraphId, sessionId); + return new ExecuteResult(paragraphResult); + } + + /** + * + * @param statementId + * @throws Exception + */ + public void cancel(String statementId) throws Exception { + zeppelinClient.cancelParagraph(noteId, statementId); + } + + /** + * + * @param statementId + * @return + * @throws Exception + */ + public ExecuteResult waitUntilFinished(String statementId) throws Exception { + ParagraphResult paragraphResult = zeppelinClient.waitUtilParagraphFinish(noteId, statementId); + return new ExecuteResult(paragraphResult); + } + + /** + * + * @param statementId + * @return + * @throws Exception + */ + public ExecuteResult waitUntilRunning(String statementId) throws Exception { + ParagraphResult paragraphResult = zeppelinClient.waitUtilParagraphRunning(noteId, statementId); + return new ExecuteResult(paragraphResult); + } + + public String getNoteId() { + return noteId; + } + + public String getWeburl() { + return weburl; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private ClientConfig clientConfig; + private String interpreter; + private Map intpProperties; + private int maxStatement = 100; + + public Builder setClientConfig(ClientConfig clientConfig) { + this.clientConfig = clientConfig; + return this; + } + + public Builder setInterpreter(String interpreter) { + this.interpreter = interpreter; + return this; + } + + public Builder setIntpProperties(Map intpProperties) { + this.intpProperties = intpProperties; + return this; + } + + public ZSession build() throws Exception { + return new ZSession(clientConfig, interpreter, intpProperties, maxStatement); + } + } + + public static void main(String[] args) throws Exception { + + ClientConfig clientConfig = new ClientConfig("http://localhost:18086", 1000); +// ZSession hiveSession = new ZSession(clientConfig, "hive", new HashMap<>(), 100); +// hiveSession.start(); +// +// ExecuteResult executeResult = hiveSession.submit("show tables"); +// executeResult = hiveSession.waitUntilFinished(executeResult.getStatementId()); +// System.out.println(executeResult.toString()); +// +// executeResult = hiveSession.submit("select eid, count(1) from employee group by eid"); +// executeResult = hiveSession.waitUntilFinished(executeResult.getStatementId()); +// System.out.println(executeResult.toString()); + + ZSession sparkSession = new ZSession(clientConfig, "sh", new HashMap<>(), 100); + sparkSession.start(); + +// ExecuteResult executeResult = sparkSession.submit("sql", "show tables"); +// executeResult = sparkSession.waitUntilFinished(executeResult.getStatementId()); +// System.out.println(executeResult.toString()); +// +// executeResult = sparkSession.submit("sql", "select eid, count(1) from employee group by eid"); +// executeResult = sparkSession.waitUntilFinished(executeResult.getStatementId()); +// System.out.println(executeResult.toString()); + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java new file mode 100644 index 00000000000..f57471c6551 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java @@ -0,0 +1,753 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.client; + +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.apache.ApacheClient; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; +import org.apache.commons.text.StringEscapeUtils; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.ssl.SSLContextBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import unirest.shaded.org.apache.http.client.HttpClient; +import unirest.shaded.org.apache.http.impl.client.HttpClients; + +import javax.net.ssl.SSLContext; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Low level api for interacting with Zeppelin. Underneath, it use the zeppelin rest api. + * You can use this class to operate Zeppelin note/paragraph, + * e.g. get/add/delete/update/execute/cancel + */ +public class ZeppelinClient { + private static final Logger LOGGER = LoggerFactory.getLogger(ZeppelinClient.class); + + private ClientConfig clientConfig; + + public ZeppelinClient(ClientConfig clientConfig) throws Exception { + this.clientConfig = clientConfig; + Unirest.config().defaultBaseUrl(clientConfig.getZeppelinRestUrl() + "/api"); + + if (clientConfig.isUseKnox()) { + try { + SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustSelfSignedStrategy() { + public boolean isTrusted(X509Certificate[] chain, String authType) { + return true; + } + }).build(); + HttpClient customHttpClient = HttpClients.custom().setSSLContext(sslContext) + .setSSLHostnameVerifier(new NoopHostnameVerifier()).build(); + Unirest.config().httpClient(ApacheClient.builder(customHttpClient)); + } catch (Exception e) { + throw new Exception("Fail to setup httpclient of Unirest", e); + } + } + + Unirest.config().setDefaultHeader("Origin", "localhost"); + } + + public ClientConfig getClientConfig() { + return clientConfig; + } + + /** + * Throw exception is the status code is not 200. + * + * @param response + * @throws Exception + */ + private void checkResponse(HttpResponse response) throws Exception { + if (response.getStatus() != 200) { + throw new Exception(String.format("Unable to call rest api, status: %s, statusText: %s, error: %s", + response.getStatus(), + response.getStatusText(), + response.getBody().toPrettyString())); + } + } + + /** + * Throw exception if the status in the json object is not `OK`. + * + * @param jsonNode + * @throws Exception + */ + private void checkJsonNodeStatus(JsonNode jsonNode) throws Exception { + if (! "OK".equalsIgnoreCase(jsonNode.getObject().getString("status"))) { + throw new Exception(StringEscapeUtils.unescapeJava(jsonNode.getObject().getString("message"))); + } + } + + /** + * Get Zeppelin version. + * + * @return + * @throws Exception + */ + public String getVersion() throws Exception { + HttpResponse response = Unirest + .get("/version") + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + return jsonNode.getObject().getJSONObject("body").getString("version"); + } + + /** + * Request a new session id. It doesn't create session (interpreter process) in zeppelin server side, but just + * create a unique session id. + * + * @param interpreter + * @return + * @throws Exception + */ + public String newSession(String interpreter) throws Exception { + HttpResponse response = Unirest + .post("/session/{interpreter}") + .routeParam("interpreter", interpreter) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + return jsonNode.getObject().getString("message"); + } + + /** + * Stop the session(interpreter process) in zeppelin server. + * + * @param interpreter + * @param sessionId + * @throws Exception + */ + public void stopSession(String interpreter, String sessionId) throws Exception { + HttpResponse response = Unirest + .delete("/session/{interpreter}/{sessionId}") + .routeParam("interpreter", interpreter) + .routeParam("sessionId", sessionId) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + } + + /** + * Get the session weburl. It is spark ui url for spark interpreter, + * or flink web ui for flink interpreter, or may be null for the interpreter that has no weburl. + * + * @param sessionId + * @throws Exception + */ + public String getSessionWebUrl(String sessionId) throws Exception { + HttpResponse response = Unirest + .get("/session/{sessionId}") + .routeParam("sessionId", sessionId) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + + JSONObject bodyObject = jsonNode.getObject().getJSONObject("body"); + if (bodyObject.has("weburl")) { + return bodyObject.getString("weburl"); + } else { + return null; + } + } + + /** + * Login zeppelin with userName and password, throw exception if login fails. + * + * @param userName + * @param password + * @throws Exception + */ + public void login(String userName, String password) throws Exception { + if (clientConfig.isUseKnox()) { + HttpResponse response = Unirest.get("/") + .basicAuth(userName, password) + .asString(); + if (response.getStatus() != 200) { + throw new Exception(String.format("Login failed, status: %s, statusText: %s", + response.getStatus(), + response.getStatusText())); + } + } else { + HttpResponse response = Unirest + .post("/login") + .field("userName", userName) + .field("password", password) + .asJson(); + if (response.getStatus() != 200) { + throw new Exception(String.format("Login failed, status: %s, statusText: %s", + response.getStatus(), + response.getStatusText())); + } + } + } + + /** + * Create a new empty note with provided notePath. + * + * @param notePath + * @return + * @throws Exception + */ + public String createNote(String notePath) throws Exception { + JSONObject bodyObject = new JSONObject(); + bodyObject.put("name", notePath); + HttpResponse response = Unirest + .post("/notebook") + .body(bodyObject.toString()) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + + return jsonNode.getObject().getString("body"); + } + + /** + * Delete note with provided noteId. + * + * @param noteId + * @throws Exception + */ + public void deleteNote(String noteId) throws Exception { + HttpResponse response = Unirest + .delete("/notebook/{noteId}") + .routeParam("noteId", noteId) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + } + + /** + * Query {@link NoteResult} with provided noteId. + * + * @param noteId + * @return + * @throws Exception + */ + public NoteResult queryNoteResult(String noteId) throws Exception { + HttpResponse response = Unirest + .get("/notebook/{noteId}") + .routeParam("noteId", noteId) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + + JSONObject noteJsonObject = jsonNode.getObject().getJSONObject("body"); + boolean isRunning = false; + if (noteJsonObject.has("info")) { + JSONObject infoJsonObject = noteJsonObject.getJSONObject("info"); + if (infoJsonObject.has("isRunning")) { + isRunning = Boolean.parseBoolean(infoJsonObject.getString("isRunning")); + } + } + + List paragraphResultList = new ArrayList<>(); + if (noteJsonObject.has("paragraphs")) { + JSONArray paragraphJsonArray = noteJsonObject.getJSONArray("paragraphs"); + for (int i = 0; i< paragraphJsonArray.length(); ++i) { + paragraphResultList.add(new ParagraphResult(paragraphJsonArray.getJSONObject(i))); + } + } + + return new NoteResult(noteId, isRunning, paragraphResultList); + } + + /** + * Execute note with provided noteId, return until note execution is completed. + * + * @param noteId + * @return + * @throws Exception + */ + public NoteResult executeNote(String noteId) throws Exception { + return executeNote(noteId, new HashMap<>()); + } + + /** + * Execute note with provided noteId and parameters, return until note execution is completed. + * + * @param noteId + * @param parameters + * @return + * @throws Exception + */ + public NoteResult executeNote(String noteId, Map parameters) throws Exception { + JSONObject bodyObject = new JSONObject(); + bodyObject.put("params", parameters); + HttpResponse response = Unirest + .post("/notebook/job/{noteId}?blocking=true&isolated=true") + .routeParam("noteId", noteId) + .body(bodyObject.toString()) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + + return queryNoteResult(noteId); + } + + /** + * Submit note to execute with provided noteId, return at once the submission is completed. + * You need to query {@link NoteResult} by yourself afterwards until note execution is completed. + * + * @param noteId + * @return + * @throws Exception + */ + public NoteResult submitNote(String noteId) throws Exception { + return submitNote(noteId, new HashMap<>()); + } + + /** + * Submit note to execute with provided noteId and parameters, return at once the submission is completed. + * You need to query {@link NoteResult} by yourself afterwards until note execution is completed. + * + * @param noteId + * @param parameters + * @return + * @throws Exception + */ + public NoteResult submitNote(String noteId, Map parameters) throws Exception { + JSONObject bodyObject = new JSONObject(); + bodyObject.put("params", parameters); + // run note in non-blocking and isolated way. + HttpResponse response = Unirest + .post("/notebook/job/{noteId}?blocking=false&isolated=true") + .routeParam("noteId", noteId) + .body(bodyObject) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + LOGGER.info("Start to run note: " + noteId); + checkJsonNodeStatus(jsonNode); + return queryNoteResult(noteId); + } + + /** + * Block there until note execution is completed. + * + * @param noteId + * @return + * @throws Exception + */ + public NoteResult waitUntilNoteFinished(String noteId) throws Exception { + while (true) { + NoteResult noteResult = queryNoteResult(noteId); + if (!noteResult.isRunning()) { + return noteResult; + } + Thread.sleep(clientConfig.getQueryInterval()); + } + } + + /** + * Block there until note execution is completed, and throw exception if note execution is not completed + * in timeoutInMills. + * + * @param noteId + * @param timeoutInMills + * @return + * @throws Exception + */ + public NoteResult waitUntilNoteFinished(String noteId, long timeoutInMills) throws Exception { + long start = System.currentTimeMillis(); + while (true && (System.currentTimeMillis() - start) < timeoutInMills) { + NoteResult noteResult = queryNoteResult(noteId); + if (!noteResult.isRunning()) { + return noteResult; + } + Thread.sleep(clientConfig.getQueryInterval()); + } + throw new Exception("Note is not finished in " + timeoutInMills / 1000 + " seconds"); + } + + /** + * Add paragraph to note with provided title and text. + * + * @param noteId + * @param title + * @param text + * @return + * @throws Exception + */ + public String addParagraph(String noteId, String title, String text) throws Exception { + JSONObject bodyObject = new JSONObject(); + bodyObject.put("title", title); + bodyObject.put("text", text); + HttpResponse response = Unirest.post("/notebook/{noteId}/paragraph") + .routeParam("noteId", noteId) + .body(bodyObject.toString()) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + + return jsonNode.getObject().getString("body"); + } + + /** + * Update paragraph with specified title and text. + * + * @param noteId + * @param paragraphId + * @param title + * @param text + * @throws Exception + */ + public void updateParagraph(String noteId, String paragraphId, String title, String text) throws Exception { + JSONObject bodyObject = new JSONObject(); + bodyObject.put("title", title); + bodyObject.put("text", text); + HttpResponse response = Unirest.put("/notebook/{noteId}/paragraph/{paragraphId}") + .routeParam("noteId", noteId) + .routeParam("paragraphId", paragraphId) + .body(bodyObject.toString()) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + } + + /** + * Execute paragraph with parameters in specified session. If sessionId is null or empty string, then it depends on + * the interpreter binding mode of Note(e.g. isolated per note), otherwise it will run in the specified session. + * + * @param noteId + * @param paragraphId + * @param sessionId + * @param parameters + * @return + * @throws Exception + */ + public ParagraphResult executeParagraph(String noteId, + String paragraphId, + String sessionId, + Map parameters) throws Exception { + JSONObject bodyObject = new JSONObject(); + bodyObject.put("params", parameters); + HttpResponse response = Unirest + .post("/notebook/run/{noteId}/{paragraphId}") + .routeParam("noteId", noteId) + .routeParam("paragraphId", paragraphId) + .queryString("sessionId", sessionId) + .body(bodyObject.toString()) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + + jsonNode = Unirest.get( + clientConfig.getZeppelinRestUrl() + "/api/notebook/" + noteId + "/paragraph/" + paragraphId) + .asJson() + .getBody(); + checkJsonNodeStatus(jsonNode); + + JSONObject paragraphJson = jsonNode.getObject().getJSONObject("body"); + return new ParagraphResult(paragraphJson); + } + + /** + * Execute paragraph with parameters. + * + * @param noteId + * @param paragraphId + * @param parameters + * @return + * @throws Exception + */ + public ParagraphResult executeParagraph(String noteId, + String paragraphId, + Map parameters) throws Exception { + return executeParagraph(noteId, paragraphId, "", parameters); + } + + /** + * Execute paragraph in specified session. If sessionId is null or empty string, then it depends on + * the interpreter binding mode of Note (e.g. isolated per note), otherwise it will run in the specified session. + * + * @param noteId + * @param paragraphId + * @param sessionId + * @return + * @throws Exception + */ + public ParagraphResult executeParagraph(String noteId, + String paragraphId, + String sessionId) throws Exception { + return executeParagraph(noteId, paragraphId, sessionId, new HashMap<>()); + } + + /** + * Execute paragraph. + * + * @param noteId + * @param paragraphId + * @return + * @throws Exception + */ + public ParagraphResult executeParagraph(String noteId, String paragraphId) throws Exception { + return executeParagraph(noteId, paragraphId, "", new HashMap<>()); + } + + /** + * Submit paragraph to execute with provided parameters and sessionId. Return at once the submission is completed. + * You need to query {@link ParagraphResult} by yourself afterwards until paragraph execution is completed. + * + * @param noteId + * @param paragraphId + * @param sessionId + * @param parameters + * @return + * @throws Exception + */ + public ParagraphResult submitParagraph(String noteId, + String paragraphId, + String sessionId, + Map parameters) throws Exception { + JSONObject bodyObject = new JSONObject(); + bodyObject.put("params", parameters); + HttpResponse response = Unirest + .post("/notebook/job/{noteId}/{paragraphId}") + .routeParam("noteId", noteId) + .routeParam("paragraphId", paragraphId) + .queryString("sessionId", sessionId) + .body(bodyObject.toString()) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + return queryParagraphResult(noteId, paragraphId); + } + + /** + * Submit paragraph to execute with provided sessionId. Return at once the submission is completed. + * You need to query {@link ParagraphResult} by yourself afterwards until paragraph execution is completed. + * + * @param noteId + * @param paragraphId + * @param sessionId + * @return + * @throws Exception + */ + public ParagraphResult submitParagraph(String noteId, + String paragraphId, + String sessionId) throws Exception { + return submitParagraph(noteId, paragraphId, sessionId, new HashMap<>()); + } + + /** + * Submit paragraph to execute with provided parameters. Return at once the submission is completed. + * You need to query {@link ParagraphResult} by yourself afterwards until paragraph execution is completed. + * + * @param noteId + * @param paragraphId + * @param parameters + * @return + * @throws Exception + */ + public ParagraphResult submitParagraph(String noteId, + String paragraphId, + Map parameters) throws Exception { + return submitParagraph(noteId, paragraphId, "", parameters); + } + + /** + * Submit paragraph to execute. Return at once the submission is completed. + * You need to query {@link ParagraphResult} by yourself afterwards until paragraph execution is completed. + * + * @param noteId + * @param paragraphId + * @return + * @throws Exception + */ + public ParagraphResult submitParagraph(String noteId, String paragraphId) throws Exception { + return submitParagraph(noteId, paragraphId, "", new HashMap<>()); + } + + /** + * This used by {@link ZSession} for creating or reusing a paragraph for executing another piece of code. + * + * @param noteId + * @param maxParagraph + * @return + * @throws Exception + */ + public String nextSessionParagraph(String noteId, int maxParagraph) throws Exception { + HttpResponse response = Unirest + .post("/notebook/{noteId}/paragraph/next") + .routeParam("noteId", noteId) + .queryString("maxParagraph", maxParagraph) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + + return jsonNode.getObject().getString("message"); + } + + /** + * Cancel a running paragraph. + * + * @param noteId + * @param paragraphId + * @throws Exception + */ + public void cancelParagraph(String noteId, String paragraphId) throws Exception { + HttpResponse response = Unirest + .delete("/notebook/job/{noteId}/{paragraphId}") + .routeParam("noteId", noteId) + .routeParam("paragraphId", paragraphId) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + } + + /** + * Query {@link ParagraphResult} + * + * @param noteId + * @param paragraphId + * @return + * @throws Exception + */ + public ParagraphResult queryParagraphResult(String noteId, String paragraphId) throws Exception { + HttpResponse response = Unirest + .get("/notebook/{noteId}/paragraph/{paragraphId}") + .routeParam("noteId", noteId) + .routeParam("paragraphId", paragraphId) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + + JSONObject paragraphJson = jsonNode.getObject().getJSONObject("body"); + return new ParagraphResult(paragraphJson); + } + + /** + * Query {@link ParagraphResult} until it is finished. + * + * @param noteId + * @param paragraphId + * @return + * @throws Exception + */ + public ParagraphResult waitUtilParagraphFinish(String noteId, String paragraphId) throws Exception { + while (true) { + ParagraphResult paragraphResult = queryParagraphResult(noteId, paragraphId); + LOGGER.debug(paragraphResult.toString()); + if (paragraphResult.getStatus().isCompleted()) { + return paragraphResult; + } + Thread.sleep(clientConfig.getQueryInterval()); + } + } + + /** + * Query {@link ParagraphResult} until it is finished or timeout. + * + * @param noteId + * @param paragraphId + * @param timeoutInMills + * @return + * @throws Exception + */ + public ParagraphResult waitUtilParagraphFinish(String noteId, String paragraphId, long timeoutInMills) throws Exception { + long start = System.currentTimeMillis(); + while (true && (System.currentTimeMillis() - start) < timeoutInMills) { + ParagraphResult paragraphResult = queryParagraphResult(noteId, paragraphId); + if (paragraphResult.getStatus().isCompleted()) { + return paragraphResult; + } + Thread.sleep(clientConfig.getQueryInterval()); + } + throw new Exception("Paragraph is not finished in " + timeoutInMills / 1000 + " seconds"); + } + + /** + * Query {@link ParagraphResult} until it is running. + * + * @param noteId + * @param paragraphId + * @return + * @throws Exception + */ + public ParagraphResult waitUtilParagraphRunning(String noteId, String paragraphId) throws Exception { + while (true) { + ParagraphResult paragraphResult = queryParagraphResult(noteId, paragraphId); + if (paragraphResult.getStatus().isRunning()) { + return paragraphResult; + } + Thread.sleep(clientConfig.getQueryInterval()); + } + } + + public void connectWS() { + + } + + public static void main(String[] args) throws Exception { +// ClientConfig clientConfig = new ClientConfig("https://localhost:8443/gateway/sandbox/zeppelin#", 1000, true); +// ZeppelinClient zeppelinClient = new ZeppelinClient(clientConfig); +// String noteId = zeppelinClient.createNote("/tmp/note6"); +// System.out.println("created note: " + noteId); + + + SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustSelfSignedStrategy() { + public boolean isTrusted(X509Certificate[] chain, String authType) { + return true; + } + }).build(); + HttpClient customHttpClient = HttpClients.custom().setSSLContext(sslContext) + .setSSLHostnameVerifier(new NoopHostnameVerifier()).build(); + Unirest.config().httpClient(ApacheClient.builder(customHttpClient)); + + HttpResponse response = Unirest.get("https://knox.c-4397d20c71f8ed65.cn-hangzhou.databricks.aliyuncs.com:8443/gateway/cluster-topo/zeppelin/") + .basicAuth("jianhan-test", "jianhan-test") + .asString(); + System.out.println(response.getStatus() + ", " + response.getStatusText() + ", " + response.getBody()); + + HttpResponse response2 = Unirest.get("https://knox.c-4397d20c71f8ed65.cn-hangzhou.databricks.aliyuncs.com:8443/gateway/cluster-topo/zeppelin/api/notebook") + .basicAuth("jianhan-test", "jianhan-test") + .asString(); + System.out.println(response2.getStatus() + ", " + response2.getStatusText() + ", " + response2.getBody()); + + response = Unirest.post("https://knox.C-4397D20C71F8ED65.cn-hangzhou.databricks.aliyuncs.com:8443/gateway/knoxsso/api/v1/websso?originalUrl=") + .basicAuth("jianhan-test", "jianhan-test") + .asString(); + System.out.println(response.getStatus() + ", " + response.getStatusText() + ", " + response.getBody()); + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java new file mode 100644 index 00000000000..623666b5f43 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client; + +import com.google.gson.Gson; +import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.listener.ZeppelinhubWebsocket; +import org.apache.zeppelin.notebook.socket.Message; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; +import org.eclipse.jetty.websocket.client.WebSocketClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +@WebSocket(maxTextMessageSize = 64 * 1024) +public class ZeppelinWebSocketClient { + private static final Logger LOGGER = LoggerFactory.getLogger(ZeppelinhubWebsocket.class); + private static final Gson GSON = new Gson(); + + private CountDownLatch connectLatch = new CountDownLatch(1); + private CountDownLatch closeLatch = new CountDownLatch(1); + + private Session session; + private MessageHandler messageHandler; + + public ZeppelinWebSocketClient(MessageHandler messageHandler) { + this.messageHandler = messageHandler; + } + + public void connect(String url) throws Exception { + WebSocketClient client = new WebSocketClient(); + client.start(); + URI echoUri = new URI(url); + ClientUpgradeRequest request = new ClientUpgradeRequest(); + request.setHeader("Origin", "*"); + client.connect(this, echoUri, request); + connectLatch.await(); + LOGGER.info("WebSocket connect established"); + } + + public boolean awaitClose(int duration, TimeUnit unit) throws InterruptedException { + return this.closeLatch.await(duration, unit); + } + + @OnWebSocketClose + public void onClose(int statusCode, String reason) { + LOGGER.info("Connection closed, statusCode: {} - reason: {}", statusCode, reason); + this.session = null; + this.closeLatch.countDown(); + } + + @OnWebSocketConnect + public void onConnect(Session session) { + LOGGER.info("Got connect: {}", session.getRemote()); + this.session = session; + connectLatch.countDown(); + } + + @OnWebSocketMessage + public void onText(Session session, String message) throws IOException { + messageHandler.onMessage(message); + } + + @OnWebSocketError + public void onError(Throwable cause) { + LOGGER.info("WebSocket Error: " + cause.getMessage()); + cause.printStackTrace(System.out); + } + + public void send(Message message) throws IOException { + session.getRemote().sendString(GSON.toJson(message)); + } + + public CountDownLatch getConnectLatch() { + return connectLatch; + } + + public static void main(String[] args) throws Exception { +// ZeppelinWebSocket socket = new ZeppelinWebSocket(); +// socket.connect("ws://localhost:18086/ws"); +// Message msg = new Message(Message.OP.LIST_NOTES); +// socket.send(msg); +// +// Thread.sleep(10 * 1000); +// socket.awaitClose(10, TimeUnit.SECONDS); + + String dest = "ws://localhost:18086/ws"; + WebSocketClient client = new WebSocketClient(); + try { + ZeppelinWebSocketClient socket = new ZeppelinWebSocketClient(new SimpleMessageHandler()); + client.start(); + URI echoUri = new URI(dest); + ClientUpgradeRequest request = new ClientUpgradeRequest(); + client.connect(socket, echoUri, request); + socket.getConnectLatch().await(); + + Message msg = new Message(Message.OP.LIST_NOTES); + socket.send(msg); + + // wait for closed socket connection. + socket.awaitClose(5, TimeUnit.SECONDS); + } catch (Throwable t) { + t.printStackTrace(); + } finally { + try { + client.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + Thread.sleep(10000 * 100); + } + +} diff --git a/zeppelin-client/src/main/resources/log4j.properties b/zeppelin-client/src/main/resources/log4j.properties new file mode 100644 index 00000000000..0940c7f7b1c --- /dev/null +++ b/zeppelin-client/src/main/resources/log4j.properties @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger = INFO, stdout + +log4j.appender.stdout = org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout = org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n + +log4j.appender.dailyfile.DatePattern=.yyyy-MM-dd +log4j.appender.dailyfile = org.apache.log4j.DailyRollingFileAppender +log4j.appender.dailyfile.File = ${zeppelin.log.file} +log4j.appender.dailyfile.layout = org.apache.log4j.PatternLayout +log4j.appender.dailyfile.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n diff --git a/zeppelin-interpreter-integration/pom.xml b/zeppelin-interpreter-integration/pom.xml index 71ab23fe6a8..5958858b0fa 100644 --- a/zeppelin-interpreter-integration/pom.xml +++ b/zeppelin-interpreter-integration/pom.xml @@ -43,6 +43,18 @@ + + org.apache.zeppelin + zeppelin-client + ${project.version} + + + org.apache.httpcomponents + httpcore-nio + + + + org.apache.zeppelin zeppelin-zengine diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java new file mode 100644 index 00000000000..0922f6dc56d --- /dev/null +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java @@ -0,0 +1,516 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.integration; + +import com.google.common.collect.Lists; +import org.apache.commons.io.IOUtils; +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.ExecuteResult; +import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.Status; +import org.apache.zeppelin.client.ZSession; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.interpreter.ExecutionContextBuilder; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.integration.DownloadUtils; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Notebook; +import org.apache.zeppelin.rest.AbstractTestRestApi; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.apache.zeppelin.utils.TestUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class ZSessionIntegrationTest extends AbstractTestRestApi { + + private static Notebook notebook; + private static String sparkHome; + private static String flinkHome; + + private ClientConfig clientConfig = new ClientConfig("http://localhost:8080"); + + + @BeforeClass + public static void setUp() throws Exception { + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HELIUM_REGISTRY.getVarName(), + "helium"); + AbstractTestRestApi.startUp(ZSessionIntegrationTest.class.getSimpleName()); + notebook = TestUtils.getInstance(Notebook.class); + sparkHome = DownloadUtils.downloadSpark("2.4.4", "2.7"); + flinkHome = DownloadUtils.downloadFlink("1.10.1"); + } + + @AfterClass + public static void destroy() throws Exception { + AbstractTestRestApi.shutDown(); + } + + @Test + public void testZSession_Shell() throws Exception { + ZSession session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("sh") + .build(); + + try { + session.start(); + assertNull(session.getWeburl()); + assertNotNull(session.getNoteId()); + + Note note = notebook.getNote(session.getNoteId()); + assertEquals(2, note.getParagraphCount()); + assertTrue(note.getParagraph(0).getText(), note.getParagraph(0).getText().startsWith("%sh.conf")); + + ExecuteResult result = session.execute("pwd"); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + + result = session.execute("invalid_command"); + assertEquals(Status.ERROR, result.getStatus()); + assertEquals(2, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("command not found")); + assertEquals("TEXT", result.getResults().get(1).getType()); + assertTrue(result.getResults().get(1).getData(), result.getResults().get(1).getData().contains("ExitValue")); + + assertEquals(4, note.getParagraphCount()); + assertEquals("%sh\ninvalid_command", note.getParagraph(3).getText()); + + } finally { + session.stop(); + } + } + + @Test + public void testZSession_Shell_Submit() throws Exception { + ZSession session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("sh") + .build(); + + try { + session.start(); + assertNull(session.getWeburl()); + assertNotNull(session.getNoteId()); + + Note note = notebook.getNote(session.getNoteId()); + assertEquals(2, note.getParagraphCount()); + assertTrue(note.getParagraph(0).getText(), note.getParagraph(0).getText().startsWith("%sh.conf")); + + ExecuteResult result = session.submit("sleep 10\npwd"); + assertFalse("Status is: " + result.getStatus().toString(), result.getStatus().isCompleted()); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + + result = session.submit("invalid_command"); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(Status.ERROR, result.getStatus()); + assertEquals(2, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("command not found")); + assertEquals("TEXT", result.getResults().get(1).getType()); + assertTrue(result.getResults().get(1).getData(), result.getResults().get(1).getData().contains("ExitValue")); + + assertEquals(4, note.getParagraphCount()); + assertEquals("%sh\ninvalid_command", note.getParagraph(3).getText()); + + } finally { + session.stop(); + } + } + + @Test + public void testZSession_Spark() throws Exception { + Map intpProperties = new HashMap<>(); + intpProperties.put("SPARK_HOME", sparkHome); + intpProperties.put("spark.master", "local[*]"); + + ZSession session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("spark") + .setIntpProperties(intpProperties) + .build(); + + try { + session.start(); + assertNotNull(session.getWeburl()); + assertNotNull(session.getNoteId()); + + // scala + ExecuteResult result = session.execute("sc.version"); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("2.4.4")); + assertEquals(0, result.getJobUrls().size()); + + // pyspark + result = session.execute("pyspark", "df = spark.createDataFrame([(1,'a'),(2,'b')])\ndf.registerTempTable('df')\ndf.show()"); + assertEquals(Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertEquals( + "+---+---+\n" + + "| _1| _2|\n" + + "+---+---+\n" + + "| 1| a|\n" + + "| 2| b|\n" + + "+---+---+", result.getResults().get(0).getData().trim()); + assertEquals(3, result.getJobUrls().size()); + + // sparkr + result = session.execute("r", "df <- as.DataFrame(faithful)\nhead(df)"); + assertEquals(Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("eruptions waiting")); + assertEquals(2, result.getJobUrls().size()); + + // spark sql + result = session.execute("sql", "select * from df"); + assertEquals(Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TABLE", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("1\ta\n2\tb\n")); + assertEquals(3, result.getJobUrls().size()); + + // spark invalid sql + result = session.execute("sql", "select * from unknown_table"); + assertEquals(Status.ERROR, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("NoSuchTableException")); + assertEquals(0, result.getJobUrls().size()); + + } finally { + session.stop(); + } + } + + @Test + public void testZSession_Spark_Submit() throws Exception { + Map intpProperties = new HashMap<>(); + intpProperties.put("SPARK_HOME", sparkHome); + intpProperties.put("spark.master", "local[*]"); + + ZSession session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("spark") + .setIntpProperties(intpProperties) + .build(); + + try { + session.start(); + assertNotNull(session.getWeburl()); + assertNotNull(session.getNoteId()); + + // scala + ExecuteResult result = session.submit("sc.version"); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("2.4.4")); + assertEquals(0, result.getJobUrls().size()); + + // pyspark + result = session.submit("pyspark", "df = spark.createDataFrame([(1,'a'),(2,'b')])\ndf.registerTempTable('df')\ndf.show()"); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertEquals( + "+---+---+\n" + + "| _1| _2|\n" + + "+---+---+\n" + + "| 1| a|\n" + + "| 2| b|\n" + + "+---+---+", result.getResults().get(0).getData().trim()); + assertEquals(3, result.getJobUrls().size()); + + // sparkr + result = session.submit("r", "df <- as.DataFrame(faithful)\nhead(df)"); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("eruptions waiting")); + assertEquals(2, result.getJobUrls().size()); + + // spark sql + result = session.submit("sql", "select * from df"); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TABLE", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("1\ta\n2\tb\n")); + assertEquals(3, result.getJobUrls().size()); + + // spark invalid sql + result = session.submit("sql", "select * from unknown_table"); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(Status.ERROR, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("NoSuchTableException")); + assertEquals(0, result.getJobUrls().size()); + + // cancel + result = session.submit("sc.range(1,100).map(e=>{Thread.sleep(1000);e}).collect()"); + assertFalse("Status is: " + result.getStatus().toString(), result.getStatus().isCompleted()); + result = session.waitUntilRunning(result.getStatementId()); + session.cancel(result.getStatementId()); + assertEquals(result.toString(), Status.RUNNING, result.getStatus()); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.ABORT, result.getStatus()); + + } finally { + session.stop(); + } + } + + @Test + public void testZSession_Flink() throws Exception { + Map intpProperties = new HashMap<>(); + intpProperties.put("FLINK_HOME", flinkHome); + + ZSession session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("flink") + .setIntpProperties(intpProperties) + .build(); + + try { + session.start(); + assertNotNull(session.getWeburl()); + assertNotNull(session.getNoteId()); + + // scala + ExecuteResult result = session.execute("val data = benv.fromElements(1, 2, 3)\ndata.collect()"); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("1, 2, 3")); + + // sql + result = session.execute(getInitStreamScript(200)); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + Map localProperties = new HashMap<>(); + localProperties.put("type", "update"); + result = session.execute("ssql", localProperties, "select url, count(1) as pv from log group by url"); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + + } finally { + session.stop(); + } + } + + @Test + public void testZSession_Flink_Submit() throws Exception { + Map intpProperties = new HashMap<>(); + intpProperties.put("FLINK_HOME", flinkHome); + + ZSession session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("flink") + .setIntpProperties(intpProperties) + .build(); + + try { + session.start(new SimpleMessageHandler()); + assertNotNull(session.getWeburl()); + assertNotNull(session.getNoteId()); + + // scala + ExecuteResult result = session.submit("val data = benv.fromElements(1, 2, 3)\ndata.collect()"); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("1, 2, 3")); + + // sql + result = session.submit(getInitStreamScript(200)); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + Map localProperties = new HashMap<>(); + localProperties.put("type", "update"); + result = session.submit("ssql", localProperties, "select url, count(1) as pv from log group by url"); + assertFalse("Status is: " + result.getStatus().toString(), result.getStatus().isCompleted()); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + + // cancel + result = session.submit("ssql", localProperties, "select url, count(1) as pv from log group by url"); + assertFalse("Status is: " + result.getStatus().toString(), result.getStatus().isCompleted()); + result = session.waitUntilRunning(result.getStatementId()); + session.cancel(result.getStatementId()); + assertEquals(result.toString(), Status.RUNNING, result.getStatus()); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.ABORT, result.getStatus()); + } finally { + session.stop(); + } + } + + @Test + public void testZSession_Python() throws Exception { + Map intpProperties = new HashMap<>(); + + ZSession session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("python") + .setIntpProperties(intpProperties) + .build(); + + try { + session.start(new SimpleMessageHandler()); + assertNull(session.getWeburl()); + assertNotNull(session.getNoteId()); + + // python +// ExecuteResult result = session.execute("1+1"); +// assertEquals(result.toString(), Status.FINISHED, result.getStatus()); +// assertEquals(1, result.getResults().size()); +// assertEquals("TEXT", result.getResults().get(0).getType()); +// assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("2")); +// +// // python +// result = session.execute("1/0"); +// assertEquals(result.toString(), Status.ERROR, result.getStatus()); +// assertEquals(1, result.getResults().size()); +// assertEquals("TEXT", result.getResults().get(0).getType()); +// assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("ZeroDivisionError")); + + // for loop + ExecuteResult result = session.execute("import time\n" + + "for i in range(1,10):\n" + + "\tprint(i)\n" + + "\ttime.sleep(1)"); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TEXT", result.getResults().get(0).getType()); + } finally { + session.stop(); + } + } + + @Test + public void testZSession_Jdbc() throws Exception { + + Map intpProperties = new HashMap<>(); + intpProperties.put("default.driver", "com.mysql.jdbc.Driver"); + intpProperties.put("default.url", "jdbc:mysql://localhost:3306/"); + intpProperties.put("default.user", "root"); + intpProperties.put("default.password", "123456"); + + ZSession session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("jdbc") + .setIntpProperties(intpProperties) + .build(); + + try { + session.start(); + assertNull(session.getWeburl()); + assertNotNull(session.getNoteId()); + + // show databases + ExecuteResult result = session.execute("show databases"); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TABLE", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("Database")); + + // select statement + result = session.execute("SELECT 1 as c1, 2 as c2"); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TABLE", result.getResults().get(0).getType()); + assertEquals("c1\tc2\n1\t2\n", result.getResults().get(0).getData()); + + } finally { + session.stop(); + } + } + + @Test + public void testZSession_Jdbc_Submit() throws Exception { + + Map intpProperties = new HashMap<>(); + intpProperties.put("default.driver", "com.mysql.jdbc.Driver"); + intpProperties.put("default.url", "jdbc:mysql://localhost:3306/"); + intpProperties.put("default.user", "root"); + intpProperties.put("default.password", "123456"); + + ZSession session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("jdbc") + .setIntpProperties(intpProperties) + .build(); + + try { + session.start(); + assertNull(session.getWeburl()); + assertNotNull(session.getNoteId()); + + // show databases + ExecuteResult result = session.submit("show databases"); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TABLE", result.getResults().get(0).getType()); + assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("Database")); + + // select statement + result = session.submit("SELECT 1 as c1, 2 as c2"); + result = session.waitUntilFinished(result.getStatementId()); + assertEquals(result.toString(), Status.FINISHED, result.getStatus()); + assertEquals(1, result.getResults().size()); + assertEquals("TABLE", result.getResults().get(0).getType()); + assertEquals("c1\tc2\n1\t2\n", result.getResults().get(0).getData()); + + } finally { + session.stop(); + } + } + + public static String getInitStreamScript(int sleep_interval) throws IOException { + return IOUtils.toString(ZSessionIntegrationTest.class.getResource("/init_stream.scala")) + .replace("{{sleep_interval}}", sleep_interval + ""); + } +} diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientIntegrationTest.java new file mode 100644 index 00000000000..d508c9edb25 --- /dev/null +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientIntegrationTest.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.integration; + +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.NoteResult; +import org.apache.zeppelin.client.ParagraphResult; +import org.apache.zeppelin.client.Status; +import org.apache.zeppelin.client.ZeppelinClient; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Notebook; +import org.apache.zeppelin.rest.AbstractTestRestApi; +import org.apache.zeppelin.utils.TestUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class ZeppelinClientIntegrationTest extends AbstractTestRestApi { + private static Notebook notebook; + + private static ClientConfig clientConfig; + private static ZeppelinClient zeppelinClient; + + @BeforeClass + public static void setUp() throws Exception { + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HELIUM_REGISTRY.getVarName(), + "helium"); + AbstractTestRestApi.startUp(ZeppelinClientIntegrationTest.class.getSimpleName()); + notebook = TestUtils.getInstance(Notebook.class); + + clientConfig = new ClientConfig("http://localhost:8080"); + zeppelinClient = new ZeppelinClient(clientConfig); + } + + @AfterClass + public static void destroy() throws Exception { + AbstractTestRestApi.shutDown(); + } + + @Test + public void testZeppelinVersion() throws Exception { + String version = zeppelinClient.getVersion(); + LOG.info("Zeppelin version: " + version); + } + + @Test + public void testNoteOperation() throws Exception { + String noteId = zeppelinClient.createNote("/project_1/note1"); + assertNotNull(notebook.getNote(noteId)); + + // create duplicated note + try { + zeppelinClient.createNote("/project_1/note1"); + fail("Should fail to create duplicated note"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("existed")); + } + + // query NoteResult + NoteResult noteResult = zeppelinClient.queryNoteResult(noteId); + assertEquals(noteId, noteResult.getNoteId()); + assertEquals(false, noteResult.isRunning()); + // note is created with 0 paragraph. + assertEquals(0, noteResult.getParagraphResultList().size()); + + // query non-existed note + try { + zeppelinClient.queryNoteResult("unknown-noteId"); + fail("Should fail to query non-existed note"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("No such note")); + } + + zeppelinClient.deleteNote(noteId); + + // deleting the same note again will fail + try { + zeppelinClient.deleteNote(noteId); + fail("Should fail to delete non-existed note"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("No such note")); + } + } + + @Test + public void testExecuteParagraph() throws Exception { + // run paragraph succeed + String noteId = zeppelinClient.createNote("/test/note_1"); + String paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%sh echo 'hello world'"); + ParagraphResult paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId); + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(Status.FINISHED, paragraphResult.getStatus()); + assertEquals(1, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertEquals("hello world\n", paragraphResult.getResults().get(0).getData()); + + // run paragraph succeed with dynamic forms + paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%sh echo 'hello ${name=abc}'"); + paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId); + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(paragraphResult.toString(), Status.FINISHED, paragraphResult.getStatus()); + assertEquals(1, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertEquals("hello abc\n", paragraphResult.getResults().get(0).getData()); + + // run paragraph succeed with parameters + Map parameters = new HashMap<>(); + parameters.put("name", "zeppelin"); + paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId, parameters); + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(Status.FINISHED, paragraphResult.getStatus()); + assertEquals(1, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertEquals("hello zeppelin\n", paragraphResult.getResults().get(0).getData()); + + // run paragraph failed + paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%sh invalid_command"); + paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId); + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(Status.ERROR, paragraphResult.getStatus()); + assertEquals(2, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertTrue(paragraphResult.getResults().get(0).getData(), paragraphResult.getResults().get(0).getData().contains("command not found")); + assertEquals("TEXT", paragraphResult.getResults().get(1).getType()); + assertTrue(paragraphResult.getResults().get(1).getData(), paragraphResult.getResults().get(1).getData().contains("ExitValue")); + + // run non-existed interpreter + paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%non_existed hello"); + paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId); + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(Status.ERROR, paragraphResult.getStatus()); + assertEquals(1, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertTrue(paragraphResult.getResults().get(0).getData(), paragraphResult.getResults().get(0).getData().contains("Interpreter non_existed not found")); + + // run non-existed paragraph + try { + zeppelinClient.executeParagraph(noteId, "invalid_paragraph_id"); + fail("Should fail to run non-existed paragraph"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("No such paragraph")); + } + } + + @Test + public void testSubmitParagraph() throws Exception { + String noteId = zeppelinClient.createNote("/test/note_2"); + String paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%sh echo 'hello world'"); + zeppelinClient.submitParagraph(noteId, paragraphId); + ParagraphResult paragraphResult = zeppelinClient.waitUtilParagraphFinish(noteId, paragraphId, 10 * 1000); + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(Status.FINISHED, paragraphResult.getStatus()); + assertEquals(1, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertEquals("hello world\n", paragraphResult.getResults().get(0).getData()); + + // submit paragraph succeed with dynamic forms + paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%sh echo 'hello ${name=abc}'"); + zeppelinClient.submitParagraph(noteId, paragraphId); + paragraphResult = zeppelinClient.waitUtilParagraphFinish(noteId, paragraphId, 10 * 1000); + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(paragraphResult.toString(), Status.FINISHED, paragraphResult.getStatus()); + assertEquals(1, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertEquals("hello abc\n", paragraphResult.getResults().get(0).getData()); + + // run paragraph succeed with parameters + Map parameters = new HashMap<>(); + parameters.put("name", "zeppelin"); + zeppelinClient.submitParagraph(noteId, paragraphId, parameters); + paragraphResult = zeppelinClient.waitUtilParagraphFinish(noteId, paragraphId, 10 * 1000); + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(Status.FINISHED, paragraphResult.getStatus()); + assertEquals(1, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertEquals("hello zeppelin\n", paragraphResult.getResults().get(0).getData()); + + // run paragraph failed + paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%sh invalid_command"); + zeppelinClient.submitParagraph(noteId, paragraphId); + paragraphResult = zeppelinClient.waitUtilParagraphFinish(noteId, paragraphId, 10 * 1000); + + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(Status.ERROR, paragraphResult.getStatus()); + assertEquals(2, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertTrue(paragraphResult.getResults().get(0).getData(), paragraphResult.getResults().get(0).getData().contains("command not found")); + assertEquals("TEXT", paragraphResult.getResults().get(1).getType()); + assertTrue(paragraphResult.getResults().get(1).getData(), paragraphResult.getResults().get(1).getData().contains("ExitValue")); + + // run non-existed interpreter + paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%non_existed hello"); + zeppelinClient.submitParagraph(noteId, paragraphId); + paragraphResult = zeppelinClient.waitUtilParagraphFinish(noteId, paragraphId, 10 * 1000); + assertEquals(paragraphId, paragraphResult.getParagraphId()); + assertEquals(Status.ERROR, paragraphResult.getStatus()); + assertEquals(1, paragraphResult.getResults().size()); + assertEquals("TEXT", paragraphResult.getResults().get(0).getType()); + assertTrue(paragraphResult.getResults().get(0).getData(), paragraphResult.getResults().get(0).getData().contains("Interpreter non_existed not found")); + + // run non-existed paragraph + try { + zeppelinClient.submitParagraph(noteId, "invalid_paragraph_id"); + fail("Should fail to run non-existed paragraph"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("No such paragraph")); + } + + // cancel paragraph + paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%sh sleep 10"); + zeppelinClient.submitParagraph(noteId, paragraphId); + zeppelinClient.waitUtilParagraphRunning(noteId, paragraphId); + zeppelinClient.cancelParagraph(noteId, paragraphId); + paragraphResult = zeppelinClient.waitUtilParagraphFinish(noteId, paragraphId); + assertEquals(Status.ABORT, paragraphResult.getStatus()); + } + + @Test + public void testExecuteNote() throws Exception { + try { + zeppelinClient.executeNote("unknown_id"); + fail("Should fail to submit non-existed note"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("No such note")); + } + String noteId = zeppelinClient.createNote("/test/note_3"); + String p0Id = zeppelinClient.addParagraph(noteId, "run sh", "%sh echo 'hello world'"); + NoteResult noteResult = zeppelinClient.executeNote(noteId, new HashMap<>()); + assertEquals(noteId, noteResult.getNoteId()); + assertEquals(false, noteResult.isRunning()); + assertEquals(1, noteResult.getParagraphResultList().size()); + ParagraphResult p0 = noteResult.getParagraphResultList().get(0); + assertEquals(Status.FINISHED, p0.getStatus()); + assertEquals(1, p0.getResults().size()); + assertEquals("TEXT", p0.getResults().get(0).getType()); + assertEquals("hello world\n", p0.getResults().get(0).getData()); + + // update paragraph with dynamic forms + zeppelinClient.updateParagraph(noteId, p0Id, "run sh", "%sh echo 'hello ${name=abc}'"); + noteResult = zeppelinClient.executeNote(noteId); + assertEquals(noteId, noteResult.getNoteId()); + assertEquals(false, noteResult.isRunning()); + assertEquals(1, noteResult.getParagraphResultList().size()); + p0 = noteResult.getParagraphResultList().get(0); + assertEquals(Status.FINISHED, p0.getStatus()); + assertEquals(1, p0.getResults().size()); + assertEquals("TEXT", p0.getResults().get(0).getType()); + assertEquals("hello abc\n", p0.getResults().get(0).getData()); + + // execute paragraph with parameters + Map parameters = new HashMap<>(); + parameters.put("name", "zeppelin"); + noteResult = zeppelinClient.executeNote(noteId, parameters); + assertEquals(noteId, noteResult.getNoteId()); + assertEquals(false, noteResult.isRunning()); + assertEquals(1, noteResult.getParagraphResultList().size()); + p0 = noteResult.getParagraphResultList().get(0); + assertEquals(Status.FINISHED, p0.getStatus()); + assertEquals(1, p0.getResults().size()); + assertEquals("TEXT", p0.getResults().get(0).getType()); + assertEquals("hello zeppelin\n", p0.getResults().get(0).getData()); + + zeppelinClient.addParagraph(noteId, "run sh", "%sh invalid_command"); + zeppelinClient.addParagraph(noteId, "run sh", "%sh pwd"); + noteResult = zeppelinClient.executeNote(noteId, parameters); + assertEquals(noteId, noteResult.getNoteId()); + assertEquals(false, noteResult.isRunning()); + assertEquals(3, noteResult.getParagraphResultList().size()); + p0 = noteResult.getParagraphResultList().get(0); + assertEquals(Status.FINISHED, p0.getStatus()); + assertEquals(1, p0.getResults().size()); + assertEquals("TEXT", p0.getResults().get(0).getType()); + assertEquals("hello zeppelin\n", p0.getResults().get(0).getData()); + + ParagraphResult p1 = noteResult.getParagraphResultList().get(1); + assertEquals(Status.ERROR, p1.getStatus()); + assertEquals("TEXT", p1.getResults().get(0).getType()); + assertTrue(p1.getResults().get(0).getData(), p1.getResults().get(0).getData().contains("command not found")); + assertEquals("TEXT", p1.getResults().get(1).getType()); + assertTrue(p1.getResults().get(1).getData(), p1.getResults().get(1).getData().contains("ExitValue")); + + // p2 will be skipped because p1 fails. + ParagraphResult p2 = noteResult.getParagraphResultList().get(2); + assertEquals(Status.READY, p2.getStatus()); + assertEquals(0, p2.getResults().size()); + } + + @Test + public void testSubmitNote() throws Exception { + try { + zeppelinClient.submitNote("unknown_id"); + fail("Should fail to submit non-existed note"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("No such note")); + } + String noteId = zeppelinClient.createNote("/test/note_4"); + String p0Id = zeppelinClient.addParagraph(noteId, "run sh", "%sh echo 'hello world'"); + zeppelinClient.submitNote(noteId); + NoteResult noteResult = zeppelinClient.waitUntilNoteFinished(noteId); + assertEquals(noteId, noteResult.getNoteId()); + assertEquals(false, noteResult.isRunning()); + assertEquals(1, noteResult.getParagraphResultList().size()); + ParagraphResult p0 = noteResult.getParagraphResultList().get(0); + assertEquals(Status.FINISHED, p0.getStatus()); + assertEquals(1, p0.getResults().size()); + assertEquals("TEXT", p0.getResults().get(0).getType()); + assertEquals("hello world\n", p0.getResults().get(0).getData()); + + // update paragraph with dynamic forms + zeppelinClient.updateParagraph(noteId, p0Id, "run sh", "%sh sleep 5\necho 'hello ${name=abc}'"); + noteResult = zeppelinClient.submitNote(noteId); + assertEquals(true, noteResult.isRunning()); + noteResult = zeppelinClient.waitUntilNoteFinished(noteId); + assertEquals(noteId, noteResult.getNoteId()); + assertEquals(false, noteResult.isRunning()); + assertEquals(1, noteResult.getParagraphResultList().size()); + p0 = noteResult.getParagraphResultList().get(0); + assertEquals(Status.FINISHED, p0.getStatus()); + assertEquals(1, p0.getResults().size()); + assertEquals("TEXT", p0.getResults().get(0).getType()); + assertEquals("hello abc\n", p0.getResults().get(0).getData()); + + // execute paragraph with parameters + Map parameters = new HashMap<>(); + parameters.put("name", "zeppelin"); + noteResult = zeppelinClient.executeNote(noteId, parameters); + assertEquals(noteId, noteResult.getNoteId()); + assertEquals(false, noteResult.isRunning()); + assertEquals(1, noteResult.getParagraphResultList().size()); + p0 = noteResult.getParagraphResultList().get(0); + assertEquals(Status.FINISHED, p0.getStatus()); + assertEquals(1, p0.getResults().size()); + assertEquals("TEXT", p0.getResults().get(0).getType()); + assertEquals("hello zeppelin\n", p0.getResults().get(0).getData()); + + zeppelinClient.addParagraph(noteId, "run sh", "%sh invalid_command"); + zeppelinClient.addParagraph(noteId, "run sh", "%sh pwd"); + zeppelinClient.submitNote(noteId, parameters); + noteResult = zeppelinClient.waitUntilNoteFinished(noteId); + assertEquals(noteId, noteResult.getNoteId()); + assertEquals(false, noteResult.isRunning()); + assertEquals(3, noteResult.getParagraphResultList().size()); + p0 = noteResult.getParagraphResultList().get(0); + assertEquals(Status.FINISHED, p0.getStatus()); + assertEquals(1, p0.getResults().size()); + assertEquals("TEXT", p0.getResults().get(0).getType()); + assertEquals("hello zeppelin\n", p0.getResults().get(0).getData()); + + ParagraphResult p1 = noteResult.getParagraphResultList().get(1); + assertEquals(Status.ERROR, p1.getStatus()); + assertEquals("TEXT", p1.getResults().get(0).getType()); + assertTrue(p1.getResults().get(0).getData(), p1.getResults().get(0).getData().contains("command not found")); + assertEquals("TEXT", p1.getResults().get(1).getType()); + assertTrue(p1.getResults().get(1).getData(), p1.getResults().get(1).getData().contains("ExitValue")); + + // p2 will be skipped because p1 fails. + ParagraphResult p2 = noteResult.getParagraphResultList().get(2); + assertEquals(Status.READY, p2.getStatus()); + assertEquals(0, p2.getResults().size()); + } +} diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java new file mode 100644 index 00000000000..36c652e7dfb --- /dev/null +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.integration; + + +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.NoteResult; +import org.apache.zeppelin.client.ParagraphResult; +import org.apache.zeppelin.client.Status; +import org.apache.zeppelin.client.ZeppelinClient; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.Notebook; +import org.apache.zeppelin.rest.AbstractTestRestApi; +import org.apache.zeppelin.utils.TestUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class ZeppelinClientWithAuthIntegrationTest extends AbstractTestRestApi { + private static Notebook notebook; + + private static ClientConfig clientConfig; + private static ZeppelinClient zeppelinClient; + + @BeforeClass + public static void setUp() throws Exception { + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HELIUM_REGISTRY.getVarName(), + "helium"); + AbstractTestRestApi.startUpWithAuthenticationEnable(ZeppelinClientWithAuthIntegrationTest.class.getSimpleName()); + notebook = TestUtils.getInstance(Notebook.class); + + clientConfig = new ClientConfig("http://localhost:8080"); + zeppelinClient = new ZeppelinClient(clientConfig); + } + + @AfterClass + public static void destroy() throws Exception { + AbstractTestRestApi.shutDown(); + } + + @Test + public void testZeppelinVersion() throws Exception { + String version = zeppelinClient.getVersion(); + LOG.info("Zeppelin version: " + version); + } + + @Test + public void testCreateNoteWithoutLogin() throws Exception { + try { + zeppelinClient.createNote("/note_1"); + fail("Should fail due to not login"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("statusText: Found")); + } + } + + @Test + public void testCreateNoteAfterLogin() throws Exception { + zeppelinClient.login("admin", "password1"); + zeppelinClient.createNote("/note_2"); + } + + @Test + public void testLoginFailed() throws Exception { + // wrong password + try { + zeppelinClient.login("admin", "invalid_password"); + fail("Should fail to login"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("Forbidden")); + } + + // wrong username + try { + zeppelinClient.login("invalid_user", "password1"); + fail("Should fail to login"); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("Forbidden")); + } + } +} + diff --git a/zeppelin-interpreter-integration/src/test/resources/log4j.properties b/zeppelin-interpreter-integration/src/test/resources/log4j.properties index 773d0af0663..e194ac771ce 100644 --- a/zeppelin-interpreter-integration/src/test/resources/log4j.properties +++ b/zeppelin-interpreter-integration/src/test/resources/log4j.properties @@ -19,7 +19,7 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n +log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n #log4j.appender.stdout.layout.ConversionPattern= #%5p [%t] (%F:%L) - %m%n #%-4r [%t] %-5p %c %x - %m%n @@ -42,3 +42,7 @@ log4j.logger.DataNucleus.Datastore=ERROR # Log all JDBC parameters log4j.logger.org.hibernate.type=ALL log4j.logger.org.apache.hadoop=WARN + +log4j.logger.org.apache.zeppelin.client=INFO +log4j.logger.org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer=DEBUG + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContext.java index 9474560dfc7..a1485e3f291 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContext.java @@ -25,15 +25,17 @@ public class ExecutionContext { private final String user; private final String noteId; + private final String interpreterGroupId; private final String defaultInterpreterGroup; private final boolean inIsolatedMode; // When is the execution triggered, e.g. when the cron job is triggered or when the rest api is triggered. private final String startTime; - public ExecutionContext(String user, String noteId, String defaultInterpreterGroup, + public ExecutionContext(String user, String noteId, String interpreterGroupId, String defaultInterpreterGroup, boolean inIsolatedMode, String startTime) { this.user = user; this.noteId = noteId; + this.interpreterGroupId = interpreterGroupId; this.defaultInterpreterGroup = defaultInterpreterGroup; this.inIsolatedMode = inIsolatedMode; this.startTime = startTime; @@ -47,6 +49,10 @@ public String getNoteId() { return noteId; } + public String getInterpreterGroupId() { + return interpreterGroupId; + } + public String getDefaultInterpreterGroup() { return defaultInterpreterGroup; } @@ -64,6 +70,7 @@ public String toString() { return "ExecutionContext{" + "user='" + user + '\'' + ", noteId='" + noteId + '\'' + + ", interpreterGroupId='" + interpreterGroupId + '\'' + ", defaultInterpreterGroup='" + defaultInterpreterGroup + '\'' + ", inIsolatedMode=" + inIsolatedMode + ", startTime=" + startTime + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContextBuilder.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContextBuilder.java index bf8be9c2487..86956d2b7a0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContextBuilder.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContextBuilder.java @@ -23,6 +23,7 @@ public class ExecutionContextBuilder { private String user; private String noteId; + private String interpreterGroupId; private String defaultInterpreterGroup = ""; private boolean inIsolatedMode = false; private String startTime = ""; @@ -52,7 +53,13 @@ public ExecutionContextBuilder setStartTime(String startTime) { return this; } + public ExecutionContextBuilder setInterpreterGroupId(String interpreterGroupId) { + this.interpreterGroupId = interpreterGroupId; + return this; + } + public ExecutionContext createExecutionContext() { - return new ExecutionContext(user, noteId, defaultInterpreterGroup, inIsolatedMode, startTime); + return new ExecutionContext(user, noteId, interpreterGroupId, defaultInterpreterGroup, inIsolatedMode, startTime); } + } \ No newline at end of file diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 36ec1bd65f9..2f05ad8c4ae 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -25,6 +25,7 @@ import org.apache.zeppelin.dep.Repository; import org.apache.zeppelin.interpreter.ExecutionContextBuilder; import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterPropertyType; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.InterpreterSettingManager; @@ -56,6 +57,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import java.io.IOException; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 52f4acf66ea..48d8f886699 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -37,6 +37,7 @@ import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import org.apache.commons.lang3.StringUtils; @@ -126,15 +127,19 @@ public NotebookRestApi( @Path("{noteId}/permissions") @ZeppelinApi public Response getNotePermissions(@PathParam("noteId") String noteId) throws IOException { - checkIfUserIsAnon(getBlockNotAuthenticatedUserErrorMsg()); - checkIfUserCanRead(noteId, - "Insufficient privileges you cannot get the list of permissions for this note"); - HashMap> permissionsMap = new HashMap<>(); - permissionsMap.put("owners", authorizationService.getOwners(noteId)); - permissionsMap.put("readers", authorizationService.getReaders(noteId)); - permissionsMap.put("writers", authorizationService.getWriters(noteId)); - permissionsMap.put("runners", authorizationService.getRunners(noteId)); - return new JsonResponse<>(Status.OK, "", permissionsMap).build(); + try { + checkIfUserIsAnon(getBlockNotAuthenticatedUserErrorMsg()); + checkIfUserCanRead(noteId, + "Insufficient privileges you cannot get the list of permissions for this note"); + HashMap> permissionsMap = new HashMap<>(); + permissionsMap.put("owners", authorizationService.getOwners(noteId)); + permissionsMap.put("readers", authorizationService.getReaders(noteId)); + permissionsMap.put("writers", authorizationService.getWriters(noteId)); + permissionsMap.put("runners", authorizationService.getRunners(noteId)); + return new JsonResponse<>(Status.OK, "", permissionsMap).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } private String ownerPermissionError(Set current, Set allowed) { @@ -241,70 +246,74 @@ private void checkIfParagraphIsNotNull(Paragraph paragraph) { @ZeppelinApi public Response putNotePermissions(@PathParam("noteId") String noteId, String req) throws IOException { - String principal = authenticationService.getPrincipal(); - Set roles = authenticationService.getAssociatedRoles(); - HashSet userAndRoles = new HashSet<>(); - userAndRoles.add(principal); - userAndRoles.addAll(roles); - - checkIfUserIsAnon(getBlockNotAuthenticatedUserErrorMsg()); - checkIfUserIsOwner(noteId, - ownerPermissionError(userAndRoles, authorizationService.getOwners(noteId))); - - HashMap> permMap = - GSON.fromJson(req, new TypeToken>>() { - }.getType()); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - - HashSet readers = permMap.get("readers"); - HashSet runners = permMap.get("runners"); - HashSet owners = permMap.get("owners"); - HashSet writers = permMap.get("writers"); - - LOGGER.info("Set permissions to note: {} with current user:{}, owners:{}, readers:{}, runners:{}, writers:{}", - noteId, principal, owners, readers, runners, writers); - - // Set readers, if runners, writers and owners is empty -> set to user requesting the change - if (readers != null && !readers.isEmpty()) { - if (runners.isEmpty()) { - runners = Sets.newHashSet(authenticationService.getPrincipal()); - } - if (writers.isEmpty()) { - writers = Sets.newHashSet(authenticationService.getPrincipal()); - } - if (owners.isEmpty()) { - owners = Sets.newHashSet(authenticationService.getPrincipal()); - } - } - // Set runners, if writers and owners is empty -> set to user requesting the change - if (runners != null && !runners.isEmpty()) { - if (writers.isEmpty()) { - writers = Sets.newHashSet(authenticationService.getPrincipal()); + try { + String principal = authenticationService.getPrincipal(); + Set roles = authenticationService.getAssociatedRoles(); + HashSet userAndRoles = new HashSet<>(); + userAndRoles.add(principal); + userAndRoles.addAll(roles); + + checkIfUserIsAnon(getBlockNotAuthenticatedUserErrorMsg()); + checkIfUserIsOwner(noteId, + ownerPermissionError(userAndRoles, authorizationService.getOwners(noteId))); + + HashMap> permMap = + GSON.fromJson(req, new TypeToken>>() { + }.getType()); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + + HashSet readers = permMap.get("readers"); + HashSet runners = permMap.get("runners"); + HashSet owners = permMap.get("owners"); + HashSet writers = permMap.get("writers"); + + LOGGER.info("Set permissions to note: {} with current user:{}, owners:{}, readers:{}, runners:{}, writers:{}", + noteId, principal, owners, readers, runners, writers); + + // Set readers, if runners, writers and owners is empty -> set to user requesting the change + if (readers != null && !readers.isEmpty()) { + if (runners.isEmpty()) { + runners = Sets.newHashSet(authenticationService.getPrincipal()); + } + if (writers.isEmpty()) { + writers = Sets.newHashSet(authenticationService.getPrincipal()); + } + if (owners.isEmpty()) { + owners = Sets.newHashSet(authenticationService.getPrincipal()); + } } - if (owners.isEmpty()) { - owners = Sets.newHashSet(authenticationService.getPrincipal()); + // Set runners, if writers and owners is empty -> set to user requesting the change + if (runners != null && !runners.isEmpty()) { + if (writers.isEmpty()) { + writers = Sets.newHashSet(authenticationService.getPrincipal()); + } + if (owners.isEmpty()) { + owners = Sets.newHashSet(authenticationService.getPrincipal()); + } } - } - // Set writers, if owners is empty -> set to user requesting the change - if (writers != null && !writers.isEmpty()) { - if (owners.isEmpty()) { - owners = Sets.newHashSet(authenticationService.getPrincipal()); + // Set writers, if owners is empty -> set to user requesting the change + if (writers != null && !writers.isEmpty()) { + if (owners.isEmpty()) { + owners = Sets.newHashSet(authenticationService.getPrincipal()); + } } - } - authorizationService.setReaders(noteId, readers); - authorizationService.setRunners(noteId, runners); - authorizationService.setWriters(noteId, writers); - authorizationService.setOwners(noteId, owners); - LOGGER.debug("After set permissions {} {} {} {}", authorizationService.getOwners(noteId), - authorizationService.getReaders(noteId), authorizationService.getRunners(noteId), - authorizationService.getWriters(noteId)); - AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); - authorizationService.saveNoteAuth(noteId, subject); - notebookServer.broadcastNote(note); - notebookServer.broadcastNoteList(subject, userAndRoles); - return new JsonResponse<>(Status.OK).build(); + authorizationService.setReaders(noteId, readers); + authorizationService.setRunners(noteId, runners); + authorizationService.setWriters(noteId, writers); + authorizationService.setOwners(noteId, owners); + LOGGER.debug("After set permissions {} {} {} {}", authorizationService.getOwners(noteId), + authorizationService.getReaders(noteId), authorizationService.getRunners(noteId), + authorizationService.getWriters(noteId)); + AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); + authorizationService.saveNoteAuth(noteId, subject); + notebookServer.broadcastNote(note); + notebookServer.broadcastNoteList(subject, userAndRoles); + return new JsonResponse<>(Status.OK).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -316,9 +325,13 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re @GET @ZeppelinApi public Response getNoteList() throws IOException { - List notesInfo = notebookService.listNotesInfo(false, getServiceContext(), - new RestServiceCallback>()); - return new JsonResponse<>(Status.OK, "", notesInfo).build(); + try { + List notesInfo = notebookService.listNotesInfo(false, getServiceContext(), + new RestServiceCallback>()); + return new JsonResponse<>(Status.OK, "", notesInfo).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -332,9 +345,13 @@ public Response getNoteList() throws IOException { @Path("{noteId}") @ZeppelinApi public Response getNote(@PathParam("noteId") String noteId) throws IOException { - Note note = - notebookService.getNote(noteId, getServiceContext(), new RestServiceCallback()); - return new JsonResponse<>(Status.OK, "", note).build(); + try { + Note note = + notebookService.getNote(noteId, getServiceContext(), new RestServiceCallback()); + return new JsonResponse<>(Status.OK, "", note).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -348,9 +365,13 @@ public Response getNote(@PathParam("noteId") String noteId) throws IOException { @Path("export/{noteId}") @ZeppelinApi public Response exportNote(@PathParam("noteId") String noteId) throws IOException { - checkIfUserCanRead(noteId, "Insufficient privileges you cannot export this note"); - String exportJson = notebook.exportNote(noteId); - return new JsonResponse<>(Status.OK, "", exportJson).build(); + try { + checkIfUserCanRead(noteId, "Insufficient privileges you cannot export this note"); + String exportJson = notebook.exportNote(noteId); + return new JsonResponse<>(Status.OK, "", exportJson).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -369,8 +390,8 @@ public Response importNote(@QueryParam("notePath") String notePath, String noteJ Note note = notebookService.importNote(notePath, noteJson, getServiceContext(), new RestServiceCallback()); return new JsonResponse<>(Status.OK, "", note.getId()).build(); - } catch (IOException e) { - return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage()).build(); + } catch (Throwable e) { + return createResponseFromException(e); } } @@ -384,22 +405,27 @@ public Response importNote(@QueryParam("notePath") String notePath, String noteJ @POST @ZeppelinApi public Response createNote(String message) throws IOException { - String user = authenticationService.getPrincipal(); - LOGGER.info("Create new note by JSON {}", message); - NewNoteRequest request = NewNoteRequest.fromJson(message); - Note note = notebookService.createNote( - request.getName(), - zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_DEFAULT), - getServiceContext(), - new RestServiceCallback<>()); - AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); - if (request.getParagraphs() != null) { - for (NewParagraphRequest paragraphRequest : request.getParagraphs()) { - Paragraph p = note.addNewParagraph(subject); - initParagraph(p, paragraphRequest, user); + try { + String user = authenticationService.getPrincipal(); + LOGGER.info("Creating new note by JSON {}", message); + NewNoteRequest request = NewNoteRequest.fromJson(message); + Note note = notebookService.createNote( + request.getName(), + zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_DEFAULT), + false, + getServiceContext(), + new RestServiceCallback<>()); + AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); + if (request.getParagraphs() != null) { + for (NewParagraphRequest paragraphRequest : request.getParagraphs()) { + Paragraph p = note.addNewParagraph(subject); + initParagraph(p, paragraphRequest, user); + } } + return new JsonResponse(Status.OK, "", note.getId()).build(); + } catch (Throwable e) { + return createResponseFromException(e); } - return new JsonResponse<>(Status.OK, "", note.getId()).build(); } /** @@ -414,16 +440,20 @@ public Response createNote(String message) throws IOException { @ZeppelinApi public Response deleteNote(@PathParam("noteId") String noteId) throws IOException { LOGGER.info("Delete note {} ", noteId); - notebookService.removeNote(noteId, - getServiceContext(), - new RestServiceCallback() { - @Override - public void onSuccess(String message, ServiceContext context) { - notebookServer.broadcastNoteList(context.getAutheInfo(), context.getUserAndRoles()); - } - }); - - return new JsonResponse<>(Status.OK, "").build(); + try { + notebookService.removeNote(noteId, + getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(String message, ServiceContext context) { + notebookServer.broadcastNoteList(context.getAutheInfo(), context.getUserAndRoles()); + } + }); + + return new JsonResponse<>(Status.OK, "").build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -440,23 +470,27 @@ public void onSuccess(String message, ServiceContext context) { @ZeppelinApi public Response cloneNote(@PathParam("noteId") String noteId, String message) throws IOException, IllegalArgumentException { - LOGGER.info("Clone note by JSON {}", message); - checkIfUserCanWrite(noteId, "Insufficient privileges you cannot clone this note"); - NewNoteRequest request = NewNoteRequest.fromJson(message); - String newNoteName = null; - if (request != null) { - newNoteName = request.getName(); + try { + LOGGER.info("Clone note by JSON {}", message); + checkIfUserCanWrite(noteId, "Insufficient privileges you cannot clone this note"); + NewNoteRequest request = NewNoteRequest.fromJson(message); + String newNoteName = null; + if (request != null) { + newNoteName = request.getName(); + } + AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); + Note newNote = notebookService.cloneNote(noteId, newNoteName, getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(Note newNote, ServiceContext context) throws IOException { + notebookServer.broadcastNote(newNote); + notebookServer.broadcastNoteList(subject, context.getUserAndRoles()); + } + }); + return new JsonResponse<>(Status.OK, "", newNote.getId()).build(); + } catch (Throwable e) { + return createResponseFromException(e); } - AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); - Note newNote = notebookService.cloneNote(noteId, newNoteName, getServiceContext(), - new RestServiceCallback(){ - @Override - public void onSuccess(Note newNote, ServiceContext context) throws IOException { - notebookServer.broadcastNote(newNote); - notebookServer.broadcastNoteList(subject, context.getUserAndRoles()); - } - }); - return new JsonResponse<>(Status.OK, "", newNote.getId()).build(); } /** @@ -471,22 +505,26 @@ public void onSuccess(Note newNote, ServiceContext context) throws IOException { @ZeppelinApi public Response renameNote(@PathParam("noteId") String noteId, String message) throws IOException { - LOGGER.info("Rename note by JSON {}", message); - RenameNoteRequest request = GSON.fromJson(message, RenameNoteRequest.class); - String newName = request.getName(); - if (newName.isEmpty()) { - LOGGER.warn("Trying to rename notebook {} with empty name parameter", noteId); - throw new BadRequestException("name can not be empty"); + try { + LOGGER.info("Rename note by JSON {}", message); + RenameNoteRequest request = GSON.fromJson(message, RenameNoteRequest.class); + String newName = request.getName(); + if (newName.isEmpty()) { + LOGGER.warn("Trying to rename notebook {} with empty name parameter", noteId); + throw new BadRequestException("name can not be empty"); + } + notebookService.renameNote(noteId, request.getName(), false, getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(Note note, ServiceContext context) throws IOException { + notebookServer.broadcastNote(note); + notebookServer.broadcastNoteList(context.getAutheInfo(), context.getUserAndRoles()); + } + }); + return new JsonResponse(Status.OK, "").build(); + } catch (Throwable e) { + return createResponseFromException(e); } - notebookService.renameNote(noteId, request.getName(), false, getServiceContext(), - new RestServiceCallback(){ - @Override - public void onSuccess(Note note, ServiceContext context) throws IOException { - notebookServer.broadcastNote(note); - notebookServer.broadcastNoteList(context.getAutheInfo(), context.getUserAndRoles()); - } - }); - return new JsonResponse(Status.OK, "").build(); } /** @@ -501,26 +539,30 @@ public void onSuccess(Note note, ServiceContext context) throws IOException { @ZeppelinApi public Response insertParagraph(@PathParam("noteId") String noteId, String message) throws IOException { - String user = authenticationService.getPrincipal(); - LOGGER.info("Insert paragraph {} {}", noteId, message); - - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanWrite(noteId, "Insufficient privileges you cannot add paragraph to this note"); - - NewParagraphRequest request = NewParagraphRequest.fromJson(message); - AuthenticationInfo subject = new AuthenticationInfo(user); - Paragraph p; - Double indexDouble = request.getIndex(); - if (indexDouble == null) { - p = note.addNewParagraph(subject); - } else { - p = note.insertNewParagraph(indexDouble.intValue(), subject); + try { + String user = authenticationService.getPrincipal(); + LOGGER.info("Insert paragraph {} {}", noteId, message); + + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanWrite(noteId, "Insufficient privileges you cannot add paragraph to this note"); + + NewParagraphRequest request = NewParagraphRequest.fromJson(message); + AuthenticationInfo subject = new AuthenticationInfo(user); + Paragraph p; + Double indexDouble = request.getIndex(); + if (indexDouble == null) { + p = note.addNewParagraph(subject); + } else { + p = note.insertNewParagraph(indexDouble.intValue(), subject); + } + initParagraph(p, request, user); + notebook.saveNote(note, subject); + notebookServer.broadcastNote(note); + return new JsonResponse<>(Status.OK, "", p.getId()).build(); + } catch (Throwable e) { + return createResponseFromException(e); } - initParagraph(p, request, user); - notebook.saveNote(note, subject); - notebookServer.broadcastNote(note); - return new JsonResponse<>(Status.OK, "", p.getId()).build(); } /** @@ -535,15 +577,19 @@ public Response insertParagraph(@PathParam("noteId") String noteId, String messa @ZeppelinApi public Response getParagraph(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId) throws IOException { - LOGGER.info("Get paragraph {} {}", noteId, paragraphId); + try { + LOGGER.info("Get paragraph {} {}", noteId, paragraphId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRead(noteId, "Insufficient privileges you cannot get this paragraph"); - Paragraph p = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(p); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRead(noteId, "Insufficient privileges you cannot get this paragraph"); + Paragraph p = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(p); - return new JsonResponse<>(Status.OK, "", p).build(); + return new JsonResponse<>(Status.OK, "", p).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -558,26 +604,29 @@ public Response getParagraph(@PathParam("noteId") String noteId, public Response updateParagraph(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId, String message) throws IOException { - String user = authenticationService.getPrincipal(); - LOGGER.info("{} will update paragraph {} {}", user, noteId, paragraphId); + try { + String user = authenticationService.getPrincipal(); + LOGGER.info("{} will update paragraph {} {}", user, noteId, paragraphId); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph"); + Paragraph p = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(p); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph"); - Paragraph p = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(p); + UpdateParagraphRequest updatedParagraph = GSON.fromJson(message, UpdateParagraphRequest.class); + p.setText(updatedParagraph.getText()); - UpdateParagraphRequest updatedParagraph = GSON.fromJson(message, UpdateParagraphRequest.class); - p.setText(updatedParagraph.getText()); + if (updatedParagraph.getTitle() != null) { + p.setTitle(updatedParagraph.getTitle()); + } - if (updatedParagraph.getTitle() != null) { - p.setTitle(updatedParagraph.getTitle()); + AuthenticationInfo subject = new AuthenticationInfo(user); + notebook.saveNote(note, subject); + notebookServer.broadcastParagraph(note, p, MSG_ID_NOT_DEFINED); + return new JsonResponse<>(Status.OK, "").build(); + } catch (Throwable e) { + return createResponseFromException(e); } - - AuthenticationInfo subject = new AuthenticationInfo(user); - notebook.saveNote(note, subject); - notebookServer.broadcastParagraph(note, p, MSG_ID_NOT_DEFINED); - return new JsonResponse<>(Status.OK, "").build(); } /** @@ -595,20 +644,24 @@ public Response updateParagraph(@PathParam("noteId") String noteId, public Response updateParagraphConfig(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId, String message) throws IOException { - String user = authenticationService.getPrincipal(); - LOGGER.info("{} will update paragraph config {} {}", user, noteId, paragraphId); - - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph config"); - Paragraph p = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(p); + try { + String user = authenticationService.getPrincipal(); + LOGGER.info("{} will update paragraph config {} {}", user, noteId, paragraphId); - Map newConfig = GSON.fromJson(message, HashMap.class); - configureParagraph(p, newConfig, user); - AuthenticationInfo subject = new AuthenticationInfo(user); - notebook.saveNote(note, subject); - return new JsonResponse<>(Status.OK, "", p).build(); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph config"); + Paragraph p = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(p); + + Map newConfig = GSON.fromJson(message, HashMap.class); + configureParagraph(p, newConfig, user); + AuthenticationInfo subject = new AuthenticationInfo(user); + notebook.saveNote(note, subject); + return new JsonResponse<>(Status.OK, "", p).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -625,17 +678,21 @@ public Response moveParagraph(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId, @PathParam("newIndex") String newIndex) throws IOException { - LOGGER.info("Move paragraph {} {} {}", noteId, paragraphId, newIndex); - notebookService.moveParagraph(noteId, paragraphId, Integer.parseInt(newIndex), - getServiceContext(), - new RestServiceCallback() { - @Override - public void onSuccess(Paragraph result, ServiceContext context) throws IOException { - notebookServer.broadcastNote(result.getNote()); - } - }); - return new JsonResponse(Status.OK, "").build(); + try { + LOGGER.info("Move paragraph {} {} {}", noteId, paragraphId, newIndex); + notebookService.moveParagraph(noteId, paragraphId, Integer.parseInt(newIndex), + getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(Paragraph result, ServiceContext context) throws IOException { + notebookServer.broadcastNote(result.getNote()); + } + }); + return new JsonResponse(Status.OK, "").build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -650,16 +707,34 @@ public void onSuccess(Paragraph result, ServiceContext context) throws IOExcepti @ZeppelinApi public Response deleteParagraph(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId) throws IOException { - LOGGER.info("Delete paragraph {} {}", noteId, paragraphId); - notebookService.removeParagraph(noteId, paragraphId, getServiceContext(), - new RestServiceCallback() { - @Override - public void onSuccess(Paragraph p, ServiceContext context) throws IOException { - notebookServer.broadcastNote(p.getNote()); - } - }); - - return new JsonResponse(Status.OK, "").build(); + try { + LOGGER.info("Delete paragraph {} {}", noteId, paragraphId); + notebookService.removeParagraph(noteId, paragraphId, getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(Paragraph p, ServiceContext context) throws IOException { + notebookServer.broadcastNote(p.getNote()); + } + }); + + return new JsonResponse(Status.OK, "").build(); + } catch (Throwable e) { + return createResponseFromException(e); + } + } + + @POST + @Path("{noteId}/paragraph/next") + public Response nextSessionParagraph(@PathParam("noteId") String noteId, + @QueryParam("maxParagraph") int maxParagraph) { + try { + Paragraph p = notebookService.getNextSessionParagraph(noteId, maxParagraph, + getServiceContext(), + new RestServiceCallback<>()); + return new JsonResponse(Status.OK, p.getId()).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -673,10 +748,14 @@ public void onSuccess(Paragraph p, ServiceContext context) throws IOException { @ZeppelinApi public Response clearAllParagraphOutput(@PathParam("noteId") String noteId) throws IOException { - LOGGER.info("Clear all paragraph output of note {}", noteId); - notebookService.clearAllParagraphOutput(noteId, getServiceContext(), - new RestServiceCallback<>()); - return new JsonResponse(Status.OK, "").build(); + try { + LOGGER.info("Clear all paragraph output of note {}", noteId); + notebookService.clearAllParagraphOutput(noteId, getServiceContext(), + new RestServiceCallback<>()); + return new JsonResponse(Status.OK, "").build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -698,33 +777,34 @@ public Response runNoteJobs(@PathParam("noteId") String noteId, @QueryParam("isolated") Boolean isolated, String message) throws IOException, IllegalArgumentException { - if (blocking == null) { - blocking = false; - } - if (isolated == null) { - isolated = false; - } - Map params = new HashMap<>(); - if (!StringUtils.isEmpty(message)) { - ParametersRequest request = - ParametersRequest.fromJson(message); - params = request.getParams(); - } - LOGGER.info("Run note jobs, noteId: {}, blocking: {}, isolated: {}, params: {}", noteId, blocking, isolated, params); - Note note = notebook.getNote(noteId); - AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); - subject.setRoles(new LinkedList<>(authenticationService.getAssociatedRoles())); - checkIfNoteIsNotNull(note); - checkIfUserCanRun(noteId, "Insufficient privileges you cannot run job for this note"); - - //TODO(zjffdu), can we run a note via rest api when cron is enabled ? try { + if (blocking == null) { + blocking = false; + } + if (isolated == null) { + isolated = false; + } + Map params = new HashMap<>(); + if (!StringUtils.isEmpty(message)) { + ParametersRequest request = + ParametersRequest.fromJson(message); + params = request.getParams(); + } + + LOGGER.info("Run note jobs, noteId: {}, blocking: {}, isolated: {}, params: {}", noteId, blocking, isolated, params); + Note note = notebook.getNote(noteId); + AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); + subject.setRoles(new LinkedList<>(authenticationService.getAssociatedRoles())); + checkIfNoteIsNotNull(note); + checkIfUserCanRun(noteId, "Insufficient privileges you cannot run job for this note"); + + //TODO(zjffdu), can we run a note via rest api when cron is enabled ? note.runAll(subject, blocking, isolated, params); return new JsonResponse<>(Status.OK).build(); - } catch (Exception ex) { - LOGGER.error("Exception from run", ex); - return new JsonResponse<>(Status.EXPECTATION_FAILED, ex.getMessage()).build(); + } catch (Throwable e) { + LOGGER.error("Exception from run", e); + return createResponseFromException(e); } } @@ -741,17 +821,21 @@ public Response runNoteJobs(@PathParam("noteId") String noteId, @ZeppelinApi public Response stopNoteJobs(@PathParam("noteId") String noteId) throws IOException, IllegalArgumentException { - LOGGER.info("Stop note jobs {} ", noteId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRun(noteId, "Insufficient privileges you cannot stop this job for this note"); - - for (Paragraph p : note.getParagraphs()) { - if (!p.isTerminated()) { - p.abort(); + try { + LOGGER.info("Stop note jobs {} ", noteId); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRun(noteId, "Insufficient privileges you cannot stop this job for this note"); + + for (Paragraph p : note.getParagraphs()) { + if (!p.isTerminated()) { + p.abort(); + } } + return new JsonResponse<>(Status.OK).build(); + } catch (Throwable e) { + return createResponseFromException(e); } - return new JsonResponse<>(Status.OK).build(); } /** @@ -767,12 +851,16 @@ public Response stopNoteJobs(@PathParam("noteId") String noteId) @ZeppelinApi public Response getNoteJobStatus(@PathParam("noteId") String noteId) throws IOException, IllegalArgumentException { - LOGGER.info("Get note job status."); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status"); - - return new JsonResponse<>(Status.OK, null, new NoteJobStatus(note)).build(); + try { + LOGGER.info("Get note job status."); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status"); + + return new JsonResponse<>(Status.OK, null, new NoteJobStatus(note)).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -790,15 +878,19 @@ public Response getNoteJobStatus(@PathParam("noteId") String noteId) public Response getNoteParagraphJobStatus(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException { - LOGGER.info("Get note paragraph job status."); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status"); + try { + LOGGER.info("Get note paragraph job status."); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status"); - Paragraph paragraph = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(paragraph); + Paragraph paragraph = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(paragraph); - return new JsonResponse<>(Status.OK, null, new ParagraphJobStatus(paragraph)).build(); + return new JsonResponse<>(Status.OK, null, new ParagraphJobStatus(paragraph)).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -814,25 +906,32 @@ public Response getNoteParagraphJobStatus(@PathParam("noteId") String noteId, @Path("job/{noteId}/{paragraphId}") @ZeppelinApi public Response runParagraph(@PathParam("noteId") String noteId, - @PathParam("paragraphId") String paragraphId, String message) + @PathParam("paragraphId") String paragraphId, + @QueryParam("sessionId") String sessionId, + String message) throws IOException, IllegalArgumentException { - LOGGER.info("Run paragraph job asynchronously {} {} {}", noteId, paragraphId, message); - - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - Paragraph paragraph = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(paragraph); - Map params = new HashMap<>(); - if (!StringUtils.isEmpty(message)) { - ParametersRequest request = - ParametersRequest.fromJson(message); - params = request.getParams(); + try { + LOGGER.info("Run paragraph job asynchronously {} {} {}", noteId, paragraphId, message); + + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + Paragraph paragraph = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(paragraph); + + Map params = new HashMap<>(); + if (!StringUtils.isEmpty(message)) { + ParametersRequest request = + ParametersRequest.fromJson(message); + params = request.getParams(); + } + notebookService.runParagraph(noteId, paragraphId, paragraph.getTitle(), + paragraph.getText(), params, new HashMap<>(), sessionId, + false, false, getServiceContext(), new RestServiceCallback<>()); + return new JsonResponse<>(Status.OK).build(); + } catch (Throwable e) { + return createResponseFromException(e); } - notebookService.runParagraph(noteId, paragraphId, paragraph.getTitle(), - paragraph.getText(), params, new HashMap<>(), - false, false, getServiceContext(), new RestServiceCallback<>()); - return new JsonResponse<>(Status.OK).build(); } /** @@ -851,35 +950,36 @@ public Response runParagraph(@PathParam("noteId") String noteId, @ZeppelinApi public Response runParagraphSynchronously(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId, + @QueryParam("sessionId") String sessionId, String message) throws IOException, IllegalArgumentException { LOGGER.info("Run paragraph synchronously {} {} {}", noteId, paragraphId, message); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - Paragraph paragraph = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(paragraph); - - Map params = new HashMap<>(); - if (!StringUtils.isEmpty(message)) { - ParametersRequest request = - ParametersRequest.fromJson(message); - params = request.getParams(); - } + try { + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + Paragraph paragraph = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(paragraph); + + Map params = new HashMap<>(); + if (!StringUtils.isEmpty(message)) { + ParametersRequest request = + ParametersRequest.fromJson(message); + params = request.getParams(); + } - if (notebookService.runParagraph(noteId, paragraphId, paragraph.getTitle(), - paragraph.getText(), params, - new HashMap<>(), false, true, getServiceContext(), new RestServiceCallback<>())) { - note = notebookService.getNote(noteId, getServiceContext(), new RestServiceCallback<>()); - Paragraph p = note.getParagraph(paragraphId); - InterpreterResult result = p.getReturn(); - if (result.code() == InterpreterResult.Code.SUCCESS) { + if (notebookService.runParagraph(noteId, paragraphId, paragraph.getTitle(), + paragraph.getText(), params, + new HashMap<>(), sessionId, false, true, getServiceContext(), new RestServiceCallback<>())) { + note = notebookService.getNote(noteId, getServiceContext(), new RestServiceCallback<>()); + Paragraph p = note.getParagraph(paragraphId); + InterpreterResult result = p.getReturn(); return new JsonResponse<>(Status.OK, result).build(); } else { - return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, result).build(); + return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, "Fail to run paragraph").build(); } - } else { - return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, "Fail to run paragraph").build(); + } catch (Throwable e) { + return createResponseFromException(e); } } @@ -898,10 +998,14 @@ public Response runParagraphSynchronously(@PathParam("noteId") String noteId, public Response cancelParagraph(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException { - LOGGER.info("stop paragraph job {} ", noteId); - notebookService.cancelParagraph(noteId, paragraphId, getServiceContext(), - new RestServiceCallback()); - return new JsonResponse<>(Status.OK).build(); + try { + LOGGER.info("stop paragraph job {} ", noteId); + notebookService.cancelParagraph(noteId, paragraphId, getServiceContext(), + new RestServiceCallback()); + return new JsonResponse<>(Status.OK).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -917,26 +1021,31 @@ public Response cancelParagraph(@PathParam("noteId") String noteId, @ZeppelinApi public Response registerCronJob(@PathParam("noteId") String noteId, String message) throws IOException, IllegalArgumentException { - LOGGER.info("Register cron job note={} request cron msg={}", noteId, message); - CronRequest request = CronRequest.fromJson(message); + try { + LOGGER.info("Register cron job note={} request cron msg={}", noteId, message); + + CronRequest request = CronRequest.fromJson(message); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRun(noteId, "Insufficient privileges you cannot set a cron job for this note"); - checkIfNoteSupportsCron(note); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRun(noteId, "Insufficient privileges you cannot set a cron job for this note"); + checkIfNoteSupportsCron(note); - if (!CronExpression.isValidExpression(request.getCronString())) { - return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build(); - } + if (!CronExpression.isValidExpression(request.getCronString())) { + return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build(); + } - Map config = note.getConfig(); - config.put("cron", request.getCronString()); - config.put("releaseresource", request.getReleaseResource()); - note.setConfig(config); - schedulerService.refreshCron(note.getId()); + Map config = note.getConfig(); + config.put("cron", request.getCronString()); + config.put("releaseresource", request.getReleaseResource()); + note.setConfig(config); + schedulerService.refreshCron(note.getId()); - return new JsonResponse<>(Status.OK).build(); + return new JsonResponse<>(Status.OK).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -952,21 +1061,26 @@ public Response registerCronJob(@PathParam("noteId") String noteId, String messa @ZeppelinApi public Response removeCronJob(@PathParam("noteId") String noteId) throws IOException, IllegalArgumentException { - LOGGER.info("Remove cron job note {}", noteId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserIsOwner(noteId, - "Insufficient privileges you cannot remove this cron job from this note"); - checkIfNoteSupportsCron(note); + try { + LOGGER.info("Remove cron job note {}", noteId); + + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserIsOwner(noteId, + "Insufficient privileges you cannot remove this cron job from this note"); + checkIfNoteSupportsCron(note); - Map config = note.getConfig(); - config.remove("cron"); - config.remove("releaseresource"); - note.setConfig(config); - schedulerService.refreshCron(note.getId()); + Map config = note.getConfig(); + config.remove("cron"); + config.remove("releaseresource"); + note.setConfig(config); + schedulerService.refreshCron(note.getId()); - return new JsonResponse<>(Status.OK).build(); + return new JsonResponse<>(Status.OK).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -982,17 +1096,22 @@ public Response removeCronJob(@PathParam("noteId") String noteId) @ZeppelinApi public Response getCronJob(@PathParam("noteId") String noteId) throws IOException, IllegalArgumentException { - LOGGER.info("Get cron job note {}", noteId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRead(noteId, "Insufficient privileges you cannot get cron information"); - checkIfNoteSupportsCron(note); - Map response = new HashMap<>(); - response.put("cron", note.getConfig().get("cron")); - response.put("releaseResource", note.getConfig().get("releaseresource")); - - return new JsonResponse<>(Status.OK, response).build(); + try { + LOGGER.info("Get cron job note {}", noteId); + + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRead(noteId, "Insufficient privileges you cannot get cron information"); + checkIfNoteSupportsCron(note); + Map response = new HashMap<>(); + response.put("cron", note.getConfig().get("cron")); + response.put("releaseResource", note.getConfig().get("releaseresource")); + + return new JsonResponse<>(Status.OK, response).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -1006,13 +1125,17 @@ public Response getCronJob(@PathParam("noteId") String noteId) @Path("jobmanager/") @ZeppelinApi public Response getJobListforNote() throws IOException, IllegalArgumentException { - LOGGER.info("Get note jobs for job manager"); - List noteJobs = jobManagerService - .getNoteJobInfoByUnixTime(0, getServiceContext(), new RestServiceCallback<>()); - Map response = new HashMap<>(); - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", noteJobs); - return new JsonResponse<>(Status.OK, response).build(); + try { + LOGGER.info("Get note jobs for job manager"); + List noteJobs = jobManagerService + .getNoteJobInfoByUnixTime(0, getServiceContext(), new RestServiceCallback<>()); + Map response = new HashMap<>(); + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", noteJobs); + return new JsonResponse<>(Status.OK, response).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -1029,14 +1152,18 @@ public Response getJobListforNote() throws IOException, IllegalArgumentException @ZeppelinApi public Response getUpdatedJobListforNote(@PathParam("lastUpdateUnixtime") long lastUpdateUnixTime) throws IOException, IllegalArgumentException { - LOGGER.info("Get updated note jobs lastUpdateTime {}", lastUpdateUnixTime); - List noteJobs = - jobManagerService.getNoteJobInfoByUnixTime(lastUpdateUnixTime, getServiceContext(), - new RestServiceCallback<>()); - Map response = new HashMap<>(); - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", noteJobs); - return new JsonResponse<>(Status.OK, response).build(); + try { + LOGGER.info("Get updated note jobs lastUpdateTime {}", lastUpdateUnixTime); + List noteJobs = + jobManagerService.getNoteJobInfoByUnixTime(lastUpdateUnixTime, getServiceContext(), + new RestServiceCallback<>()); + Map response = new HashMap<>(); + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", noteJobs); + return new JsonResponse<>(Status.OK, response).build(); + } catch (Throwable e) { + return createResponseFromException(e); + } } /** @@ -1046,26 +1173,30 @@ public Response getUpdatedJobListforNote(@PathParam("lastUpdateUnixtime") long l @Path("search") @ZeppelinApi public Response search(@QueryParam("q") String queryTerm) { - LOGGER.info("Searching notes for: {}", queryTerm); - String principal = authenticationService.getPrincipal(); - Set roles = authenticationService.getAssociatedRoles(); - HashSet userAndRoles = new HashSet<>(); - userAndRoles.add(principal); - userAndRoles.addAll(roles); - List> notesFound = noteSearchService.query(queryTerm); - for (int i = 0; i < notesFound.size(); i++) { - String[] ids = notesFound.get(i).get("id").split("/", 2); - String noteId = ids[0]; - if (!authorizationService.isOwner(noteId, userAndRoles) && - !authorizationService.isReader(noteId, userAndRoles) && - !authorizationService.isWriter(noteId, userAndRoles) && - !authorizationService.isRunner(noteId, userAndRoles)) { - notesFound.remove(i); - i--; + try { + LOGGER.info("Searching notes for: {}", queryTerm); + String principal = authenticationService.getPrincipal(); + Set roles = authenticationService.getAssociatedRoles(); + HashSet userAndRoles = new HashSet<>(); + userAndRoles.add(principal); + userAndRoles.addAll(roles); + List> notesFound = noteSearchService.query(queryTerm); + for (int i = 0; i < notesFound.size(); i++) { + String[] ids = notesFound.get(i).get("id").split("/", 2); + String noteId = ids[0]; + if (!authorizationService.isOwner(noteId, userAndRoles) && + !authorizationService.isReader(noteId, userAndRoles) && + !authorizationService.isWriter(noteId, userAndRoles) && + !authorizationService.isRunner(noteId, userAndRoles)) { + notesFound.remove(i); + i--; + } } + LOGGER.info("{} notes found", notesFound.size()); + return new JsonResponse<>(Status.OK, notesFound).build(); + } catch (Throwable e) { + return createResponseFromException(e); } - LOGGER.info("{} notes found", notesFound.size()); - return new JsonResponse<>(Status.OK, notesFound).build(); } @@ -1109,4 +1240,12 @@ private void configureParagraph(Paragraph p, Map newConfig, Stri p.setConfig(origConfig); } + + private Response createResponseFromException(Throwable e) { + if (e instanceof WebApplicationException) { + return ((WebApplicationException) e).getResponse(); + } else { + return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, ExceptionUtils.getStackTrace(e)).build(); + } + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java new file mode 100644 index 00000000000..4fdae91ceff --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.rest; + +import java.util.HashSet; +import java.util.Set; + +public class SessionManager { + + private static final int RETRY = 3; + private Set sessions = new HashSet<>(); + + public synchronized String newSession(String interpreter) throws Exception { + int i = 0; + while (i < RETRY) { + String sessionId = interpreter + "_" + System.currentTimeMillis(); + if (sessions.contains(sessionId)) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } else { + sessions.add(sessionId); + return sessionId; + } + } + + throw new Exception("Unable to generate session id"); + } + + public void removeSession(String sessionId) { + this.sessions.remove(sessionId); + } + +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java new file mode 100644 index 00000000000..f1b7086298d --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.rest; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.zeppelin.annotation.ZeppelinApi; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; +import org.apache.zeppelin.notebook.Notebook; +import org.apache.zeppelin.server.JsonResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; +import java.util.HashMap; +import java.util.Map; + +/** + * Rest api endpoint for the ZSession. + */ +@Path("/session") +@Produces("application/json") +@Singleton +public class SessionRestApi { + + private static final Logger LOGGER = LoggerFactory.getLogger(SessionRestApi.class); + + private Notebook notebook; + private InterpreterSettingManager interpreterSettingManager; + private SessionManager sessionManager; + + @Inject + public SessionRestApi(Notebook notebook, InterpreterSettingManager interpreterSettingManager) { + this.notebook = notebook; + this.interpreterSettingManager = interpreterSettingManager; + this.sessionManager = new SessionManager(); + } + + @POST + @Path("{interpreter}") + public Response newSession(@PathParam("interpreter") String interpreter) { + LOGGER.info("New session for interpreter: {}", interpreter); + try { + String sessionId = sessionManager.newSession(interpreter); + LOGGER.info("Allocate new session id: " + sessionId); + return new JsonResponse<>(Response.Status.OK, sessionId).build(); + } catch (Exception e) { + return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, + "Fail to start session", ExceptionUtils.getStackTrace(e)).build(); + } + } + + @DELETE + @Path("{interpreter}/{sessionId}") + public Response stopSession(@PathParam("interpreter") String interpreter, + @PathParam("sessionId") String sessionId) { + LOGGER.info("Stop session: {} for interpreter: {}", sessionId, interpreter); + try { + sessionManager.removeSession(interpreter); + notebook.getInterpreterSettingManager().get(interpreter).closeInterpreters(sessionId); + return new JsonResponse<>(Response.Status.OK, sessionId).build(); + } catch (Exception e) { + return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, + "Fail to stop session", ExceptionUtils.getStackTrace(e)).build(); + } + } + + /** + * Get a session info. + */ + @GET + @Path("{sessionId}") + @ZeppelinApi + public Response getSession(@PathParam("sessionId") String sessionId) { + try { + InterpreterGroup intpGroup = interpreterSettingManager.getInterpreterGroupById(sessionId); + if (intpGroup == null) { + return new JsonResponse<>(Response.Status.NOT_FOUND).build(); + } else { + Map session = new HashMap<>(); + session.put("id", sessionId); + session.put("weburl", intpGroup.getWebUrl()); + return new JsonResponse<>(Response.Status.OK, "", session).build(); + } + } catch (Throwable e) { + LOGGER.error("Exception in SessionRestApi while getSession ", e); + return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage(), + ExceptionUtils.getStackTrace(e)).build(); + } + } +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java index a1b27b5bb86..8f69998ca64 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java @@ -59,6 +59,7 @@ import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.user.AuthenticationInfo; import org.bitbucket.cowwoc.diffmatchpatch.DiffMatchPatch; +import org.eclipse.jgit.errors.MissingObjectException; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -136,8 +137,19 @@ public Note getNote(String noteId, } + /** + * + * @param notePath + * @param defaultInterpreterGroup + * @param addingEmptyParagraph + * @param context + * @param callback + * @return + * @throws IOException + */ public Note createNote(String notePath, String defaultInterpreterGroup, + boolean addingEmptyParagraph, ServiceContext context, ServiceCallback callback) throws IOException { @@ -150,7 +162,9 @@ public Note createNote(String notePath, Note note = notebook.createNote(normalizeNotePath(notePath), defaultInterpreterGroup, context.getAutheInfo(), false); // it's an empty note. so add one paragraph - note.addNewParagraph(context.getAutheInfo()); + if (addingEmptyParagraph) { + note.addNewParagraph(context.getAutheInfo()); + } notebook.saveNote(note, context.getAutheInfo()); callback.onSuccess(note, context); return note; @@ -304,6 +318,7 @@ public boolean runParagraph(String noteId, String text, Map params, Map config, + String sessionId, boolean failIfDisabled, boolean blocking, ServiceContext context, @@ -353,7 +368,7 @@ public boolean runParagraph(String noteId, try { notebook.saveNote(note, context.getAutheInfo()); - note.run(p.getId(), blocking, context.getAutheInfo().getUser()); + note.run(p.getId(), sessionId, blocking, context.getAutheInfo().getUser()); callback.onSuccess(p, context); return true; } catch (Exception ex) { @@ -409,7 +424,7 @@ public boolean runAllParagraphs(String noteId, Map params = (Map) raw.get("params"); Map config = (Map) raw.get("config"); - if (!runParagraph(noteId, paragraphId, title, text, params, config, false, true, + if (!runParagraph(noteId, paragraphId, title, text, params, config, null, false, true, context, callback)) { // stop execution when one paragraph fails. return false; @@ -619,6 +634,37 @@ public void updateParagraph(String noteId, callback.onSuccess(p, context); } + public Paragraph getNextSessionParagraph(String noteId, + int maxParagraph, + ServiceContext context, + ServiceCallback callback) throws IOException { + if (!checkPermission(noteId, Permission.WRITER, Message.OP.PARAGRAPH_CLEAR_OUTPUT, context, + callback)) { + throw new IOException("No privilege to access this note"); + } + Note note = notebook.getNote(noteId); + if (note == null) { + callback.onFailure(new NoteNotFoundException(noteId), context); + throw new IOException("No such note"); + } + if (note.getParagraphCount() < maxParagraph) { + return note.addNewParagraph(context.getAutheInfo()); + } else { + boolean removed = false; + for (int i = 1; i< note.getParagraphCount(); ++i) { + if (note.getParagraph(i).getStatus().isCompleted()) { + note.removeParagraph(context.getAutheInfo().getUser(), note.getParagraph(i).getId()); + removed = true; + break; + } + } + if (!removed) { + throw new IOException("All the paragraphs are not completed, unable to find available paragraph"); + } + return note.addNewParagraph(context.getAutheInfo()); + } + } + public void clearParagraphOutput(String noteId, String paragraphId, ServiceContext context, diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 2db1d26e8fa..0a16c21141b 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -962,7 +962,7 @@ private void createNote(NotebookSocket conn, String noteName = (String) message.get("name"); String defaultInterpreterGroup = (String) message.get("defaultInterpreterGroup"); - getNotebookService().createNote(noteName, defaultInterpreterGroup, getServiceContext(message), + getNotebookService().createNote(noteName, defaultInterpreterGroup, true, getServiceContext(message), new WebSocketServiceCallback(conn) { @Override public void onSuccess(Note note, ServiceContext context) throws IOException { @@ -1516,7 +1516,7 @@ private void runParagraph(NotebookSocket conn, String title = (String) fromMessage.get("title"); Map params = (Map) fromMessage.get("params"); Map config = (Map) fromMessage.get("config"); - getNotebookService().runParagraph(noteId, paragraphId, title, text, params, config, + getNotebookService().runParagraph(noteId, paragraphId, title, text, params, config, null, false, false, getServiceContext(fromMessage), new WebSocketServiceCallback(conn) { @Override diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookWebSocketCreator.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookWebSocketCreator.java index 7033929c10f..6a9c8a8c09f 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookWebSocketCreator.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookWebSocketCreator.java @@ -35,6 +35,7 @@ public class NotebookWebSocketCreator implements WebSocketCreator { public NotebookWebSocketCreator(NotebookServer notebookServer) { this.notebookServer = notebookServer; } + public Object createWebSocket(ServletUpgradeRequest request, ServletUpgradeResponse response) { String origin = request.getHeader("Origin"); if (notebookServer.checkOrigin(request.getHttpServletRequest(), origin)) { diff --git a/zeppelin-server/src/main/resources/log4j.properties b/zeppelin-server/src/main/resources/log4j.properties new file mode 100644 index 00000000000..0940c7f7b1c --- /dev/null +++ b/zeppelin-server/src/main/resources/log4j.properties @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger = INFO, stdout + +log4j.appender.stdout = org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout = org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n + +log4j.appender.dailyfile.DatePattern=.yyyy-MM-dd +log4j.appender.dailyfile = org.apache.log4j.DailyRollingFileAppender +log4j.appender.dailyfile.File = ${zeppelin.log.file} +log4j.appender.dailyfile.layout = org.apache.log4j.PatternLayout +log4j.appender.dailyfile.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 9ebd83c3c6c..b6c626d6260 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -186,6 +186,7 @@ private static void start(boolean withAuth, zeppelinHome = new File(".."); LOG.info("ZEPPELIN_HOME: " + zeppelinHome.getAbsolutePath()); confDir = new File(zeppelinHome, "conf_" + testClassName); + FileUtils.deleteDirectory(confDir); LOG.info("ZEPPELIN_CONF_DIR: " + confDir.getAbsolutePath()); confDir.mkdirs(); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java index 6039bf2008f..261f1bae7eb 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java @@ -176,10 +176,10 @@ public void testRunParagraphSynchronously() throws IOException { p.setText(text); post = httpPost("/notebook/run/" + note1.getId() + "/" + p.getId(), ""); - assertEquals(500, post.getStatusCode()); + assertEquals(200, post.getStatusCode()); resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken>() {}.getType()); - assertEquals("INTERNAL_SERVER_ERROR", resp.get("status")); + assertEquals("OK", resp.get("status")); StringMap stringMap = (StringMap) resp.get("body"); assertEquals("ERROR", stringMap.get("code")); List interpreterResults = (List) stringMap.get("msg"); @@ -470,12 +470,7 @@ public void testRunAllParagraph_FirstFailed() throws IOException { p2.setText("%python user2='abc'\nprint(user2)"); PostMethod post = httpPost("/notebook/job/" + note1.getId() + "?blocking=true", ""); - assertThat(post, isExpectationFailed()); - Map resp = gson.fromJson(post.getResponseBodyAsString(), - new TypeToken>() {}.getType()); - assertEquals(resp.get("status"), "EXPECTATION_FAILED"); - assertTrue(resp.get("message").toString().contains("Fail to run note because paragraph")); - post.releaseConnection(); + assertThat(post, isAllowed()); assertEquals(Job.Status.ERROR, p1.getStatus()); // p2 will be skipped because p1 is failed. diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookSecurityRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookSecurityRestApiTest.java index b949d057d83..a3d67c86bb3 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookSecurityRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookSecurityRestApiTest.java @@ -58,7 +58,7 @@ public void setUp() {} @Test public void testThatUserCanCreateAndRemoveNote() throws IOException { - String noteId = createNoteForUser("test", "admin", "password1"); + String noteId = createNoteForUser("test_1", "admin", "password1"); assertNotNull(noteId); String id = getNoteIdForUser(noteId, "admin", "password1"); assertThat(id, is(noteId)); @@ -67,7 +67,7 @@ public void testThatUserCanCreateAndRemoveNote() throws IOException { @Test public void testThatOtherUserCanAccessNoteIfPermissionNotSet() throws IOException { - String noteId = createNoteForUser("test", "admin", "password1"); + String noteId = createNoteForUser("test_2", "admin", "password1"); userTryGetNote(noteId, "user1", "password2", isAllowed()); @@ -76,7 +76,7 @@ public void testThatOtherUserCanAccessNoteIfPermissionNotSet() throws IOExceptio @Test public void testThatOtherUserCannotAccessNoteIfPermissionSet() throws IOException { - String noteId = createNoteForUser("test", "admin", "password1"); + String noteId = createNoteForUser("test_3", "admin", "password1"); //set permission String payload = "{ \"owners\": [\"admin\"], \"readers\": [\"user2\"], " + @@ -94,7 +94,7 @@ public void testThatOtherUserCannotAccessNoteIfPermissionSet() throws IOExceptio @Test public void testThatWriterCannotRemoveNote() throws IOException { - String noteId = createNoteForUser("test", "admin", "password1"); + String noteId = createNoteForUser("test_4", "admin", "password1"); //set permission String payload = "{ \"owners\": [\"admin\", \"user1\"], \"readers\": [\"user2\"], " + diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/service/NotebookServiceTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/service/NotebookServiceTest.java index a463617806d..4c2156af477 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/service/NotebookServiceTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/service/NotebookServiceTest.java @@ -155,14 +155,14 @@ public void testNoteOperations() throws IOException { verify(callback).onSuccess(homeNote, context); // create note - Note note1 = notebookService.createNote("/folder_1/note1", "test", context, callback); + Note note1 = notebookService.createNote("/folder_1/note1", "test", true, context, callback); assertEquals("note1", note1.getName()); assertEquals(1, note1.getParagraphCount()); verify(callback).onSuccess(note1, context); // create duplicated note reset(callback); - Note note2 = notebookService.createNote("/folder_1/note1", "test", context, callback); + Note note2 = notebookService.createNote("/folder_1/note1", "test", true, context, callback); assertNull(note2); ArgumentCaptor exception = ArgumentCaptor.forClass(Exception.class); verify(callback).onFailure(exception.capture(), any(ServiceContext.class)); @@ -203,7 +203,7 @@ public void testNoteOperations() throws IOException { assertEquals("/folder_4/new_name", notesInfo.get(0).getPath()); // create another note - note2 = notebookService.createNote("/note2", "test", context, callback); + note2 = notebookService.createNote("/note2", "test", true, context, callback); assertEquals("note2", note2.getName()); verify(callback).onSuccess(note2, context); @@ -334,18 +334,18 @@ public void testNoteOperations() throws IOException { @Test public void testParagraphOperations() throws IOException { // create note - Note note1 = notebookService.createNote("note1", "python", context, callback); + Note note1 = notebookService.createNote("note1", "python", false, context, callback); assertEquals("note1", note1.getName()); - assertEquals(1, note1.getParagraphCount()); + assertEquals(0, note1.getParagraphCount()); verify(callback).onSuccess(note1, context); // add paragraph reset(callback); - Paragraph p = notebookService.insertParagraph(note1.getId(), 1, new HashMap<>(), context, + Paragraph p = notebookService.insertParagraph(note1.getId(), 0, new HashMap<>(), context, callback); assertNotNull(p); verify(callback).onSuccess(p, context); - assertEquals(2, note1.getParagraphCount()); + assertEquals(1, note1.getParagraphCount()); // update paragraph reset(callback); @@ -365,7 +365,7 @@ public void testParagraphOperations() throws IOException { p.getConfig().put("colWidth", "6.0"); p.getConfig().put("title", true); boolean runStatus = notebookService.runParagraph(note1.getId(), p.getId(), "my_title", "1+1", - new HashMap<>(), new HashMap<>(), false, false, context, callback); + new HashMap<>(), new HashMap<>(), null, false, false, context, callback); assertTrue(runStatus); verify(callback).onSuccess(p, context); assertEquals(2, p.getConfig().size()); @@ -373,7 +373,7 @@ public void testParagraphOperations() throws IOException { // run paragraph synchronously via correct code reset(callback); runStatus = notebookService.runParagraph(note1.getId(), p.getId(), "my_title", "1+1", - new HashMap<>(), new HashMap<>(), false, true, context, callback); + new HashMap<>(), new HashMap<>(), null, false, true, context, callback); assertTrue(runStatus); verify(callback).onSuccess(p, context); assertEquals(2, p.getConfig().size()); @@ -387,7 +387,7 @@ public void testParagraphOperations() throws IOException { reset(callback); runStatus = notebookService.runParagraph(note1.getId(), p.getId(), "my_title", "invalid_code", - new HashMap<>(), new HashMap<>(), false, true, context, callback); + new HashMap<>(), new HashMap<>(), null, false, true, context, callback); assertTrue(runStatus); // TODO(zjffdu) Enable it after ZEPPELIN-3699 // assertNotNull(p.getResult()); diff --git a/zeppelin-server/src/test/resources/zeppelin-site.xml b/zeppelin-server/src/test/resources/zeppelin-site.xml index e46fce76600..32373bbf107 100644 --- a/zeppelin-server/src/test/resources/zeppelin-site.xml +++ b/zeppelin-server/src/test/resources/zeppelin-site.xml @@ -142,7 +142,7 @@ zeppelin.server.allowed.origins - http://onehost:8080,http://otherhost.com, + * Allowed sources for REST and WebSocket requests. + + + + - org.apache.zeppelin - zeppelin-zengine - ${project.version} + org.eclipse.jetty.websocket + websocket-client + ${jetty.version} + com.konghq unirest-java diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java index 93e47767fa5..469e0ad42cc 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java @@ -18,7 +18,6 @@ package org.apache.zeppelin.client; import com.google.gson.Gson; -import org.apache.zeppelin.notebook.socket.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/JsonSerializable.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/JsonSerializable.java new file mode 100644 index 00000000000..0b5b570e6d1 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/JsonSerializable.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client; + +/** + * Interface for class that can be serialized to json + */ +public interface JsonSerializable { + + String toJson(); +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java new file mode 100644 index 00000000000..865b7541c21 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client; + +import com.google.gson.Gson; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Zeppelin websocket massage template class. + */ +public class Message implements JsonSerializable { + /** + * Representation of event type. + */ + public static enum OP { + GET_HOME_NOTE, // [c-s] load note for home screen + + GET_NOTE, // [c-s] client load note + // @param id note id + + NOTE, // [s-c] note info + // @param note serialized Note object + + PARAGRAPH, // [s-c] paragraph info + // @param paragraph serialized paragraph object + + PROGRESS, // [s-c] progress update + // @param id paragraph id + // @param progress percentage progress + + NEW_NOTE, // [c-s] create new notebook + DEL_NOTE, // [c-s] delete notebook + // @param id note id + REMOVE_FOLDER, + MOVE_NOTE_TO_TRASH, + MOVE_FOLDER_TO_TRASH, + RESTORE_FOLDER, + RESTORE_NOTE, + RESTORE_ALL, + EMPTY_TRASH, + CLONE_NOTE, // [c-s] clone new notebook + // @param id id of note to clone + // @param name name for the cloned note + IMPORT_NOTE, // [c-s] import notebook + // @param object notebook + + CONVERT_NOTE_NBFORMAT, // [c-s] converting note to nbformat + CONVERTED_NOTE_NBFORMAT, // [s-c] converting note to nbformat + + NOTE_UPDATE, + + NOTE_RENAME, + + UPDATE_PERSONALIZED_MODE, // [c-s] update personalized mode (boolean) + // @param note id and boolean personalized mode value + + FOLDER_RENAME, + + RUN_PARAGRAPH, // [c-s] run paragraph + // @param id paragraph id + // @param paragraph paragraph content.ie. script + // @param config paragraph config + // @param params paragraph params + + COMMIT_PARAGRAPH, // [c-s] commit paragraph + // @param id paragraph id + // @param title paragraph title + // @param paragraph paragraph content.ie. script + // @param config paragraph config + // @param params paragraph params + + CANCEL_PARAGRAPH, // [c-s] cancel paragraph run + // @param id paragraph id + + MOVE_PARAGRAPH, // [c-s] move paragraph order + // @param id paragraph id + // @param index index the paragraph want to go + + INSERT_PARAGRAPH, // [c-s] create new paragraph below current paragraph + // @param target index + + COPY_PARAGRAPH, // [c-s] create new para below current para as a copy of current para + // @param target index + // @param title paragraph title + // @param paragraph paragraph content.ie. script + // @param config paragraph config + // @param params paragraph params + + EDITOR_SETTING, // [c-s] ask paragraph editor setting + // @param paragraph text keyword written in paragraph + // ex) spark.spark or spark + + COMPLETION, // [c-s] ask completion candidates + // @param id + // @param buf current code + // @param cursor cursor position in code + + COMPLETION_LIST, // [s-c] send back completion candidates list + // @param id + // @param completions list of string + + LIST_NOTES, // [c-s] ask list of note + RELOAD_NOTES_FROM_REPO, // [c-s] reload notes from repo + + NOTES_INFO, // [s-c] list of note infos + // @param notes serialized List object + + PARAGRAPH_REMOVE, + PARAGRAPH_CLEAR_OUTPUT, // [c-s] clear output of paragraph + PARAGRAPH_CLEAR_ALL_OUTPUT, // [c-s] clear output of all paragraphs + PARAGRAPH_APPEND_OUTPUT, // [s-c] append output + PARAGRAPH_UPDATE_OUTPUT, // [s-c] update (replace) output + PING, + AUTH_INFO, + + ANGULAR_OBJECT_UPDATE, // [s-c] add/update angular object + ANGULAR_OBJECT_REMOVE, // [s-c] add angular object del + + ANGULAR_OBJECT_UPDATED, // [c-s] angular object value updated, + + ANGULAR_OBJECT_CLIENT_BIND, // [c-s] angular object updated from AngularJS z object + + ANGULAR_OBJECT_CLIENT_UNBIND, // [c-s] angular object unbind from AngularJS z object + + LIST_CONFIGURATIONS, // [c-s] ask all key/value pairs of configurations + CONFIGURATIONS_INFO, // [s-c] all key/value pairs of configurations + // @param settings serialized Map object + + CHECKPOINT_NOTE, // [c-s] checkpoint note to storage repository + // @param noteId + // @param checkpointName + + LIST_REVISION_HISTORY, // [c-s] list revision history of the notebook + // @param noteId + NOTE_REVISION, // [c-s] get certain revision of note + // @param noteId + // @param revisionId + SET_NOTE_REVISION, // [c-s] set current notebook head to this revision + // @param noteId + // @param revisionId + NOTE_REVISION_FOR_COMPARE, // [c-s] get certain revision of note for compare + // @param noteId + // @param revisionId + // @param position + APP_APPEND_OUTPUT, // [s-c] append output + APP_UPDATE_OUTPUT, // [s-c] update (replace) output + APP_LOAD, // [s-c] on app load + APP_STATUS_CHANGE, // [s-c] on app status change + + LIST_NOTE_JOBS, // [c-s] get note job management information + LIST_UPDATE_NOTE_JOBS, // [c-s] get job management information for until unixtime + UNSUBSCRIBE_UPDATE_NOTE_JOBS, // [c-s] unsubscribe job information for job management + // @param unixTime + GET_INTERPRETER_BINDINGS, // [c-s] get interpreter bindings + SAVE_INTERPRETER_BINDINGS, // [c-s] save interpreter bindings + INTERPRETER_BINDINGS, // [s-c] interpreter bindings + + GET_INTERPRETER_SETTINGS, // [c-s] get interpreter settings + INTERPRETER_SETTINGS, // [s-c] interpreter settings + ERROR_INFO, // [s-c] error information to be sent + SESSION_LOGOUT, // [s-c] error information to be sent + WATCHER, // [s-c] Change websocket to watcher mode. + PARAGRAPH_ADDED, // [s-c] paragraph is added + PARAGRAPH_REMOVED, // [s-c] paragraph deleted + PARAGRAPH_MOVED, // [s-c] paragraph moved + NOTE_UPDATED, // [s-c] paragraph updated(name, config) + RUN_ALL_PARAGRAPHS, // [c-s] run all paragraphs + PARAGRAPH_EXECUTED_BY_SPELL, // [c-s] paragraph was executed by spell + RUN_PARAGRAPH_USING_SPELL, // [s-c] run paragraph using spell + PARAS_INFO, // [s-c] paragraph runtime infos + SAVE_NOTE_FORMS, // save note forms + REMOVE_NOTE_FORMS, // remove note forms + INTERPRETER_INSTALL_STARTED, // [s-c] start to download an interpreter + INTERPRETER_INSTALL_RESULT, // [s-c] Status of an interpreter installation + COLLABORATIVE_MODE_STATUS, // [s-c] collaborative mode status + PATCH_PARAGRAPH, // [c-s][s-c] patch editor text + NOTE_RUNNING_STATUS, // [s-c] sequential run status will be change + NOTICE // [s-c] Notice + } + + // these messages will be ignored during the sequential run of the note + private static final Set disabledForRunningNoteMessages = Collections + .unmodifiableSet(new HashSet<>(Arrays.asList( + OP.COMMIT_PARAGRAPH, + OP.RUN_PARAGRAPH, + OP.RUN_PARAGRAPH_USING_SPELL, + OP.RUN_ALL_PARAGRAPHS, + OP.PARAGRAPH_CLEAR_OUTPUT, + OP.PARAGRAPH_CLEAR_ALL_OUTPUT, + OP.INSERT_PARAGRAPH, + OP.MOVE_PARAGRAPH, + OP.COPY_PARAGRAPH, + OP.PARAGRAPH_REMOVE, + OP.MOVE_NOTE_TO_TRASH, + OP.DEL_NOTE, + OP.PATCH_PARAGRAPH))); + + private static final Gson gson = new Gson(); + public static final Message EMPTY = new Message(null); + + public OP op; + public Map data = new HashMap<>(); + public String ticket = "anonymous"; + public String principal = "anonymous"; + public String roles = ""; + + // Unique id generated from client side. to identify message. + // When message from server is response to the client request + // includes the msgId in response message, client can pair request and response message. + // When server send message that is not response to the client request, set null; + public String msgId = MSG_ID_NOT_DEFINED; + public static String MSG_ID_NOT_DEFINED = null; + + public Message(OP op) { + this.op = op; + } + + public Message withMsgId(String msgId) { + this.msgId = msgId; + return this; + } + + public Message put(String k, Object v) { + data.put(k, v); + return this; + } + + public Object get(String k) { + return data.get(k); + } + + public static boolean isDisabledForRunningNotes(OP eventType) { + return disabledForRunningNoteMessages.contains(eventType); + } + + public T getType(String key) { + return (T) data.get(key); + } + + public T getType(String key, Logger LOG) { + try { + return getType(key); + } catch (ClassCastException e) { + LOG.error("Failed to get " + key + " from message (Invalid type). " , e); + return null; + } + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Message{"); + sb.append("data=").append(data); + sb.append(", op=").append(op); + sb.append('}'); + return sb.toString(); + } + + public String toJson() { + return gson.toJson(this); + } + + public static Message fromJson(String json) { + return gson.fromJson(json, Message.class); + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index d9b3027aa88..76715977a45 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -18,7 +18,6 @@ package org.apache.zeppelin.client; import org.apache.commons.lang3.StringUtils; -import org.apache.zeppelin.notebook.socket.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java index 623666b5f43..c7c99fdcc6e 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java @@ -18,8 +18,6 @@ package org.apache.zeppelin.client; import com.google.gson.Gson; -import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.listener.ZeppelinhubWebsocket; -import org.apache.zeppelin.notebook.socket.Message; import org.eclipse.jetty.websocket.api.Session; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; @@ -38,7 +36,7 @@ @WebSocket(maxTextMessageSize = 64 * 1024) public class ZeppelinWebSocketClient { - private static final Logger LOGGER = LoggerFactory.getLogger(ZeppelinhubWebsocket.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ZeppelinWebSocketClient.class); private static final Gson GSON = new Gson(); private CountDownLatch connectLatch = new CountDownLatch(1); From 5ca49953a3afb8d391f2336853103ecc5634995c Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Thu, 20 Aug 2020 23:37:51 +0800 Subject: [PATCH 03/19] update --- .../client/AbstractMessageHandler.java | 23 +++++++++++++++---- .../zeppelin/client/MessageHandler.java | 3 +++ .../apache/zeppelin/client/NoteResult.java | 3 +++ .../zeppelin/client/ParagraphResult.java | 5 ++++ .../org/apache/zeppelin/client/Result.java | 4 ++++ .../zeppelin/client/SimpleMessageHandler.java | 3 +++ .../org/apache/zeppelin/client/ZSession.java | 11 ++++++++- .../client/ZeppelinWebSocketClient.java | 5 ++++ 8 files changed, 51 insertions(+), 6 deletions(-) diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java index 469e0ad42cc..444d66068c1 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java @@ -21,6 +21,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Abstract class of MessageHandler. + */ public abstract class AbstractMessageHandler implements MessageHandler { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMessageHandler.class); @@ -37,11 +40,7 @@ public void onMessage(String msg) { ", RECEIVE ROLES: " + messageReceived.roles + ", RECEIVE DATA: " + messageReceived.data); } - if (LOGGER.isTraceEnabled()) { - LOGGER.trace("RECEIVE MSG = " + messageReceived); - } - // Lets be elegant here switch (messageReceived.op) { case PARAGRAPH_UPDATE_OUTPUT: String noteId = (String) messageReceived.data.get("noteId"); @@ -64,11 +63,25 @@ public void onMessage(String msg) { } catch (Exception e) { LOGGER.error("Can't handle message: " + msg, e); } - } + /** + * Invoked when there's new statement output appended. + * + * @param statementId + * @param index + * @param output + */ public abstract void onStatementAppendOutput(String statementId, int index, String output); + /** + * Invoked when statement's output is updated. + * + * @param statementId + * @param index + * @param type + * @param output + */ public abstract void onStatementUpdateOutput(String statementId, int index, String type, String output); } diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java index b52737e80f7..e7362386e3f 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java @@ -18,6 +18,9 @@ package org.apache.zeppelin.client; +/** + * Interface of how to handle websocket message sent from ZeppelinServer. + */ public interface MessageHandler { void onMessage(String msg); diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java index e6a5a363af3..8257498c0af 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java @@ -20,6 +20,9 @@ import java.util.List; +/** + * + */ public class NoteResult { private String noteId; private boolean isRunning; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java index 95ee88eceda..0f305b591dc 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java @@ -25,6 +25,11 @@ import java.util.ArrayList; import java.util.List; + +/** + * Represent the paragraph execution result. + * + */ public class ParagraphResult { private String paragraphId; private Status status; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java index d965fb5a0b6..4df2a908532 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java @@ -20,6 +20,10 @@ import kong.unirest.json.JSONObject; +/** + * Represent one segment of result of paragraph. The result of paragraph could consists of + * multiple Results. + */ public class Result { private String type; private String data; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java index 258f869c40e..d37b4d75949 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java @@ -20,6 +20,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Simple implementation of AbstractMessageHandler which only print output. + */ public class SimpleMessageHandler extends AbstractMessageHandler { private static final Logger LOGGER = LoggerFactory.getLogger(SimpleMessageHandler.class); diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index 76715977a45..0ad21b2c144 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -35,6 +35,7 @@ public class ZSession { private ZeppelinClient zeppelinClient; private String interpreter; private Map intpProperties; + // max number of retained statements, each statement represent one paragraph. private int maxStatement; private String sessionId; @@ -58,14 +59,22 @@ public ZSession(ClientConfig clientConfig, this.maxStatement = maxStatement; } + /** + * Start this ZSession, underneath it would create a note for this ZSession and + * start a dedicated interpreter group. + * This method won't establish websocket connection. + * + * @throws Exception + */ public void start() throws Exception { start(null); } /** * Start this ZSession, underneath it would create a note for this ZSession and - * start a dedicated interpreter group. + * start a dedicated interpreter process. * + * @param messageHandler * @throws Exception */ public void start(MessageHandler messageHandler) throws Exception { diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java index c7c99fdcc6e..747374817fb 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java @@ -34,6 +34,11 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; + +/** + * Represent websocket client. + * + */ @WebSocket(maxTextMessageSize = 64 * 1024) public class ZeppelinWebSocketClient { private static final Logger LOGGER = LoggerFactory.getLogger(ZeppelinWebSocketClient.class); From d4bbe9522ad89e8a5fec8a5cb685a8170b11e5eb Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Fri, 21 Aug 2020 12:56:14 +0800 Subject: [PATCH 04/19] add zeppelin client examples --- pom.xml | 1 + zeppelin-client-examples/pom.xml | 89 +++++++++++++++ .../client/examples/FlinkAdvancedExample.java | 107 +++++++++++++++++ .../examples/FlinkAdvancedExample2.java | 104 +++++++++++++++++ .../client/examples/FlinkExample.java | 99 ++++++++++++++++ .../client/examples/SparkAdvancedExample.java | 99 ++++++++++++++++ .../client/examples/SparkExample.java | 108 ++++++++++++++++++ .../src/main/resources/init_stream.scala | 46 ++++++++ .../client/CompositeMessageHandler.java | 59 ++++++++++ .../apache/zeppelin/client/ExecuteResult.java | 5 +- .../org/apache/zeppelin/client/Message.java | 1 + .../apache/zeppelin/client/NoteResult.java | 2 +- .../client/ParagraphResultCollector.java | 64 ----------- ...dler.java => StatementMessageHandler.java} | 47 ++++---- .../org/apache/zeppelin/client/ZSession.java | 104 ++++++++++++++++- .../client/ZeppelinWebSocketClient.java | 31 +++-- .../src/test/resources/zeppelin-site.xml | 2 +- 17 files changed, 865 insertions(+), 103 deletions(-) create mode 100644 zeppelin-client-examples/pom.xml create mode 100644 zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java create mode 100644 zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java create mode 100644 zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkExample.java create mode 100644 zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java create mode 100644 zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkExample.java create mode 100644 zeppelin-client-examples/src/main/resources/init_stream.scala create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/CompositeMessageHandler.java delete mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResultCollector.java rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{FlinkMessageHandler.java => StatementMessageHandler.java} (55%) diff --git a/pom.xml b/pom.xml index 2cbc42a6ec5..78f66de9fdb 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,7 @@ ksql sparql zeppelin-client + zeppelin-client-examples zeppelin-web zeppelin-server zeppelin-jupyter diff --git a/zeppelin-client-examples/pom.xml b/zeppelin-client-examples/pom.xml new file mode 100644 index 00000000000..357aa6daad4 --- /dev/null +++ b/zeppelin-client-examples/pom.xml @@ -0,0 +1,89 @@ + + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.9.0-SNAPSHOT + ../pom.xml + + + org.apache.zeppelin + zeppelin-client-examples + jar + 0.9.0-SNAPSHOT + Zeppelin: Client Examples + Zeppelin Client Examples + + + + + org.apache.zeppelin + zeppelin-client + ${project.version} + + + + commons-io + commons-io + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + junit + junit + test + + + + org.mockito + mockito-all + test + + + + + + + src/main/resources + true + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + + diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java new file mode 100644 index 00000000000..2369f39d40b --- /dev/null +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client.examples; + +import org.apache.commons.io.IOUtils; +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.ExecuteResult; +import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.ZSession; + +import java.util.HashMap; +import java.util.Map; + +public class FlinkAdvancedExample { + public static void main(String[] args) { + + ZSession session = null; + try { + ClientConfig clientConfig = new ClientConfig("http://localhost:8080"); + Map intpProperties = new HashMap<>(); + + session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("flink") + .setIntpProperties(intpProperties) + .build(); + + // if MessageHandler is specified, then websocket is enabled. + // you can get continuous output from Zeppelin via websocket. + session.start(new SimpleMessageHandler()); + System.out.println("Flink Web UI: " + session.getWeburl()); + + String code = "benv.fromElements(1,2,3,4,5,6,7,8,9,10).map(e=> {Thread.sleep(1000); e}).print()"; + System.out.println("Submit code: " + code); + // use submit to run flink code in non-blocking way. + ExecuteResult result = session.submit(code); + System.out.println("Job status: " + result.getStatus()); + while(!result.getStatus().isCompleted()) { + result = session.queryStatement(result.getStatementId()); + System.out.println("Job status: " + result.getStatus() + ", progress: " + result.getProgress()); + Thread.sleep(1000); + } + System.out.println("Job status: " + result.getStatus() + ", data: " + result.getResults().get(0).getData()); + + System.out.println("-----------------------------------------------------------------------------"); + System.out.println("Submit code: " + code); + result = session.submit("benv.fromElements(1,2,3,4,5,6,7,8,9,10).map(e=> {Thread.sleep(1000); e}).print()"); + System.out.println("Job status: " + result.getStatus()); + result = session.waitUntilFinished(result.getStatementId()); + System.out.println("Job status: " + result.getStatus() + ", data: " + result.getResults().get(0).getData()); + + System.out.println("-----------------------------------------------------------------------------"); + code = "for(i <- 1 to 10) {\n" + + " Thread.sleep(1000)\n" + + " println(i)\n" + + "}"; + System.out.println("Submit code: " + code); + result = session.execute(code); + System.out.println("Job status: " + result.getStatus() + ", data: " + result.getResults().get(0).getData()); + + System.out.println("-----------------------------------------------------------------------------"); + String initCode = IOUtils.toString(FlinkAdvancedExample.class.getResource("/init_stream.scala")); + result = session.execute(initCode); + System.out.println("Job status: " + result.getStatus() + ", data: " + result.getResults().get(0).getData()); + + // run flink ssql + Map localProperties = new HashMap<>(); + localProperties.put("type", "update"); + result = session.submit("ssql", localProperties, "select url, count(1) as pv from log group by url"); + session.waitUntilFinished(result.getStatementId()); + + result = session.submit("ssql", localProperties, "select url, count(1) as pv from log group by url"); + session.waitUntilRunning(result.getStatementId()); + Thread.sleep(10 * 1000); + System.out.println("Try to cancel statement: " + result.getStatementId()); + session.cancel(result.getStatementId()); + session.waitUntilFinished(result.getStatementId()); + System.out.println("Job status: " + result.getStatus()); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (session != null) { + try { + session.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } +} diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java new file mode 100644 index 00000000000..0787d61789f --- /dev/null +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.client.examples; + +import org.apache.commons.io.IOUtils; +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.CompositeMessageHandler; +import org.apache.zeppelin.client.ExecuteResult; +import org.apache.zeppelin.client.StatementMessageHandler; +import org.apache.zeppelin.client.ZSession; + +import java.util.HashMap; +import java.util.Map; + +public class FlinkAdvancedExample2 { + public static void main(String[] args) { + + ZSession session = null; + try { + ClientConfig clientConfig = new ClientConfig("http://localhost:8080"); + Map intpProperties = new HashMap<>(); + + session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("flink") + .setIntpProperties(intpProperties) + .build(); + + // CompositeMessageHandler allow you to add StatementMessageHandler for each statement. + // otherwise you have to use a global MessageHandler. + session.start(new CompositeMessageHandler()); + System.out.println("Flink Web UI: " + session.getWeburl()); + + System.out.println("-----------------------------------------------------------------------------"); + String initCode = IOUtils.toString(FlinkAdvancedExample.class.getResource("/init_stream.scala")); + ExecuteResult result = session.execute(initCode); + System.out.println("Job status: " + result.getStatus() + ", data: " + result.getResults().get(0).getData()); + + // run flink ssql + Map localProperties = new HashMap<>(); + localProperties.put("type", "update"); + result = session.submit("ssql", localProperties, "select url, count(1) as pv from log group by url", + new MyStatementMessageHandler1()); + session.waitUntilFinished(result.getStatementId()); + + result = session.submit("ssql", localProperties, "select upper(url), count(1) as pv from log group by url", + new MyStatementMessageHandler2()); + session.waitUntilFinished(result.getStatementId()); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (session != null) { + try { + session.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + + public static class MyStatementMessageHandler1 implements StatementMessageHandler { + + @Override + public void onStatementAppendOutput(String statementId, int index, String output) { + System.out.println("MyStatementMessageHandler1, append output: " + output); + } + + @Override + public void onStatementUpdateOutput(String statementId, int index, String type, String output) { + System.out.println("MyStatementMessageHandler1, update output: " + output); + } + } + + public static class MyStatementMessageHandler2 implements StatementMessageHandler { + + @Override + public void onStatementAppendOutput(String statementId, int index, String output) { + System.out.println("MyStatementMessageHandler2, append output: " + output); + } + + @Override + public void onStatementUpdateOutput(String statementId, int index, String type, String output) { + System.out.println("MyStatementMessageHandler2, update output: " + output); + } + } +} diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkExample.java new file mode 100644 index 00000000000..7bea74b7a7d --- /dev/null +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkExample.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client.examples; + +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.ExecuteResult; +import org.apache.zeppelin.client.ZSession; + +import java.util.HashMap; +import java.util.Map; + +public class FlinkExample { + public static void main(String[] args) { + + ZSession session = null; + try { + ClientConfig clientConfig = new ClientConfig("http://localhost:8080"); + Map intpProperties = new HashMap<>(); + + session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("flink") + .setIntpProperties(intpProperties) + .build(); + + session.start(); + System.out.println("Flink Web UI: " + session.getWeburl()); + + // scala (single result) + ExecuteResult result = session.execute("benv.fromElements(1,2,3).print()"); + System.out.println("Result: " + result.getResults().get(0).getData()); + + // scala (multiple result) + result = session.execute("val data = benv.fromElements(1,2,3).map(e=>(e, e * 2))\n" + + "data.print()\n" + + "z.show(data)"); + + // The first result is text output + System.out.println("Result 1: type: " + result.getResults().get(0).getType() + + ", data: " + result.getResults().get(0).getData() ); + // The second result is table output + System.out.println("Result 2: type: " + result.getResults().get(1).getType() + + ", data: " + result.getResults().get(1).getData() ); + System.out.println("Flink Job Urls:\n" + StringUtils.join(result.getJobUrls(), "\n")); + + // error output + result = session.execute("1/0"); + System.out.println("Result status: " + result.getStatus() + + ", data: " + result.getResults().get(0).getData()); + + // pyflink + result = session.execute("pyflink", "type(b_env)"); + System.out.println("benv: " + result.getResults().get(0).getData()); + // matplotlib + result = session.execute("ipyflink", "%matplotlib inline\n" + + "import matplotlib.pyplot as plt\n" + + "plt.plot([1,2,3,4])\n" + + "plt.ylabel('some numbers')\n" + + "plt.show()"); + System.out.println("Matplotlib result, type: " + result.getResults().get(0).getType() + + ", data: " + result.getResults().get(0).getData()); + + // flink sql + result = session.execute("ssql", "show tables"); + System.out.println("Flink tables: " + result.getResults().get(0).getData()); + + // flink invalid sql + result = session.execute("bsql", "select * from unknown_table"); + System.out.println("Result status: " + result.getStatus() + + ", data: " + result.getResults().get(0).getData()); + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (session != null) { + try { + session.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } +} diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java new file mode 100644 index 00000000000..d053b6787f1 --- /dev/null +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client.examples; + +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.ExecuteResult; +import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.ZSession; + +import java.util.HashMap; +import java.util.Map; + +public class SparkAdvancedExample { + + public static void main(String[] args) { + + ZSession session = null; + try { + ClientConfig clientConfig = new ClientConfig("http://localhost:8080"); + Map intpProperties = new HashMap<>(); + intpProperties.put("spark.master", "local[*]"); + + session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("spark") + .setIntpProperties(intpProperties) + .build(); + + // if MessageHandler is specified, then websocket is enabled. + // you can get continuous output from Zeppelin via websocket. + session.start(new SimpleMessageHandler()); + System.out.println("Spark Web UI: " + session.getWeburl()); + + String code = "sc.range(1,10).map(e=> {Thread.sleep(2000); e}).sum()"; + System.out.println("Submit code: " + code); + // use submit to run spark code in non-blocking way. + ExecuteResult result = session.submit(code); + System.out.println("Job status: " + result.getStatus()); + while(!result.getStatus().isCompleted()) { + result = session.queryStatement(result.getStatementId()); + System.out.println("Job status: " + result.getStatus() + ", progress: " + result.getProgress()); + Thread.sleep(1000); + } + System.out.println("Job status: " + result.getStatus() + ", data: " + result.getResults().get(0).getData()); + + System.out.println("-----------------------------------------------------------------------------"); + System.out.println("Submit code: " + code); + result = session.submit("sc.range(1,10).map(e=> {Thread.sleep(2000); e}).sum()"); + System.out.println("Job status: " + result.getStatus()); + result = session.waitUntilFinished(result.getStatementId()); + System.out.println("Job status: " + result.getStatus() + ", data: " + result.getResults().get(0).getData()); + + System.out.println("-----------------------------------------------------------------------------"); + System.out.println("Submit code: " + code); + result = session.submit("sc.range(1,10).map(e=> {Thread.sleep(2000); e}).sum()"); + System.out.println("Job status: " + result.getStatus()); + session.waitUntilRunning(result.getStatementId()); + System.out.println("Try to cancel statement: " + result.getStatementId()); + session.cancel(result.getStatementId()); + result = session.waitUntilFinished(result.getStatementId()); + System.out.println("Job status: " + result.getStatus() + ", data: " + result.getResults().get(0).getData()); + + System.out.println("-----------------------------------------------------------------------------"); + code = "for(i <- 1 to 10) {\n" + + " Thread.sleep(1000)\n" + + " println(i)\n" + + "}"; + System.out.println("Submit code: " + code); + result = session.execute(code); + System.out.println("Job status: " + result.getStatus() + ", data: " + result.getResults().get(0).getData()); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (session != null) { + try { + session.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } +} diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkExample.java new file mode 100644 index 00000000000..462d1f4e432 --- /dev/null +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkExample.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client.examples; + +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.ExecuteResult; +import org.apache.zeppelin.client.ZSession; + +import java.util.HashMap; +import java.util.Map; + +public class SparkExample { + + public static void main(String[] args) { + + ZSession session = null; + try { + ClientConfig clientConfig = new ClientConfig("http://localhost:8080"); + Map intpProperties = new HashMap<>(); + intpProperties.put("spark.master", "local[*]"); + + session = ZSession.builder() + .setClientConfig(clientConfig) + .setInterpreter("spark") + .setIntpProperties(intpProperties) + .build(); + + session.start(); + System.out.println("Spark Web UI: " + session.getWeburl()); + + // scala (single result) + ExecuteResult result = session.execute("println(sc.version)"); + System.out.println("Spark Version: " + result.getResults().get(0).getData()); + + // scala (multiple result) + result = session.execute("println(sc.version)\n" + + "val df = spark.createDataFrame(Seq((1,\"a\"), (2,\"b\")))\n" + + "z.show(df)"); + + // The first result is text output + System.out.println("Result 1: type: " + result.getResults().get(0).getType() + + ", data: " + result.getResults().get(0).getData() ); + // The second result is table output + System.out.println("Result 2: type: " + result.getResults().get(1).getType() + + ", data: " + result.getResults().get(1).getData() ); + System.out.println("Spark Job Urls:\n" + StringUtils.join(result.getJobUrls(), "\n")); + + // error output + result = session.execute("1/0"); + System.out.println("Result status: " + result.getStatus() + + ", data: " + result.getResults().get(0).getData()); + + // pyspark + result = session.execute("pyspark", "df = spark.createDataFrame([(1,'a'),(2,'b')])\n" + + "df.registerTempTable('df')\n" + + "df.show()"); + System.out.println("PySpark dataframe: " + result.getResults().get(0).getData()); + + // matplotlib + result = session.execute("ipyspark", "%matplotlib inline\n" + + "import matplotlib.pyplot as plt\n" + + "plt.plot([1,2,3,4])\n" + + "plt.ylabel('some numbers')\n" + + "plt.show()"); + System.out.println("Matplotlib result, type: " + result.getResults().get(0).getType() + + ", data: " + result.getResults().get(0).getData()); + + // sparkr + result = session.execute("r", "df <- as.DataFrame(faithful)\nhead(df)"); + System.out.println("Sparkr dataframe: " + result.getResults().get(0).getData()); + + // spark sql + result = session.execute("sql", "select * from df"); + System.out.println("Spark Sql dataframe: " + result.getResults().get(0).getData()); + + // spark invalid sql + result = session.execute("sql", "select * from unknown_table"); + System.out.println("Result status: " + result.getStatus() + + ", data: " + result.getResults().get(0).getData()); + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (session != null) { + try { + session.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } +} diff --git a/zeppelin-client-examples/src/main/resources/init_stream.scala b/zeppelin-client-examples/src/main/resources/init_stream.scala new file mode 100644 index 00000000000..c59a8f588ea --- /dev/null +++ b/zeppelin-client-examples/src/main/resources/init_stream.scala @@ -0,0 +1,46 @@ +import org.apache.flink.streaming.api.functions.source.SourceFunction +import org.apache.flink.table.api.TableEnvironment +import org.apache.flink.streaming.api.TimeCharacteristic +import org.apache.flink.streaming.api.checkpoint.ListCheckpointed +import java.util.Collections +import scala.collection.JavaConversions._ + +senv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) +senv.enableCheckpointing(5000) + +val data = senv.addSource(new SourceFunction[(Long, String)] with ListCheckpointed[java.lang.Long] { + + val pages = Seq("home", "search", "search", "product", "product", "product") + var count: Long = 0 + var running : Boolean = true + // startTime is 2018/1/1 + var startTime: Long = new java.util.Date(2018 - 1900,0,1).getTime + var sleepInterval = 500 + + override def run(ctx: SourceFunction.SourceContext[(Long, String)]): Unit = { + val lock = ctx.getCheckpointLock + + while (count < 60 && running) { + lock.synchronized({ + ctx.collect((startTime + count * sleepInterval, pages(count.toInt % pages.size))) + count += 1 + Thread.sleep(sleepInterval) + }) + } + } + + override def cancel(): Unit = { + running = false + } + + override def snapshotState(checkpointId: Long, timestamp: Long): java.util.List[java.lang.Long] = { + Collections.singletonList(count) + } + + override def restoreState(state: java.util.List[java.lang.Long]): Unit = { + state.foreach(s => count = s) + } + +}).assignAscendingTimestamps(_._1) + +stenv.registerDataStream("log", data, 'time, 'url, 'rowtime.rowtime) diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/CompositeMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/CompositeMessageHandler.java new file mode 100644 index 00000000000..c902c823b40 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/CompositeMessageHandler.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.client; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * You can add StatementHandler for each statement. + */ +public class CompositeMessageHandler extends AbstractMessageHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(CompositeMessageHandler.class); + + private Map messageHandlers = new HashMap(); + + public void addStatementMessageHandler(String statementId, + StatementMessageHandler messageHandler) { + messageHandlers.put(statementId, messageHandler); + } + + @Override + public void onStatementAppendOutput(String statementId, int index, String output) { + StatementMessageHandler messageHandler = messageHandlers.get(statementId); + if (messageHandler == null) { + LOGGER.warn("No messagehandler for statement: " + statementId); + return; + } + messageHandler.onStatementAppendOutput(statementId, index, output); + } + + @Override + public void onStatementUpdateOutput(String statementId, int index, String type, String output) { + StatementMessageHandler messageHandler = messageHandlers.get(statementId); + if (messageHandler == null) { + LOGGER.warn("No messagehandler for statement: " + statementId); + return; + } + messageHandler.onStatementUpdateOutput(statementId, index, type, output); + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java index 2ebf11aaa5b..8ad9e8c26d4 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java @@ -21,7 +21,6 @@ import org.apache.commons.lang3.StringUtils; import java.util.List; -import java.util.Map; /** * Execution result of each statement. @@ -60,6 +59,10 @@ public List getJobUrls() { return jobUrls; } + public int getProgress() { + return progress; + } + @Override public String toString() { return "ExecuteResult{" + diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java index 865b7541c21..a5901527d25 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java @@ -28,6 +28,7 @@ import java.util.Set; /** + * Copied from zeppelin-zengine (TODO, zjffdu). Should resume the same piece of code instead of copying. * Zeppelin websocket massage template class. */ public class Message implements JsonSerializable { diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java index 8257498c0af..ededfdcffe0 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/NoteResult.java @@ -21,7 +21,7 @@ import java.util.List; /** - * + * Represent the note execution result. */ public class NoteResult { private String noteId; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResultCollector.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResultCollector.java deleted file mode 100644 index 4238a3e5f8b..00000000000 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResultCollector.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.client; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -public class ParagraphResultCollector { - - private static final Logger LOGGER = LoggerFactory.getLogger(ParagraphResultCollector.class); - - private String noteId; - private String paragraphId; - private List results = new ArrayList<>(); - - public void onOutputAppend(String noteId, - String paragraphId, - int index, - String output) { - if (index > (results.size() - 1)) { - LOGGER.warn("Get output append for index: " + index + - ", but currently there's only " + results.size() + " results"); - } - results.get(index).appendData(output); - } - - - public void onOutputUpdated(String noteId, - String paragraphId, - int index, - String type, - String output) { - Result result = new Result(type, output); - if (index < results.size()) { - results.set(index, result); - } else { - results.add(result); - } - } - - public void onOutputClear(String noteId, - String paragraphId) { - - } - -} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/FlinkMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/StatementMessageHandler.java similarity index 55% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/FlinkMessageHandler.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/StatementMessageHandler.java index bd1f105eadc..9a59db4a4ad 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/FlinkMessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/StatementMessageHandler.java @@ -15,31 +15,30 @@ * limitations under the License. */ - package org.apache.zeppelin.client; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Map; - -public class FlinkMessageHandler extends AbstractMessageHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(FlinkMessageHandler.class); - - private Map resultMap = new HashMap<>(); - - @Override - public void onStatementAppendOutput(String statementId, int index, String output) { - - } +/** + * MessageHandler for each statement. + */ +public interface StatementMessageHandler { + + /** + * Invoked when there's new statement output appended. + * + * @param statementId + * @param index + * @param output + */ + void onStatementAppendOutput(String statementId, int index, String output); + + /** + * Invoked when statement's output is updated. + * + * @param statementId + * @param index + * @param type + * @param output + */ + void onStatementUpdateOutput(String statementId, int index, String type, String output); - @Override - public void onStatementUpdateOutput(String statementId, int index, String type, String output) { - if (resultMap.containsKey(statementId)) { -// LOGGER.info("result") - } - resultMap.put(statementId, new Result(type, output)); - } } diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index 0ad21b2c144..7903a7bde2e 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -123,9 +123,13 @@ public void stop() throws Exception { if (sessionId != null) { zeppelinClient.stopSession(interpreter, sessionId); } + if (webSocketClient != null) { + webSocketClient.stop(); + } } /** + * Run code in non-blocking way. * * @param code * @return @@ -135,6 +139,18 @@ public ExecuteResult execute(String code) throws Exception { return execute("", code); } + /** + * Run code in non-blocking way. + * + * @param code + * @param messageHandler + * @return + * @throws Exception + */ + public ExecuteResult execute(String code, StatementMessageHandler messageHandler) throws Exception { + return execute("", code, messageHandler); + } + /** * * @param subInterpreter @@ -146,15 +162,45 @@ public ExecuteResult execute(String subInterpreter, String code) throws Exceptio return execute(subInterpreter, new HashMap<>(), code); } + /** + * + * @param subInterpreter + * @param code + * @param messageHandler + * @return + * @throws Exception + */ + public ExecuteResult execute(String subInterpreter, String code, StatementMessageHandler messageHandler) throws Exception { + return execute(subInterpreter, new HashMap<>(), code, messageHandler); + } + + /** + * + * @param subInterpreter + * @param localProperties + * @param code + * @return + * @throws Exception + */ + public ExecuteResult execute(String subInterpreter, + Map localProperties, + String code) throws Exception { + return execute(subInterpreter, localProperties, code, null); + } + /** * * @param subInterpreter * @param localProperties * @param code + * @param messageHandler * @return * @throws Exception */ - public ExecuteResult execute(String subInterpreter, Map localProperties, String code) throws Exception { + public ExecuteResult execute(String subInterpreter, + Map localProperties, + String code, + StatementMessageHandler messageHandler) throws Exception { StringBuilder builder = new StringBuilder("%" + interpreter); if (!StringUtils.isBlank(subInterpreter)) { builder.append("." + subInterpreter); @@ -172,6 +218,10 @@ public ExecuteResult execute(String subInterpreter, Map localPro String nextParagraphId = zeppelinClient.nextSessionParagraph(noteId, maxStatement); zeppelinClient.updateParagraph(noteId, nextParagraphId, "", text); + + if (messageHandler != null) { + webSocketClient.addStatementMessageHandler(nextParagraphId, messageHandler); + } ParagraphResult paragraphResult = zeppelinClient.executeParagraph(noteId, nextParagraphId, sessionId); return new ExecuteResult(paragraphResult); } @@ -186,6 +236,17 @@ public ExecuteResult submit(String code) throws Exception { return submit("", code); } + /** + * + * @param code + * @param messageHandler + * @return + * @throws Exception + */ + public ExecuteResult submit(String code, StatementMessageHandler messageHandler) throws Exception { + return submit("", code, messageHandler); + } + /** * * @param subInterpreter @@ -197,6 +258,18 @@ public ExecuteResult submit(String subInterpreter, String code) throws Exception return submit(subInterpreter, new HashMap<>(), code); } + /** + * + * @param subInterpreter + * @param code + * @param messageHandler + * @return + * @throws Exception + */ + public ExecuteResult submit(String subInterpreter, String code, StatementMessageHandler messageHandler) throws Exception { + return submit(subInterpreter, new HashMap<>(), code, messageHandler); + } + /** * * @param subInterpreter @@ -205,6 +278,21 @@ public ExecuteResult submit(String subInterpreter, String code) throws Exception * @throws Exception */ public ExecuteResult submit(String subInterpreter, Map localProperties, String code) throws Exception { + return submit(subInterpreter, localProperties, code, null); + } + + /** + * + * @param subInterpreter + * @param code + * @param messageHandler + * @return + * @throws Exception + */ + public ExecuteResult submit(String subInterpreter, + Map localProperties, + String code, + StatementMessageHandler messageHandler) throws Exception { StringBuilder builder = new StringBuilder("%" + interpreter); if (!StringUtils.isBlank(subInterpreter)) { builder.append("." + subInterpreter); @@ -221,6 +309,9 @@ public ExecuteResult submit(String subInterpreter, Map localProp String text = builder.toString(); String nextParagraphId = zeppelinClient.nextSessionParagraph(noteId, maxStatement); zeppelinClient.updateParagraph(noteId, nextParagraphId, "", text); + if (messageHandler != null) { + webSocketClient.addStatementMessageHandler(nextParagraphId, messageHandler); + } ParagraphResult paragraphResult = zeppelinClient.submitParagraph(noteId, nextParagraphId, sessionId); return new ExecuteResult(paragraphResult); } @@ -234,6 +325,17 @@ public void cancel(String statementId) throws Exception { zeppelinClient.cancelParagraph(noteId, statementId); } + /** + * + * @param statementId + * @return + * @throws Exception + */ + public ExecuteResult queryStatement(String statementId) throws Exception { + ParagraphResult paragraphResult = zeppelinClient.queryParagraphResult(noteId, statementId); + return new ExecuteResult(paragraphResult); + } + /** * * @param statementId diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java index 747374817fb..ad8a3abe4fb 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java @@ -49,22 +49,33 @@ public class ZeppelinWebSocketClient { private Session session; private MessageHandler messageHandler; + private WebSocketClient wsClient; public ZeppelinWebSocketClient(MessageHandler messageHandler) { this.messageHandler = messageHandler; } public void connect(String url) throws Exception { - WebSocketClient client = new WebSocketClient(); - client.start(); + this.wsClient = new WebSocketClient(); + wsClient.start(); URI echoUri = new URI(url); ClientUpgradeRequest request = new ClientUpgradeRequest(); request.setHeader("Origin", "*"); - client.connect(this, echoUri, request); + wsClient.connect(this, echoUri, request); connectLatch.await(); LOGGER.info("WebSocket connect established"); } + public void addStatementMessageHandler(String statementId, + StatementMessageHandler statementMessageHandler) throws Exception { + if (messageHandler instanceof CompositeMessageHandler) { + ((CompositeMessageHandler) messageHandler).addStatementMessageHandler(statementId, statementMessageHandler); + } else { + throw new Exception("StatementMessageHandler is only supported by: " + + CompositeMessageHandler.class.getSimpleName()); + } + } + public boolean awaitClose(int duration, TimeUnit unit) throws InterruptedException { return this.closeLatch.await(duration, unit); } @@ -102,15 +113,13 @@ public CountDownLatch getConnectLatch() { return connectLatch; } - public static void main(String[] args) throws Exception { -// ZeppelinWebSocket socket = new ZeppelinWebSocket(); -// socket.connect("ws://localhost:18086/ws"); -// Message msg = new Message(Message.OP.LIST_NOTES); -// socket.send(msg); -// -// Thread.sleep(10 * 1000); -// socket.awaitClose(10, TimeUnit.SECONDS); + public void stop() throws Exception { + if (this.wsClient != null) { + this.wsClient.stop(); + } + } + public static void main(String[] args) throws Exception { String dest = "ws://localhost:18086/ws"; WebSocketClient client = new WebSocketClient(); try { diff --git a/zeppelin-server/src/test/resources/zeppelin-site.xml b/zeppelin-server/src/test/resources/zeppelin-site.xml index 32373bbf107..e46fce76600 100644 --- a/zeppelin-server/src/test/resources/zeppelin-site.xml +++ b/zeppelin-server/src/test/resources/zeppelin-site.xml @@ -142,7 +142,7 @@ zeppelin.server.allowed.origins - * + http://onehost:8080,http://otherhost.com, Allowed sources for REST and WebSocket requests. - - - - org.eclipse.jetty.websocket websocket-client ${jetty.version} - com.konghq unirest-java diff --git a/zeppelin-integration/src/test/resources/log4j.properties b/zeppelin-integration/src/test/resources/log4j.properties index 83689930c1a..c666d57493d 100644 --- a/zeppelin-integration/src/test/resources/log4j.properties +++ b/zeppelin-integration/src/test/resources/log4j.properties @@ -41,6 +41,3 @@ log4j.logger.DataNucleus.Datastore=ERROR # Log all JDBC parameters log4j.logger.org.hibernate.type=ALL - -log4j.logger.org.apache.zeppelin.interpreter=DEBUG -log4j.logger.org.apache.zeppelin.spark=DEBUG diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/AbstractInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/AbstractInterpreter.java index bb494d2ae0b..75d18858140 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/AbstractInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/AbstractInterpreter.java @@ -17,7 +17,6 @@ package org.apache.zeppelin.interpreter; -import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import java.util.ArrayList; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 2f05ad8c4ae..36ec1bd65f9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -25,7 +25,6 @@ import org.apache.zeppelin.dep.Repository; import org.apache.zeppelin.interpreter.ExecutionContextBuilder; import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterPropertyType; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.InterpreterSettingManager; @@ -57,7 +56,6 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import java.io.IOException; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java index 8f69998ca64..fd1a0b4fd61 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java @@ -22,8 +22,6 @@ import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN; import com.google.common.base.Strings; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -35,7 +33,6 @@ import java.util.Set; import javax.inject.Inject; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.display.AngularObject; @@ -59,7 +56,6 @@ import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.user.AuthenticationInfo; import org.bitbucket.cowwoc.diffmatchpatch.DiffMatchPatch; -import org.eclipse.jgit.errors.MissingObjectException; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; diff --git a/zeppelin-server/src/main/resources/log4j.properties b/zeppelin-server/src/main/resources/log4j.properties deleted file mode 100644 index 0940c7f7b1c..00000000000 --- a/zeppelin-server/src/main/resources/log4j.properties +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -log4j.rootLogger = INFO, stdout - -log4j.appender.stdout = org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout = org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n - -log4j.appender.dailyfile.DatePattern=.yyyy-MM-dd -log4j.appender.dailyfile = org.apache.log4j.DailyRollingFileAppender -log4j.appender.dailyfile.File = ${zeppelin.log.file} -log4j.appender.dailyfile.layout = org.apache.log4j.PatternLayout -log4j.appender.dailyfile.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n From 1dbc414bac394c62295e07f1b0cd54270f96bfb9 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 24 Aug 2020 11:04:55 +0800 Subject: [PATCH 07/19] move classes to websocket folder --- .../client/examples/FlinkAdvancedExample.java | 2 +- .../client/examples/FlinkAdvancedExample2.java | 4 ++-- .../apache/zeppelin/client/examples/HiveExample.java | 2 +- .../zeppelin/client/examples/PrestoExample.java | 2 +- .../zeppelin/client/examples/PythonExample.java | 2 +- .../org/apache/zeppelin/client/examples/RExample.java | 2 +- .../client/examples/SparkAdvancedExample.java | 2 +- .../java/org/apache/zeppelin/client/ZSession.java | 4 ++++ .../{ => websocket}/AbstractMessageHandler.java | 2 +- .../{ => websocket}/CompositeMessageHandler.java | 2 +- .../client/{ => websocket}/JsonSerializable.java | 2 +- .../zeppelin/client/{ => websocket}/Message.java | 2 +- .../client/{ => websocket}/MessageHandler.java | 2 +- .../client/{ => websocket}/SimpleMessageHandler.java | 2 +- .../{ => websocket}/StatementMessageHandler.java | 2 +- .../{ => websocket}/ZeppelinWebSocketClient.java | 2 +- .../zeppelin/integration/ZSessionIntegrationTest.java | 11 +---------- 17 files changed, 21 insertions(+), 26 deletions(-) rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{ => websocket}/AbstractMessageHandler.java (98%) rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{ => websocket}/CompositeMessageHandler.java (97%) rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{ => websocket}/JsonSerializable.java (95%) rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{ => websocket}/Message.java (99%) rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{ => websocket}/MessageHandler.java (95%) rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{ => websocket}/SimpleMessageHandler.java (96%) rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{ => websocket}/StatementMessageHandler.java (96%) rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{ => websocket}/ZeppelinWebSocketClient.java (99%) diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java index 2369f39d40b..0cd9e38cf54 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java @@ -20,7 +20,7 @@ import org.apache.commons.io.IOUtils; import org.apache.zeppelin.client.ClientConfig; import org.apache.zeppelin.client.ExecuteResult; -import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.websocket.SimpleMessageHandler; import org.apache.zeppelin.client.ZSession; import java.util.HashMap; diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java index 0787d61789f..272ab625da6 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java @@ -20,9 +20,9 @@ import org.apache.commons.io.IOUtils; import org.apache.zeppelin.client.ClientConfig; -import org.apache.zeppelin.client.CompositeMessageHandler; +import org.apache.zeppelin.client.websocket.CompositeMessageHandler; import org.apache.zeppelin.client.ExecuteResult; -import org.apache.zeppelin.client.StatementMessageHandler; +import org.apache.zeppelin.client.websocket.StatementMessageHandler; import org.apache.zeppelin.client.ZSession; import java.util.HashMap; diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/HiveExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/HiveExample.java index a200607369f..94adda77776 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/HiveExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/HiveExample.java @@ -19,7 +19,7 @@ import org.apache.zeppelin.client.ClientConfig; import org.apache.zeppelin.client.ExecuteResult; -import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.websocket.SimpleMessageHandler; import org.apache.zeppelin.client.ZSession; import java.util.HashMap; diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PrestoExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PrestoExample.java index 6a54cf2b8d5..dc9fd5029ed 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PrestoExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PrestoExample.java @@ -20,7 +20,7 @@ import org.apache.zeppelin.client.ClientConfig; import org.apache.zeppelin.client.ExecuteResult; -import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.websocket.SimpleMessageHandler; import org.apache.zeppelin.client.ZSession; import java.util.HashMap; diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PythonExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PythonExample.java index 54f36199093..d0e41be5f2f 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PythonExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PythonExample.java @@ -19,7 +19,7 @@ import org.apache.zeppelin.client.ClientConfig; import org.apache.zeppelin.client.ExecuteResult; -import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.websocket.SimpleMessageHandler; import org.apache.zeppelin.client.ZSession; import java.util.HashMap; diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/RExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/RExample.java index 6e7f2907d54..b2a44aea763 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/RExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/RExample.java @@ -19,7 +19,7 @@ import org.apache.zeppelin.client.ClientConfig; import org.apache.zeppelin.client.ExecuteResult; -import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.websocket.SimpleMessageHandler; import org.apache.zeppelin.client.ZSession; import java.util.HashMap; diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java index d053b6787f1..3115359d973 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java @@ -19,7 +19,7 @@ import org.apache.zeppelin.client.ClientConfig; import org.apache.zeppelin.client.ExecuteResult; -import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.websocket.SimpleMessageHandler; import org.apache.zeppelin.client.ZSession; import java.util.HashMap; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index 46909f7bdd4..ee345ceab0d 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -18,6 +18,10 @@ package org.apache.zeppelin.client; import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.client.websocket.Message; +import org.apache.zeppelin.client.websocket.MessageHandler; +import org.apache.zeppelin.client.websocket.StatementMessageHandler; +import org.apache.zeppelin.client.websocket.ZeppelinWebSocketClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/AbstractMessageHandler.java similarity index 98% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/AbstractMessageHandler.java index 444d66068c1..1aa9eeeefd3 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/AbstractMessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/AbstractMessageHandler.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.zeppelin.client; +package org.apache.zeppelin.client.websocket; import com.google.gson.Gson; import org.slf4j.Logger; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/CompositeMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/CompositeMessageHandler.java similarity index 97% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/CompositeMessageHandler.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/CompositeMessageHandler.java index c902c823b40..5f5094b77f3 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/CompositeMessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/CompositeMessageHandler.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.zeppelin.client; +package org.apache.zeppelin.client.websocket; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/JsonSerializable.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/JsonSerializable.java similarity index 95% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/JsonSerializable.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/JsonSerializable.java index 0b5b570e6d1..914243c8798 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/JsonSerializable.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/JsonSerializable.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.zeppelin.client; +package org.apache.zeppelin.client.websocket; /** * Interface for class that can be serialized to json diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/Message.java similarity index 99% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/Message.java index a5901527d25..e20d0b5b744 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Message.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/Message.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.zeppelin.client; +package org.apache.zeppelin.client.websocket; import com.google.gson.Gson; import org.slf4j.Logger; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/MessageHandler.java similarity index 95% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/MessageHandler.java index e7362386e3f..5cb8f20fed8 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/MessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/MessageHandler.java @@ -16,7 +16,7 @@ */ -package org.apache.zeppelin.client; +package org.apache.zeppelin.client.websocket; /** * Interface of how to handle websocket message sent from ZeppelinServer. diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/SimpleMessageHandler.java similarity index 96% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/SimpleMessageHandler.java index d37b4d75949..ca7b8810807 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SimpleMessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/SimpleMessageHandler.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.zeppelin.client; +package org.apache.zeppelin.client.websocket; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/StatementMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/StatementMessageHandler.java similarity index 96% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/StatementMessageHandler.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/StatementMessageHandler.java index 9a59db4a4ad..2ee9e6b60ec 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/StatementMessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/StatementMessageHandler.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.zeppelin.client; +package org.apache.zeppelin.client.websocket; /** * MessageHandler for each statement. diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/ZeppelinWebSocketClient.java similarity index 99% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/ZeppelinWebSocketClient.java index efa25465ef3..d770b3edc28 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinWebSocketClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/ZeppelinWebSocketClient.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.zeppelin.client; +package org.apache.zeppelin.client.websocket; import com.google.gson.Gson; import org.eclipse.jetty.websocket.api.Session; diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java index 0922f6dc56d..bd076b2c753 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java @@ -17,26 +17,17 @@ package org.apache.zeppelin.integration; -import com.google.common.collect.Lists; import org.apache.commons.io.IOUtils; import org.apache.zeppelin.client.ClientConfig; import org.apache.zeppelin.client.ExecuteResult; -import org.apache.zeppelin.client.SimpleMessageHandler; +import org.apache.zeppelin.client.websocket.SimpleMessageHandler; import org.apache.zeppelin.client.Status; import org.apache.zeppelin.client.ZSession; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.interpreter.ExecutionContextBuilder; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.integration.DownloadUtils; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.rest.AbstractTestRestApi; -import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.utils.TestUtils; import org.junit.AfterClass; import org.junit.BeforeClass; From c7701765acae16620993c1f20cd8380da226ac08 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 25 Aug 2020 22:24:29 +0800 Subject: [PATCH 08/19] add session status api --- .../apache/zeppelin/client/SessionResult.java | 39 ++++++++++++ .../java/org/apache/zeppelin/client/Test.java | 15 +++++ .../zeppelin/client/ZeppelinClient.java | 33 ++++++++++ .../apache/zeppelin/rest/SessionManager.java | 63 +++++++++++++++++++ .../apache/zeppelin/rest/SessionRestApi.java | 35 +++++++++-- .../apache/zeppelin/rest/message/Session.java | 38 +++++++++++ .../zeppelin/service/NotebookService.java | 2 + .../remote/RemoteInterpreterProcess.java | 10 +++ 8 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java create mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/Test.java create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java new file mode 100644 index 00000000000..2b2a8cac992 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java @@ -0,0 +1,39 @@ +package org.apache.zeppelin.client; + +import kong.unirest.json.JSONObject; + +public class SessionResult { + + private String sessionId; + private String interpreter; + private String state; + private String weburl; + private String startTime; + + public SessionResult(JSONObject sessionJson) { + this.sessionId = sessionJson.getString("sessionId"); + this.interpreter = sessionJson.getString("interpreter"); + this.state = sessionJson.getString("state"); + if (sessionJson.has("weburl")) { + this.weburl = sessionJson.getString("weburl"); + } else { + this.weburl = ""; + } + if (sessionJson.has("startTime")) { + this.startTime = sessionJson.getString("startTime"); + } else { + this.startTime = ""; + } + } + + @Override + public String toString() { + return "SessionResult{" + + "sessionId='" + sessionId + '\'' + + ", interpreter='" + interpreter + '\'' + + ", state='" + state + '\'' + + ", weburl='" + weburl + '\'' + + ", startTime='" + startTime + '\'' + + '}'; + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Test.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Test.java new file mode 100644 index 00000000000..b0ce4734f67 --- /dev/null +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Test.java @@ -0,0 +1,15 @@ +package org.apache.zeppelin.client; + +import org.apache.commons.lang3.StringUtils; + +import java.util.List; + +public class Test { + + public static void main(String[] args) throws Exception { + ClientConfig clientConfig = new ClientConfig("http://localhost:8080"); + ZeppelinClient zeppelinClient = new ZeppelinClient(clientConfig); + List sessionResultList = zeppelinClient.listSessions(); + System.out.println(StringUtils.join("\n", sessionResultList)); + } +} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java index 7e041d25f88..9519720fe5d 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.client; +import kong.unirest.GetRequest; import kong.unirest.HttpResponse; import kong.unirest.JsonNode; import kong.unirest.Unirest; @@ -180,6 +181,38 @@ public String getSessionWebUrl(String sessionId) throws Exception { } } + /** + * + * @return + * @throws Exception + */ + public List listSessions() throws Exception { + return listSessions(null); + } + + /** + * + * @param interpreter + * @return + * @throws Exception + */ + public List listSessions(String interpreter) throws Exception { + GetRequest getRequest = Unirest.get("/session"); + if (interpreter != null) { + getRequest.queryString("interpreter", interpreter); + } + HttpResponse response = getRequest.asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + JSONArray sessionJsonArray = jsonNode.getObject().getJSONArray("body"); + List sessionResults = new ArrayList<>(); + for (int i = 0; i< sessionJsonArray.length();++i) { + sessionResults.add(new SessionResult(sessionJsonArray.getJSONObject(i))); + } + return sessionResults; + } + /** * Login zeppelin with userName and password, throw exception if login fails. * diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java index 4fdae91ceff..2d056ffd5cc 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java @@ -18,13 +18,30 @@ package org.apache.zeppelin.rest; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; +import org.apache.zeppelin.interpreter.ManagedInterpreterGroup; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.rest.message.Session; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; public class SessionManager { + private static final Logger LOGGER = LoggerFactory.getLogger(SessionManager.class); + private static final int RETRY = 3; private Set sessions = new HashSet<>(); + private InterpreterSettingManager interpreterSettingManager; + + public SessionManager(InterpreterSettingManager interpreterSettingManager) { + this.interpreterSettingManager = interpreterSettingManager; + } public synchronized String newSession(String interpreter) throws Exception { int i = 0; @@ -49,4 +66,50 @@ public void removeSession(String sessionId) { this.sessions.remove(sessionId); } + public Session getSession(String sessionId) throws Exception { + InterpreterGroup interpreterGroup = this.interpreterSettingManager.getInterpreterGroupById(sessionId); + if (interpreterGroup != null) { + RemoteInterpreterProcess remoteInterpreterProcess = + ((ManagedInterpreterGroup) interpreterGroup).getRemoteInterpreterProcess(); + String state = ""; + String startTime = ""; + if (remoteInterpreterProcess == null) { + state = "Ready"; + } else if (remoteInterpreterProcess != null) { + startTime = remoteInterpreterProcess.getStartTime(); + if (remoteInterpreterProcess.isRunning()) { + state = "Running"; + } else { + state = "Stopped"; + } + } + return new Session(sessionId, + ((ManagedInterpreterGroup) interpreterGroup).getInterpreterSetting().getName(), + state, interpreterGroup.getWebUrl(), startTime); + } + LOGGER.warn("No such session: " + sessionId); + return null; + } + + public List getAllSessionStatus() throws Exception { + List sessionList = new ArrayList<>(); + for (String sessionId : sessions) { + Session status = getSession(sessionId); + if (status != null) { + sessionList.add(status); + } + } + return sessionList; + } + + public List getAllSessionStatus(String interpreterGroup) throws Exception { + List sessionList = new ArrayList<>(); + for (String sessionId : sessions) { + Session status = getSession(sessionId); + if (status != null && interpreterGroup.equals(status.getInterpreter())) { + sessionList.add(status); + } + } + return sessionList; + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java index f1b7086298d..278044801f1 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java @@ -17,11 +17,13 @@ package org.apache.zeppelin.rest; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.zeppelin.annotation.ZeppelinApi; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.Notebook; +import org.apache.zeppelin.rest.message.Session; import org.apache.zeppelin.server.JsonResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +36,10 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Response; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -56,7 +60,29 @@ public class SessionRestApi { public SessionRestApi(Notebook notebook, InterpreterSettingManager interpreterSettingManager) { this.notebook = notebook; this.interpreterSettingManager = interpreterSettingManager; - this.sessionManager = new SessionManager(); + this.sessionManager = new SessionManager(interpreterSettingManager); + } + + @GET + @Path("/") + public Response listSessions(@QueryParam("interpreter") String interpreter) { + if (StringUtils.isBlank(interpreter)) { + LOGGER.info("List all sessions"); + } else { + LOGGER.info("List all sessions for interpreter: " + interpreter); + } + try { + List sessionList = null; + if (StringUtils.isBlank(interpreter)) { + sessionList = sessionManager.getAllSessionStatus(); + } else { + sessionList = sessionManager.getAllSessionStatus(interpreter); + } + return new JsonResponse<>(Response.Status.OK, sessionList).build(); + } catch (Exception e) { + return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, + "Fail to list session", ExceptionUtils.getStackTrace(e)).build(); + } } @POST @@ -96,13 +122,10 @@ public Response stopSession(@PathParam("interpreter") String interpreter, @ZeppelinApi public Response getSession(@PathParam("sessionId") String sessionId) { try { - InterpreterGroup intpGroup = interpreterSettingManager.getInterpreterGroupById(sessionId); - if (intpGroup == null) { + Session session = sessionManager.getSession(sessionId); + if (session == null) { return new JsonResponse<>(Response.Status.NOT_FOUND).build(); } else { - Map session = new HashMap<>(); - session.put("id", sessionId); - session.put("weburl", intpGroup.getWebUrl()); return new JsonResponse<>(Response.Status.OK, "", session).build(); } } catch (Throwable e) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java new file mode 100644 index 00000000000..5e1c1e61fe1 --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java @@ -0,0 +1,38 @@ +package org.apache.zeppelin.rest.message; + +public class Session { + + private String sessionId; + private String interpreter; + private String state; + private String weburl; + private String startTime; + + public Session(String sessionId, String interpreter, String state, String weburl, String startTime) { + this.sessionId = sessionId; + this.interpreter = interpreter; + this.state = state; + this.weburl = weburl; + this.startTime = startTime; + } + + public String getSessionId() { + return sessionId; + } + + public String getState() { + return state; + } + + public String getInterpreter() { + return interpreter; + } + + public String getWeburl() { + return weburl; + } + + public String getStartTime() { + return startTime; + } +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java index fd1a0b4fd61..ba9aa022c81 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java @@ -49,6 +49,7 @@ import org.apache.zeppelin.notebook.repo.NotebookRepoWithVersionControl; import org.apache.zeppelin.notebook.scheduler.SchedulerService; import org.apache.zeppelin.notebook.socket.Message; +import org.apache.zeppelin.rest.SessionManager; import org.apache.zeppelin.rest.exception.BadRequestException; import org.apache.zeppelin.rest.exception.ForbiddenException; import org.apache.zeppelin.rest.exception.NoteNotFoundException; @@ -82,6 +83,7 @@ public class NotebookService { private Notebook notebook; private AuthorizationService authorizationService; private SchedulerService schedulerService; + private SessionManager sessionManager; @Inject public NotebookService( diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 616bfc3ad8a..209d5dccc11 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -27,6 +27,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; /** * Abstract class for interpreter process @@ -34,11 +36,14 @@ public abstract class RemoteInterpreterProcess implements InterpreterClient { private static final Logger LOGGER = LoggerFactory.getLogger(RemoteInterpreterProcess.class); private static final Gson GSON = new Gson(); + private static final SimpleDateFormat START_TIME_FORMATTER = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private int connectTimeout; protected String intpEventServerHost; protected int intpEventServerPort; private PooledRemoteClient remoteClient; + private String startTime; public RemoteInterpreterProcess(int connectTimeout, int connectionPoolSize, @@ -47,6 +52,7 @@ public RemoteInterpreterProcess(int connectTimeout, this.connectTimeout = connectTimeout; this.intpEventServerHost = intpEventServerHost; this.intpEventServerPort = intpEventServerPort; + this.startTime = START_TIME_FORMATTER.format(new Date()); this.remoteClient = new PooledRemoteClient(() -> { TSocket transport = new TSocket(getHost(), getPort()); try { @@ -63,6 +69,10 @@ public int getConnectTimeout() { return connectTimeout; } + public String getStartTime() { + return startTime; + } + public void shutdown() { if (remoteClient != null) { remoteClient.shutdown(); From fe22986ff8dedb5770ec0b24a5b662261b83faa6 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 26 Aug 2020 10:43:17 +0800 Subject: [PATCH 09/19] add sessionId to ZSession --- .../src/main/java/org/apache/zeppelin/client/ZSession.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index ee345ceab0d..e0264fa0c3e 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -371,6 +371,10 @@ public String getWeburl() { return weburl; } + public String getSessionId() { + return sessionId; + } + public static Builder builder() { return new Builder(); } From 7fbd48147d6046fdc0ee0477f33c3a69a334fbc0 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 26 Aug 2020 10:47:25 +0800 Subject: [PATCH 10/19] minor update --- .../src/main/java/org/apache/zeppelin/client/ZSession.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index e0264fa0c3e..f62dd687aac 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -375,6 +375,10 @@ public String getSessionId() { return sessionId; } + public ZeppelinClient getZeppelinClient() { + return zeppelinClient; + } + public static Builder builder() { return new Builder(); } From bc723e766997669d679acf0d2c068268e62dfd8c Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 26 Aug 2020 10:59:24 +0800 Subject: [PATCH 11/19] minor update --- .../apache/zeppelin/client/SessionResult.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java index 2b2a8cac992..516b26524e0 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java @@ -26,6 +26,26 @@ public SessionResult(JSONObject sessionJson) { } } + public String getSessionId() { + return sessionId; + } + + public String getInterpreter() { + return interpreter; + } + + public String getState() { + return state; + } + + public String getWeburl() { + return weburl; + } + + public String getStartTime() { + return startTime; + } + @Override public String toString() { return "SessionResult{" + From f5e1e2e7635878bc31d654889841f46897af8d6d Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 26 Aug 2020 13:28:05 +0800 Subject: [PATCH 12/19] add session reconnect --- .../apache/zeppelin/client/SessionResult.java | 6 ++ .../org/apache/zeppelin/client/ZSession.java | 62 ++++++++++++++++++- .../zeppelin/client/ZeppelinClient.java | 18 ++++++ .../apache/zeppelin/rest/SessionManager.java | 22 ++++++- .../apache/zeppelin/rest/SessionRestApi.java | 2 +- .../apache/zeppelin/rest/message/Session.java | 8 ++- 6 files changed, 110 insertions(+), 8 deletions(-) diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java index 516b26524e0..08bf248acd1 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java @@ -5,6 +5,7 @@ public class SessionResult { private String sessionId; + private String noteId; private String interpreter; private String state; private String weburl; @@ -12,6 +13,7 @@ public class SessionResult { public SessionResult(JSONObject sessionJson) { this.sessionId = sessionJson.getString("sessionId"); + this.noteId = sessionJson.getString("noteId"); this.interpreter = sessionJson.getString("interpreter"); this.state = sessionJson.getString("state"); if (sessionJson.has("weburl")) { @@ -30,6 +32,10 @@ public String getSessionId() { return sessionId; } + public String getNoteId() { + return noteId; + } + public String getInterpreter() { return interpreter; } diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index f62dd687aac..e62dd76e82a 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -63,6 +63,14 @@ public ZSession(ClientConfig clientConfig, this.maxStatement = maxStatement; } + private ZSession(ClientConfig clientConfig, + String interpreter, + String sessionId) throws Exception { + this.zeppelinClient = new ZeppelinClient(clientConfig); + this.interpreter = interpreter; + this.sessionId = sessionId; + } + /** * Start this ZSession, underneath it would create a note for this ZSession and * start a dedicated interpreter group. @@ -133,6 +141,53 @@ public void stop() throws Exception { } } + /** + * Session has been started in ZeppelinServer, this method is just to reconnect it. + * This method is used for connect to an existing session in ZeppelinServer, instead of + * start it from ZSession. + * @throws Exception + */ + public static ZSession createFromExistingSession(ClientConfig clientConfig, + String interpreter, + String sessionId) throws Exception { + return createFromExistingSession(clientConfig, interpreter, sessionId, null); + } + + /** + * Session has been started in ZeppelinServer, this method is just to reconnect it. + * This method is used for connect to an existing session in ZeppelinServer, instead of + * start it from ZSession. + * @throws Exception + */ + public static ZSession createFromExistingSession(ClientConfig clientConfig, + String interpreter, + String sessionId, + MessageHandler messageHandler) throws Exception { + ZSession session = new ZSession(clientConfig, interpreter, sessionId); + session.reconnect(messageHandler); + return session; + } + + private void reconnect(MessageHandler messageHandler) throws Exception { + SessionResult sessionResult = this.zeppelinClient.getSession(sessionId); + this.noteId = sessionResult.getNoteId(); + if (!sessionResult.getState().equalsIgnoreCase("Running")) { + throw new Exception("Session " + sessionId + " is not running, state: " + sessionResult.getState()); + } + this.weburl = sessionResult.getWeburl(); + + if (messageHandler != null) { + this.webSocketClient = new ZeppelinWebSocketClient(messageHandler); + this.webSocketClient.connect(zeppelinClient.getClientConfig().getZeppelinRestUrl() + .replace("https", "ws").replace("http", "ws") + "/ws"); + + // call GET_NOTE to establish websocket connection between this session and zeppelin-server + Message msg = new Message(Message.OP.GET_NOTE); + msg.put("id", this.noteId); + this.webSocketClient.send(msg); + } + } + /** * Run code in non-blocking way. * @@ -411,7 +466,7 @@ public ZSession build() throws Exception { public static void main(String[] args) throws Exception { - ClientConfig clientConfig = new ClientConfig("http://localhost:18086", 1000); + ClientConfig clientConfig = new ClientConfig("http://localhost:8080", 1000); // ZSession hiveSession = new ZSession(clientConfig, "hive", new HashMap<>(), 100); // hiveSession.start(); // @@ -423,8 +478,9 @@ public static void main(String[] args) throws Exception { // executeResult = hiveSession.waitUntilFinished(executeResult.getStatementId()); // System.out.println(executeResult.toString()); - ZSession sparkSession = new ZSession(clientConfig, "sh", new HashMap<>(), 100); - sparkSession.start(); + ZSession sparkSession = ZSession.createFromExistingSession(clientConfig, "hive", "hive_1598418780469"); + ExecuteResult executeResult = sparkSession.execute("show databases"); + System.out.println(executeResult); // ExecuteResult executeResult = sparkSession.submit("sql", "show tables"); // executeResult = sparkSession.waitUntilFinished(executeResult.getStatementId()); diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java index 9519720fe5d..874272da6c8 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java @@ -157,6 +157,24 @@ public void stopSession(String interpreter, String sessionId) throws Exception { checkJsonNodeStatus(jsonNode); } + /** + * + * @param sessionId + * @throws Exception + */ + public SessionResult getSession(String sessionId) throws Exception { + HttpResponse response = Unirest + .get("/session/{sessionId}") + .routeParam("sessionId", sessionId) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + + JSONObject bodyObject = jsonNode.getObject().getJSONObject("body"); + return new SessionResult(bodyObject); + } + /** * Get the session weburl. It is spark ui url for spark interpreter, * or flink web ui for flink interpreter, or may be null for the interpreter that has no weburl. diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java index 2d056ffd5cc..78d8c873b6b 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java @@ -22,6 +22,8 @@ import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.ManagedInterpreterGroup; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.notebook.NoteInfo; +import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.rest.message.Session; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +32,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; public class SessionManager { @@ -38,8 +41,10 @@ public class SessionManager { private static final int RETRY = 3; private Set sessions = new HashSet<>(); private InterpreterSettingManager interpreterSettingManager; + private Notebook notebook; - public SessionManager(InterpreterSettingManager interpreterSettingManager) { + public SessionManager(Notebook notebook, InterpreterSettingManager interpreterSettingManager) { + this.notebook = notebook; this.interpreterSettingManager = interpreterSettingManager; } @@ -83,8 +88,19 @@ public Session getSession(String sessionId) throws Exception { state = "Stopped"; } } - return new Session(sessionId, - ((ManagedInterpreterGroup) interpreterGroup).getInterpreterSetting().getName(), + String noteId = ""; + String interpreter = ((ManagedInterpreterGroup) interpreterGroup).getInterpreterSetting().getName(); + String notePath = "/_ZSession/" + interpreter + "/" + sessionId; + List notesInfo = notebook.getNotesInfo().stream() + .filter(e -> e.getPath().equals(notePath)) + .collect(Collectors.toList()); + if (notesInfo.size() != 0) { + noteId = notesInfo.get(0).getId(); + if (notesInfo.size() > 1) { + LOGGER.warn("Found more than 1 notes with path: " + notePath); + } + } + return new Session(sessionId, noteId, interpreter, state, interpreterGroup.getWebUrl(), startTime); } LOGGER.warn("No such session: " + sessionId); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java index 278044801f1..dd906829919 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java @@ -60,7 +60,7 @@ public class SessionRestApi { public SessionRestApi(Notebook notebook, InterpreterSettingManager interpreterSettingManager) { this.notebook = notebook; this.interpreterSettingManager = interpreterSettingManager; - this.sessionManager = new SessionManager(interpreterSettingManager); + this.sessionManager = new SessionManager(notebook, interpreterSettingManager); } @GET diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java index 5e1c1e61fe1..8ac7bd7779f 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java @@ -3,13 +3,15 @@ public class Session { private String sessionId; + private String noteId; private String interpreter; private String state; private String weburl; private String startTime; - public Session(String sessionId, String interpreter, String state, String weburl, String startTime) { + public Session(String sessionId, String noteId, String interpreter, String state, String weburl, String startTime) { this.sessionId = sessionId; + this.noteId = noteId; this.interpreter = interpreter; this.state = state; this.weburl = weburl; @@ -20,6 +22,10 @@ public String getSessionId() { return sessionId; } + public String getNoteId() { + return noteId; + } + public String getState() { return state; } From d7d11f77c8ee808d27cea2857c6b0c97b696d15d Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 26 Aug 2020 14:10:25 +0800 Subject: [PATCH 13/19] minor update --- .../src/main/java/org/apache/zeppelin/client/ZSession.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index e62dd76e82a..29a606b5ff5 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -430,6 +430,10 @@ public String getSessionId() { return sessionId; } + public String getInterpreter() { + return interpreter; + } + public ZeppelinClient getZeppelinClient() { return zeppelinClient; } From 487cb8c058549bb3aecf0884497d8c68ba8033df Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 26 Aug 2020 17:31:21 +0800 Subject: [PATCH 14/19] minor update --- .../zeppelin/client/ZeppelinClient.java | 23 ++----------------- .../apache/zeppelin/rest/NotebookRestApi.java | 2 +- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java index 874272da6c8..015281f16a3 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java @@ -512,27 +512,8 @@ public ParagraphResult executeParagraph(String noteId, String paragraphId, String sessionId, Map parameters) throws Exception { - JSONObject bodyObject = new JSONObject(); - bodyObject.put("params", parameters); - HttpResponse response = Unirest - .post("/notebook/run/{noteId}/{paragraphId}") - .routeParam("noteId", noteId) - .routeParam("paragraphId", paragraphId) - .queryString("sessionId", sessionId) - .body(bodyObject.toString()) - .asJson(); - checkResponse(response); - JsonNode jsonNode = response.getBody(); - checkJsonNodeStatus(jsonNode); - - jsonNode = Unirest.get( - clientConfig.getZeppelinRestUrl() + "/api/notebook/" + noteId + "/paragraph/" + paragraphId) - .asJson() - .getBody(); - checkJsonNodeStatus(jsonNode); - - JSONObject paragraphJson = jsonNode.getObject().getJSONObject("body"); - return new ParagraphResult(paragraphJson); + submitParagraph(noteId, paragraphId, sessionId, parameters); + return waitUtilParagraphFinish(noteId, paragraphId); } /** diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 915f69f90e9..75b5620e0f2 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -918,7 +918,7 @@ public Response runParagraph(@PathParam("noteId") String noteId, try { LOGGER.info("Run paragraph job asynchronously {} {} {}", noteId, paragraphId, message); - Note note = notebook.getNote(noteId); + Note note = notebook.getNote(noteId, true); checkIfNoteIsNotNull(note); Paragraph paragraph = note.getParagraph(paragraphId); checkIfParagraphIsNotNull(paragraph); From 53fe1c13ddc3b897d9051d2f46940cc29a02bfe3 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 26 Aug 2020 17:55:18 +0800 Subject: [PATCH 15/19] update examples --- .../zeppelin/client/examples/FlinkAdvancedExample.java | 4 ++++ .../zeppelin/client/examples/FlinkAdvancedExample2.java | 6 ++++++ .../org/apache/zeppelin/client/examples/FlinkExample.java | 4 ++++ .../org/apache/zeppelin/client/examples/HiveExample.java | 4 ++++ .../org/apache/zeppelin/client/examples/PrestoExample.java | 4 ++++ .../org/apache/zeppelin/client/examples/PythonExample.java | 3 +++ .../java/org/apache/zeppelin/client/examples/RExample.java | 4 ++++ .../zeppelin/client/examples/SparkAdvancedExample.java | 3 +++ .../org/apache/zeppelin/client/examples/SparkExample.java | 4 ++++ .../zeppelin/client/examples/ZeppelinClientExample.java | 4 ++++ .../zeppelin/client/examples/ZeppelinClientExample2.java | 2 +- 11 files changed, 41 insertions(+), 1 deletion(-) diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java index 0cd9e38cf54..64b61843db7 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample.java @@ -26,6 +26,10 @@ import java.util.HashMap; import java.util.Map; +/** + * Advanced example of run flink streaming sql via session api. + * You can capture the streaming output via SimpleMessageHandler + */ public class FlinkAdvancedExample { public static void main(String[] args) { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java index 272ab625da6..787bd136484 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkAdvancedExample2.java @@ -28,6 +28,12 @@ import java.util.HashMap; import java.util.Map; +/** + * Advanced example of run flink streaming sql via session api. + * You can capture the streaming output via CompositeMessageHandler. + * You can specify StatementMessageHandler(MyStatementMessageHandler1, MyStatementMessageHandler2) + * for each flink job. + */ public class FlinkAdvancedExample2 { public static void main(String[] args) { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkExample.java index 7bea74b7a7d..562156ff75f 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/FlinkExample.java @@ -25,6 +25,10 @@ import java.util.HashMap; import java.util.Map; + +/** + * Basic example of run flink code (scala, sql, python) via session api. + */ public class FlinkExample { public static void main(String[] args) { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/HiveExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/HiveExample.java index 94adda77776..1e14a865505 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/HiveExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/HiveExample.java @@ -25,6 +25,10 @@ import java.util.HashMap; import java.util.Map; +/** + * Basic example of run hive sql via session api. + * And you can capture the job progress info via SimpleMessageHandler. + */ public class HiveExample { public static void main(String[] args) { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PrestoExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PrestoExample.java index dc9fd5029ed..5373b0082ca 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PrestoExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PrestoExample.java @@ -26,6 +26,10 @@ import java.util.HashMap; import java.util.Map; + +/** + * Basic example of run presto sql via session api. + */ public class PrestoExample { public static void main(String[] args) { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PythonExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PythonExample.java index d0e41be5f2f..6a17ffb41d7 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PythonExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/PythonExample.java @@ -25,6 +25,9 @@ import java.util.HashMap; import java.util.Map; +/** + * Basic example of run python code via session api. + */ public class PythonExample { public static void main(String[] args) { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/RExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/RExample.java index b2a44aea763..1407374ba9a 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/RExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/RExample.java @@ -25,6 +25,10 @@ import java.util.HashMap; import java.util.Map; + +/** + * Basic example of run r code via session api. + */ public class RExample { public static void main(String[] args) { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java index 3115359d973..ec0933f8c46 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkAdvancedExample.java @@ -25,6 +25,9 @@ import java.util.HashMap; import java.util.Map; +/** + * Advanced example of run spark code via session api. + */ public class SparkAdvancedExample { public static void main(String[] args) { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkExample.java index 462d1f4e432..001c603cc33 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/SparkExample.java @@ -25,6 +25,10 @@ import java.util.HashMap; import java.util.Map; + +/** + * Basic example of run spark code (scala, sql, python, r) via session api. + */ public class SparkExample { public static void main(String[] args) { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample.java index 068ea8976a7..6dbae588010 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample.java @@ -22,6 +22,10 @@ import org.apache.zeppelin.client.ParagraphResult; import org.apache.zeppelin.client.ZeppelinClient; + +/** + * Basic example of running zeppelin note/paragraph via ZeppelinClient (low level api) + */ public class ZeppelinClientExample { public static void main(String[] args) throws Exception { diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample2.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample2.java index c469d04caf3..2e0e53162f1 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample2.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample2.java @@ -26,7 +26,7 @@ import java.util.Map; /** - * This example use ZeppelinClient to run the spark tutorial note. + * Basic example of running existing note via ZeppelinClient (low level api) * */ public class ZeppelinClientExample2 { From 4205c1c5e2725033a48186678cb62894e68d9cb5 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 26 Aug 2020 23:29:59 +0800 Subject: [PATCH 16/19] code refactoring --- .travis.yml | 20 ++++++ .../apache/zeppelin/client/ExecuteResult.java | 6 +- .../zeppelin/client/ParagraphResult.java | 8 ++- .../org/apache/zeppelin/client/Result.java | 4 -- .../{SessionResult.java => SessionInfo.java} | 26 +++++++- .../java/org/apache/zeppelin/client/Test.java | 15 ----- .../org/apache/zeppelin/client/ZSession.java | 21 +++---- .../zeppelin/client/ZeppelinClient.java | 32 +++++----- .../websocket/CompositeMessageHandler.java | 2 +- .../zeppelin/client/websocket/Message.java | 2 +- .../websocket/ZeppelinWebSocketClient.java | 29 --------- .../integration/ZSessionIntegrationTest.java | 12 ++-- ...ZeppelinClientWithAuthIntegrationTest.java | 2 +- .../src/test/resources/log4j.properties | 2 - .../apache/zeppelin/rest/NotebookRestApi.java | 2 +- .../apache/zeppelin/rest/SessionManager.java | 28 +++++---- .../apache/zeppelin/rest/SessionRestApi.java | 23 +++---- .../apache/zeppelin/rest/message/Session.java | 44 ------------- .../zeppelin/rest/message/SessionInfo.java | 61 +++++++++++++++++++ .../zeppelin/service/NotebookService.java | 1 - 20 files changed, 178 insertions(+), 162 deletions(-) rename zeppelin-client/src/main/java/org/apache/zeppelin/client/{SessionResult.java => SessionInfo.java} (60%) delete mode 100644 zeppelin-client/src/main/java/org/apache/zeppelin/client/Test.java delete mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/SessionInfo.java diff --git a/.travis.yml b/.travis.yml index faf1b0cbf04..9ba1ef6e3e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -215,6 +215,26 @@ jobs: - mvn test -DskipRat -pl $(echo .,zeppelin-interpreter,zeppelin-interpreter-shaded,${INTERPRETERS} | sed 's/!//g') -Pscala-2.10 -B + - name: "Test zeppelin-client integration test" + jdk: "openjdk8" + dist: xenial + before_install: + - export PYTHON=3 + - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc + - bash -x ./testing/install_external_dependencies.sh + - source ~/.environ + install: + - mvn install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark/spark-dependencies,markdown,flink/interpreter,jdbc,shell -am + - mvn clean package -T 2C -pl zeppelin-plugins -amd -B + before_script: + - echo "export ZEPPELIN_HELIUM_REGISTRY=helium" >> conf/zeppelin-env.sh + - echo "export SPARK_PRINT_LAUNCH_COMMAND=true" >> conf/zeppelin-env.sh + - export SPARK_PRINT_LAUNCH_COMMAND=true + - tail conf/zeppelin-env.sh + script: + - mvn test -DskipRat -pl zeppelin-interpreter-integration -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest + + - name: "Test flink 1.10 & flink integration test" jdk: "openjdk8" dist: xenial diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java index 8ad9e8c26d4..c71bf06cf46 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ExecuteResult.java @@ -30,9 +30,13 @@ public class ExecuteResult { private String statementId; private Status status; + // each statement may return multiple results private List results; + // if there's any job in the statement, then it will also contain job urls. + // e.g. spark job url private List jobUrls; - private List bufferedResults; + // if there's any job in the statement, then it will also contain a progress + // range from 0 to 100. e.g. spark job progress. private int progress; public ExecuteResult(ParagraphResult paragraphResult) { diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java index 0f305b591dc..a28c604a9ba 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ParagraphResult.java @@ -33,8 +33,13 @@ public class ParagraphResult { private String paragraphId; private Status status; + // if there's any job in the statement, then it will also contain a progress + // range from 0 to 100. e.g. spark job progress. private int progress; + // each paragraph may return multiple results private List results; + // if there's any job in the statement, then it will also contain job urls. + // e.g. spark job url private List jobUrls; public ParagraphResult(JSONObject paragraphJson) { @@ -90,10 +95,11 @@ public List getJobUrls() { } /** + * Get results in text format. * * @return */ - public String getMessage() { + public String getResultInText() { StringBuilder builder = new StringBuilder(); if (results != null) { for (Result result : results) { diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java index 4df2a908532..e1c5b901fd9 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Result.java @@ -46,10 +46,6 @@ public String getData() { return data; } - public void appendData(String newData) { - this.data = this.data + newData; - } - @Override public String toString() { return "Result{" + diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionInfo.java similarity index 60% rename from zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java rename to zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionInfo.java index 08bf248acd1..b2b9eeee071 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionResult.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionInfo.java @@ -1,8 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.zeppelin.client; import kong.unirest.json.JSONObject; -public class SessionResult { + +/** + * Represent each ZSession info. + */ +public class SessionInfo { private String sessionId; private String noteId; @@ -11,7 +33,7 @@ public class SessionResult { private String weburl; private String startTime; - public SessionResult(JSONObject sessionJson) { + public SessionInfo(JSONObject sessionJson) { this.sessionId = sessionJson.getString("sessionId"); this.noteId = sessionJson.getString("noteId"); this.interpreter = sessionJson.getString("interpreter"); diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Test.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/Test.java deleted file mode 100644 index b0ce4734f67..00000000000 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/Test.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.apache.zeppelin.client; - -import org.apache.commons.lang3.StringUtils; - -import java.util.List; - -public class Test { - - public static void main(String[] args) throws Exception { - ClientConfig clientConfig = new ClientConfig("http://localhost:8080"); - ZeppelinClient zeppelinClient = new ZeppelinClient(clientConfig); - List sessionResultList = zeppelinClient.listSessions(); - System.out.println(StringUtils.join("\n", sessionResultList)); - } -} diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index 29a606b5ff5..c133b9953b3 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -44,7 +44,7 @@ public class ZSession { private String sessionId; private String noteId; - private String weburl; + private SessionInfo sessionInfo; private ZeppelinWebSocketClient webSocketClient; @@ -103,7 +103,7 @@ public void start(MessageHandler messageHandler) throws Exception { String paragraphId = zeppelinClient.addParagraph(noteId, "Session Configuration", builder.toString()); ParagraphResult paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId, sessionId); if (paragraphResult.getStatus() != Status.FINISHED) { - throw new Exception("Fail to configure session, " + paragraphResult.getMessage()); + throw new Exception("Fail to configure session, " + paragraphResult.getResultInText()); } // start session @@ -111,9 +111,9 @@ public void start(MessageHandler messageHandler) throws Exception { paragraphId = zeppelinClient.addParagraph(noteId, "Session Init", "%" + interpreter + "(init=true)"); paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId, sessionId); if (paragraphResult.getStatus() != Status.FINISHED) { - throw new Exception("Fail to init session, " + paragraphResult.getMessage()); + throw new Exception("Fail to init session, " + paragraphResult.getResultInText()); } - this.weburl = zeppelinClient.getSessionWebUrl(sessionId); + this.sessionInfo = zeppelinClient.getSession(sessionId); if (messageHandler != null) { this.webSocketClient = new ZeppelinWebSocketClient(messageHandler); @@ -134,7 +134,7 @@ public void start(MessageHandler messageHandler) throws Exception { */ public void stop() throws Exception { if (sessionId != null) { - zeppelinClient.stopSession(interpreter, sessionId); + zeppelinClient.stopSession(sessionId); } if (webSocketClient != null) { webSocketClient.stop(); @@ -169,12 +169,11 @@ public static ZSession createFromExistingSession(ClientConfig clientConfig, } private void reconnect(MessageHandler messageHandler) throws Exception { - SessionResult sessionResult = this.zeppelinClient.getSession(sessionId); - this.noteId = sessionResult.getNoteId(); - if (!sessionResult.getState().equalsIgnoreCase("Running")) { - throw new Exception("Session " + sessionId + " is not running, state: " + sessionResult.getState()); + this.sessionInfo = this.zeppelinClient.getSession(sessionId); + this.noteId = sessionInfo.getNoteId(); + if (!sessionInfo.getState().equalsIgnoreCase("Running")) { + throw new Exception("Session " + sessionId + " is not running, state: " + sessionInfo.getState()); } - this.weburl = sessionResult.getWeburl(); if (messageHandler != null) { this.webSocketClient = new ZeppelinWebSocketClient(messageHandler); @@ -423,7 +422,7 @@ public String getNoteId() { } public String getWeburl() { - return weburl; + return sessionInfo.getWeburl(); } public String getSessionId() { diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java index 015281f16a3..6bef42c8843 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java @@ -69,8 +69,6 @@ public boolean isTrusted(X509Certificate[] chain, String authType) { throw new Exception("Fail to setup httpclient of Unirest", e); } } - - Unirest.config().setDefaultHeader("Origin", "localhost"); } public ClientConfig getClientConfig() { @@ -78,12 +76,15 @@ public ClientConfig getClientConfig() { } /** - * Throw exception is the status code is not 200. + * Throw exception if the status code is not 200. * * @param response * @throws Exception */ private void checkResponse(HttpResponse response) throws Exception { + if (response.getStatus() == 302) { + throw new Exception("Please login first"); + } if (response.getStatus() != 200) { throw new Exception(String.format("Unable to call rest api, status: %s, statusText: %s, message: %s", response.getStatus(), @@ -122,7 +123,7 @@ public String getVersion() throws Exception { /** * Request a new session id. It doesn't create session (interpreter process) in zeppelin server side, but just - * create a unique session id. + * create an unique session id. * * @param interpreter * @return @@ -140,16 +141,14 @@ public String newSession(String interpreter) throws Exception { } /** - * Stop the session(interpreter process) in zeppelin server. + * Stop the session(interpreter process) in Zeppelin server. * - * @param interpreter * @param sessionId * @throws Exception */ - public void stopSession(String interpreter, String sessionId) throws Exception { + public void stopSession(String sessionId) throws Exception { HttpResponse response = Unirest - .delete("/session/{interpreter}/{sessionId}") - .routeParam("interpreter", interpreter) + .delete("/session/{sessionId}") .routeParam("sessionId", sessionId) .asJson(); checkResponse(response); @@ -158,11 +157,12 @@ public void stopSession(String interpreter, String sessionId) throws Exception { } /** + * Get session info for the provided sessionId. * * @param sessionId * @throws Exception */ - public SessionResult getSession(String sessionId) throws Exception { + public SessionInfo getSession(String sessionId) throws Exception { HttpResponse response = Unirest .get("/session/{sessionId}") .routeParam("sessionId", sessionId) @@ -172,7 +172,7 @@ public SessionResult getSession(String sessionId) throws Exception { checkJsonNodeStatus(jsonNode); JSONObject bodyObject = jsonNode.getObject().getJSONObject("body"); - return new SessionResult(bodyObject); + return new SessionInfo(bodyObject); } /** @@ -204,7 +204,7 @@ public String getSessionWebUrl(String sessionId) throws Exception { * @return * @throws Exception */ - public List listSessions() throws Exception { + public List listSessions() throws Exception { return listSessions(null); } @@ -214,7 +214,7 @@ public List listSessions() throws Exception { * @return * @throws Exception */ - public List listSessions(String interpreter) throws Exception { + public List listSessions(String interpreter) throws Exception { GetRequest getRequest = Unirest.get("/session"); if (interpreter != null) { getRequest.queryString("interpreter", interpreter); @@ -224,11 +224,11 @@ public List listSessions(String interpreter) throws Exception { JsonNode jsonNode = response.getBody(); checkJsonNodeStatus(jsonNode); JSONArray sessionJsonArray = jsonNode.getObject().getJSONArray("body"); - List sessionResults = new ArrayList<>(); + List sessionInfos = new ArrayList<>(); for (int i = 0; i< sessionJsonArray.length();++i) { - sessionResults.add(new SessionResult(sessionJsonArray.getJSONObject(i))); + sessionInfos.add(new SessionInfo(sessionJsonArray.getJSONObject(i))); } - return sessionResults; + return sessionInfos; } /** diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/CompositeMessageHandler.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/CompositeMessageHandler.java index 5f5094b77f3..51169fbc8dc 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/CompositeMessageHandler.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/CompositeMessageHandler.java @@ -24,7 +24,7 @@ import java.util.Map; /** - * You can add StatementHandler for each statement. + * CompositeMessageHandler allos you to add StatementHandler for each statement. */ public class CompositeMessageHandler extends AbstractMessageHandler { diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/Message.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/Message.java index e20d0b5b744..cca3a465db4 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/Message.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/Message.java @@ -29,7 +29,7 @@ /** * Copied from zeppelin-zengine (TODO, zjffdu). Should resume the same piece of code instead of copying. - * Zeppelin websocket massage template class. + * Zeppelin websocket message template class. */ public class Message implements JsonSerializable { /** diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/ZeppelinWebSocketClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/ZeppelinWebSocketClient.java index d770b3edc28..4a72c0a09d0 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/ZeppelinWebSocketClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/websocket/ZeppelinWebSocketClient.java @@ -119,33 +119,4 @@ public void stop() throws Exception { } } - public static void main(String[] args) throws Exception { - String dest = "ws://localhost:18086/ws"; - WebSocketClient client = new WebSocketClient(); - try { - ZeppelinWebSocketClient socket = new ZeppelinWebSocketClient(new SimpleMessageHandler()); - client.start(); - URI echoUri = new URI(dest); - ClientUpgradeRequest request = new ClientUpgradeRequest(); - client.connect(socket, echoUri, request); - socket.getConnectLatch().await(); - - Message msg = new Message(Message.OP.LIST_NOTES); - socket.send(msg); - - // wait for closed socket connection. - socket.awaitClose(5, TimeUnit.SECONDS); - } catch (Throwable t) { - t.printStackTrace(); - } finally { - try { - client.stop(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - Thread.sleep(10000 * 100); - } - } diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java index bd076b2c753..e3ee5af024b 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java @@ -76,7 +76,7 @@ public void testZSession_Shell() throws Exception { try { session.start(); - assertNull(session.getWeburl()); + assertEquals("", session.getWeburl()); assertNotNull(session.getNoteId()); Note note = notebook.getNote(session.getNoteId()); @@ -113,7 +113,7 @@ public void testZSession_Shell_Submit() throws Exception { try { session.start(); - assertNull(session.getWeburl()); + assertEquals("", session.getWeburl()); assertNotNull(session.getNoteId()); Note note = notebook.getNote(session.getNoteId()); @@ -390,7 +390,7 @@ public void testZSession_Python() throws Exception { try { session.start(new SimpleMessageHandler()); - assertNull(session.getWeburl()); + assertEquals("", session.getWeburl()); assertNotNull(session.getNoteId()); // python @@ -427,7 +427,6 @@ public void testZSession_Jdbc() throws Exception { intpProperties.put("default.driver", "com.mysql.jdbc.Driver"); intpProperties.put("default.url", "jdbc:mysql://localhost:3306/"); intpProperties.put("default.user", "root"); - intpProperties.put("default.password", "123456"); ZSession session = ZSession.builder() .setClientConfig(clientConfig) @@ -437,7 +436,7 @@ public void testZSession_Jdbc() throws Exception { try { session.start(); - assertNull(session.getWeburl()); + assertEquals("", session.getWeburl()); assertNotNull(session.getNoteId()); // show databases @@ -466,7 +465,6 @@ public void testZSession_Jdbc_Submit() throws Exception { intpProperties.put("default.driver", "com.mysql.jdbc.Driver"); intpProperties.put("default.url", "jdbc:mysql://localhost:3306/"); intpProperties.put("default.user", "root"); - intpProperties.put("default.password", "123456"); ZSession session = ZSession.builder() .setClientConfig(clientConfig) @@ -476,7 +474,7 @@ public void testZSession_Jdbc_Submit() throws Exception { try { session.start(); - assertNull(session.getWeburl()); + assertEquals("", session.getWeburl()); assertNotNull(session.getNoteId()); // show databases diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java index 36c652e7dfb..16c139128a3 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java @@ -73,7 +73,7 @@ public void testCreateNoteWithoutLogin() throws Exception { zeppelinClient.createNote("/note_1"); fail("Should fail due to not login"); } catch (Exception e) { - assertTrue(e.getMessage(), e.getMessage().contains("statusText: Found")); + assertTrue(e.getMessage(), e.getMessage().contains("login first")); } } diff --git a/zeppelin-interpreter-integration/src/test/resources/log4j.properties b/zeppelin-interpreter-integration/src/test/resources/log4j.properties index e194ac771ce..403c0784d05 100644 --- a/zeppelin-interpreter-integration/src/test/resources/log4j.properties +++ b/zeppelin-interpreter-integration/src/test/resources/log4j.properties @@ -44,5 +44,3 @@ log4j.logger.org.hibernate.type=ALL log4j.logger.org.apache.hadoop=WARN log4j.logger.org.apache.zeppelin.client=INFO -log4j.logger.org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer=DEBUG - diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 75b5620e0f2..915f69f90e9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -918,7 +918,7 @@ public Response runParagraph(@PathParam("noteId") String noteId, try { LOGGER.info("Run paragraph job asynchronously {} {} {}", noteId, paragraphId, message); - Note note = notebook.getNote(noteId, true); + Note note = notebook.getNote(noteId); checkIfNoteIsNotNull(note); Paragraph paragraph = note.getParagraph(paragraphId); checkIfParagraphIsNotNull(paragraph); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java index 78d8c873b6b..6ac48d4863e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java @@ -24,7 +24,7 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.notebook.Notebook; -import org.apache.zeppelin.rest.message.Session; +import org.apache.zeppelin.rest.message.SessionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,9 +69,15 @@ public synchronized String newSession(String interpreter) throws Exception { public void removeSession(String sessionId) { this.sessions.remove(sessionId); + InterpreterGroup interpreterGroup = this.interpreterSettingManager.getInterpreterGroupById(sessionId); + if (interpreterGroup == null) { + LOGGER.info("No interpreterGroup for session: " + sessionId); + return; + } + ((ManagedInterpreterGroup) interpreterGroup).getInterpreterSetting().closeInterpreters(sessionId); } - public Session getSession(String sessionId) throws Exception { + public SessionInfo getSession(String sessionId) throws Exception { InterpreterGroup interpreterGroup = this.interpreterSettingManager.getInterpreterGroupById(sessionId); if (interpreterGroup != null) { RemoteInterpreterProcess remoteInterpreterProcess = @@ -100,28 +106,28 @@ public Session getSession(String sessionId) throws Exception { LOGGER.warn("Found more than 1 notes with path: " + notePath); } } - return new Session(sessionId, noteId, interpreter, + return new SessionInfo(sessionId, noteId, interpreter, state, interpreterGroup.getWebUrl(), startTime); } LOGGER.warn("No such session: " + sessionId); return null; } - public List getAllSessionStatus() throws Exception { - List sessionList = new ArrayList<>(); + public List getAllSessions() throws Exception { + List sessionList = new ArrayList<>(); for (String sessionId : sessions) { - Session status = getSession(sessionId); - if (status != null) { - sessionList.add(status); + SessionInfo session = getSession(sessionId); + if (session != null) { + sessionList.add(session); } } return sessionList; } - public List getAllSessionStatus(String interpreterGroup) throws Exception { - List sessionList = new ArrayList<>(); + public List getAllSessions(String interpreterGroup) throws Exception { + List sessionList = new ArrayList<>(); for (String sessionId : sessions) { - Session status = getSession(sessionId); + SessionInfo status = getSession(sessionId); if (status != null && interpreterGroup.equals(status.getInterpreter())) { sessionList.add(status); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java index dd906829919..2ab53aecb81 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java @@ -20,10 +20,9 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.zeppelin.annotation.ZeppelinApi; -import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.Notebook; -import org.apache.zeppelin.rest.message.Session; +import org.apache.zeppelin.rest.message.SessionInfo; import org.apache.zeppelin.server.JsonResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,9 +37,7 @@ import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Response; -import java.util.HashMap; import java.util.List; -import java.util.Map; /** * Rest api endpoint for the ZSession. @@ -72,11 +69,11 @@ public Response listSessions(@QueryParam("interpreter") String interpreter) { LOGGER.info("List all sessions for interpreter: " + interpreter); } try { - List sessionList = null; + List sessionList = null; if (StringUtils.isBlank(interpreter)) { - sessionList = sessionManager.getAllSessionStatus(); + sessionList = sessionManager.getAllSessions(); } else { - sessionList = sessionManager.getAllSessionStatus(interpreter); + sessionList = sessionManager.getAllSessions(interpreter); } return new JsonResponse<>(Response.Status.OK, sessionList).build(); } catch (Exception e) { @@ -100,13 +97,11 @@ public Response newSession(@PathParam("interpreter") String interpreter) { } @DELETE - @Path("{interpreter}/{sessionId}") - public Response stopSession(@PathParam("interpreter") String interpreter, - @PathParam("sessionId") String sessionId) { - LOGGER.info("Stop session: {} for interpreter: {}", sessionId, interpreter); + @Path("{sessionId}") + public Response stopSession(@PathParam("sessionId") String sessionId) { + LOGGER.info("Stop session: {}", sessionId); try { - sessionManager.removeSession(interpreter); - notebook.getInterpreterSettingManager().get(interpreter).closeInterpreters(sessionId); + sessionManager.removeSession(sessionId); return new JsonResponse<>(Response.Status.OK, sessionId).build(); } catch (Exception e) { return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, @@ -122,7 +117,7 @@ public Response stopSession(@PathParam("interpreter") String interpreter, @ZeppelinApi public Response getSession(@PathParam("sessionId") String sessionId) { try { - Session session = sessionManager.getSession(sessionId); + SessionInfo session = sessionManager.getSession(sessionId); if (session == null) { return new JsonResponse<>(Response.Status.NOT_FOUND).build(); } else { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java deleted file mode 100644 index 8ac7bd7779f..00000000000 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/Session.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.apache.zeppelin.rest.message; - -public class Session { - - private String sessionId; - private String noteId; - private String interpreter; - private String state; - private String weburl; - private String startTime; - - public Session(String sessionId, String noteId, String interpreter, String state, String weburl, String startTime) { - this.sessionId = sessionId; - this.noteId = noteId; - this.interpreter = interpreter; - this.state = state; - this.weburl = weburl; - this.startTime = startTime; - } - - public String getSessionId() { - return sessionId; - } - - public String getNoteId() { - return noteId; - } - - public String getState() { - return state; - } - - public String getInterpreter() { - return interpreter; - } - - public String getWeburl() { - return weburl; - } - - public String getStartTime() { - return startTime; - } -} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/SessionInfo.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/SessionInfo.java new file mode 100644 index 00000000000..5ddf4913247 --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/SessionInfo.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.rest.message; + +public class SessionInfo { + + private String sessionId; + private String noteId; + private String interpreter; + private String state; + private String weburl; + private String startTime; + + public SessionInfo(String sessionId, String noteId, String interpreter, String state, String weburl, String startTime) { + this.sessionId = sessionId; + this.noteId = noteId; + this.interpreter = interpreter; + this.state = state; + this.weburl = weburl; + this.startTime = startTime; + } + + public String getSessionId() { + return sessionId; + } + + public String getNoteId() { + return noteId; + } + + public String getState() { + return state; + } + + public String getInterpreter() { + return interpreter; + } + + public String getWeburl() { + return weburl; + } + + public String getStartTime() { + return startTime; + } +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java index ba9aa022c81..37633451fae 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java @@ -83,7 +83,6 @@ public class NotebookService { private Notebook notebook; private AuthorizationService authorizationService; private SchedulerService schedulerService; - private SessionManager sessionManager; @Inject public NotebookService( From bb053712904dd47b131b79eed926069f22b45a47 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Sat, 29 Aug 2020 14:54:34 +0800 Subject: [PATCH 17/19] update --- .../examples/ZeppelinClientExample2.java | 34 ++++-- .../apache/zeppelin/client/SessionInfo.java | 24 ++-- .../org/apache/zeppelin/client/ZSession.java | 72 ++++++------ .../zeppelin/client/ZeppelinClient.java | 75 ++++++------- .../integration/ZSessionIntegrationTest.java | 20 ++-- .../ZeppelinClientIntegrationTest.java | 10 +- ...ZeppelinClientWithAuthIntegrationTest.java | 2 + .../interpreter/ExecutionContext.java | 4 +- .../zeppelin/rest/InterpreterRestApi.java | 30 ++--- .../apache/zeppelin/rest/SessionManager.java | 104 ++++++++++++------ .../apache/zeppelin/rest/SessionRestApi.java | 17 +-- .../zeppelin/rest/message/NewNoteRequest.java | 13 ++- .../message/RestartInterpreterRequest.java | 8 +- .../zeppelin/rest/message/SessionInfo.java | 25 +++++ 14 files changed, 258 insertions(+), 180 deletions(-) diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample2.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample2.java index 2e0e53162f1..05e183cc19d 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample2.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample2.java @@ -38,20 +38,34 @@ public static void main(String[] args) throws Exception { String zeppelinVersion = zClient.getVersion(); System.out.println("Zeppelin version: " + zeppelinVersion); - ParagraphResult paragraphResult = zClient.executeParagraph("2A94M5J1Z", "20150210-015259_1403135953"); - System.out.println("Execute the 1st spark tutorial paragraph, paragraph result: " + paragraphResult); + // execute note 2A94M5J1Z paragraph by paragraph + try { + ParagraphResult paragraphResult = zClient.executeParagraph("2A94M5J1Z", "20150210-015259_1403135953"); + System.out.println("Execute the 1st spark tutorial paragraph, paragraph result: " + paragraphResult); - paragraphResult = zClient.executeParagraph("2A94M5J1Z", "20150210-015302_1492795503"); - System.out.println("Execute the 2nd spark tutorial paragraph, paragraph result: " + paragraphResult); + paragraphResult = zClient.executeParagraph("2A94M5J1Z", "20150210-015302_1492795503"); + System.out.println("Execute the 2nd spark tutorial paragraph, paragraph result: " + paragraphResult); + Map parameters = new HashMap<>(); + parameters.put("maxAge", "40"); + paragraphResult = zClient.executeParagraph("2A94M5J1Z", "20150212-145404_867439529", parameters); + System.out.println("Execute the 3rd spark tutorial paragraph, paragraph result: " + paragraphResult); + + parameters = new HashMap<>(); + parameters.put("marital", "married"); + paragraphResult = zClient.executeParagraph("2A94M5J1Z", "20150213-230422_1600658137", parameters); + System.out.println("Execute the 4th spark tutorial paragraph, paragraph result: " + paragraphResult); + } finally { + // you need to stop interpreter explicitly if you are running paragraph separately. + zClient.stopInterpreter("2A94M5J1Z", "spark"); + } + + // execute this whole note, this note will run under a didicated interpreter process which will be + // stopped after note execution. Map parameters = new HashMap<>(); parameters.put("maxAge", "40"); - paragraphResult = zClient.executeParagraph("2A94M5J1Z", "20150212-145404_867439529", parameters); - System.out.println("Execute the 3rd spark tutorial paragraph, paragraph result: " + paragraphResult); - - parameters = new HashMap<>(); parameters.put("marital", "married"); - paragraphResult = zClient.executeParagraph("2A94M5J1Z", "20150213-230422_1600658137", parameters); - System.out.println("Execute the 4th spark tutorial paragraph, paragraph result: " + paragraphResult); + NoteResult noteResult = zClient.executeNote("2A94M5J1Z", parameters); + System.out.println("Execute the spark tutorial note, note result: " + noteResult); } } diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionInfo.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionInfo.java index b2b9eeee071..4eac5c454c4 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionInfo.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/SessionInfo.java @@ -33,20 +33,28 @@ public class SessionInfo { private String weburl; private String startTime; + public SessionInfo(String sessionId) { + this.sessionId = sessionId; + } + public SessionInfo(JSONObject sessionJson) { - this.sessionId = sessionJson.getString("sessionId"); - this.noteId = sessionJson.getString("noteId"); - this.interpreter = sessionJson.getString("interpreter"); - this.state = sessionJson.getString("state"); + if (sessionJson.has("sessionId")) { + this.sessionId = sessionJson.getString("sessionId"); + } + if (sessionJson.has("noteId")) { + this.noteId = sessionJson.getString("noteId"); + } + if (sessionJson.has("interpreter")) { + this.interpreter = sessionJson.getString("interpreter"); + } + if (sessionJson.has("state")) { + this.state = sessionJson.getString("state"); + } if (sessionJson.has("weburl")) { this.weburl = sessionJson.getString("weburl"); - } else { - this.weburl = ""; } if (sessionJson.has("startTime")) { this.startTime = sessionJson.getString("startTime"); - } else { - this.startTime = ""; } } diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java index c133b9953b3..ffedf40e4ed 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZSession.java @@ -42,8 +42,6 @@ public class ZSession { // max number of retained statements, each statement represent one paragraph. private int maxStatement; - private String sessionId; - private String noteId; private SessionInfo sessionInfo; private ZeppelinWebSocketClient webSocketClient; @@ -64,11 +62,11 @@ public ZSession(ClientConfig clientConfig, } private ZSession(ClientConfig clientConfig, - String interpreter, - String sessionId) throws Exception { + String interpreter, + String sessionId) throws Exception { this.zeppelinClient = new ZeppelinClient(clientConfig); this.interpreter = interpreter; - this.sessionId = sessionId; + this.sessionInfo = new SessionInfo(sessionId); } /** @@ -90,9 +88,8 @@ public void start() throws Exception { * @throws Exception */ public void start(MessageHandler messageHandler) throws Exception { - this.sessionId = zeppelinClient.newSession(interpreter); + this.sessionInfo = zeppelinClient.newSession(interpreter); - this.noteId = zeppelinClient.createNote("/_ZSession/" + interpreter + "/" + sessionId, interpreter); // inline configuration StringBuilder builder = new StringBuilder("%" + interpreter + ".conf\n"); if (intpProperties != null) { @@ -100,20 +97,20 @@ public void start(MessageHandler messageHandler) throws Exception { builder.append(entry.getKey() + " " + entry.getValue() + "\n"); } } - String paragraphId = zeppelinClient.addParagraph(noteId, "Session Configuration", builder.toString()); - ParagraphResult paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId, sessionId); + String paragraphId = zeppelinClient.addParagraph(getNoteId(), "Session Configuration", builder.toString()); + ParagraphResult paragraphResult = zeppelinClient.executeParagraph(getNoteId(), paragraphId, getSessionId()); if (paragraphResult.getStatus() != Status.FINISHED) { throw new Exception("Fail to configure session, " + paragraphResult.getResultInText()); } // start session // add local properties (init) to avoid skip empty paragraph. - paragraphId = zeppelinClient.addParagraph(noteId, "Session Init", "%" + interpreter + "(init=true)"); - paragraphResult = zeppelinClient.executeParagraph(noteId, paragraphId, sessionId); + paragraphId = zeppelinClient.addParagraph(getNoteId(), "Session Init", "%" + interpreter + "(init=true)"); + paragraphResult = zeppelinClient.executeParagraph(getNoteId(), paragraphId, getSessionId()); if (paragraphResult.getStatus() != Status.FINISHED) { throw new Exception("Fail to init session, " + paragraphResult.getResultInText()); } - this.sessionInfo = zeppelinClient.getSession(sessionId); + this.sessionInfo = zeppelinClient.getSession(getSessionId()); if (messageHandler != null) { this.webSocketClient = new ZeppelinWebSocketClient(messageHandler); @@ -122,7 +119,7 @@ public void start(MessageHandler messageHandler) throws Exception { // call GET_NOTE to establish websocket connection between this session and zeppelin-server Message msg = new Message(Message.OP.GET_NOTE); - msg.put("id", this.noteId); + msg.put("id", getNoteId()); this.webSocketClient.send(msg); } } @@ -133,8 +130,8 @@ public void start(MessageHandler messageHandler) throws Exception { * @throws Exception */ public void stop() throws Exception { - if (sessionId != null) { - zeppelinClient.stopSession(sessionId); + if (getSessionId() != null) { + zeppelinClient.stopSession(getSessionId()); } if (webSocketClient != null) { webSocketClient.stop(); @@ -169,10 +166,9 @@ public static ZSession createFromExistingSession(ClientConfig clientConfig, } private void reconnect(MessageHandler messageHandler) throws Exception { - this.sessionInfo = this.zeppelinClient.getSession(sessionId); - this.noteId = sessionInfo.getNoteId(); + this.sessionInfo = this.zeppelinClient.getSession(getSessionId()); if (!sessionInfo.getState().equalsIgnoreCase("Running")) { - throw new Exception("Session " + sessionId + " is not running, state: " + sessionInfo.getState()); + throw new Exception("Session " + getSessionId() + " is not running, state: " + sessionInfo.getState()); } if (messageHandler != null) { @@ -182,7 +178,7 @@ private void reconnect(MessageHandler messageHandler) throws Exception { // call GET_NOTE to establish websocket connection between this session and zeppelin-server Message msg = new Message(Message.OP.GET_NOTE); - msg.put("id", this.noteId); + msg.put("id", getNoteId()); this.webSocketClient.send(msg); } } @@ -275,13 +271,13 @@ public ExecuteResult execute(String subInterpreter, String text = builder.toString(); - String nextParagraphId = zeppelinClient.nextSessionParagraph(noteId, maxStatement); - zeppelinClient.updateParagraph(noteId, nextParagraphId, "", text); + String nextParagraphId = zeppelinClient.nextSessionParagraph(getNoteId(), maxStatement); + zeppelinClient.updateParagraph(getNoteId(), nextParagraphId, "", text); if (messageHandler != null) { webSocketClient.addStatementMessageHandler(nextParagraphId, messageHandler); } - ParagraphResult paragraphResult = zeppelinClient.executeParagraph(noteId, nextParagraphId, sessionId); + ParagraphResult paragraphResult = zeppelinClient.executeParagraph(getNoteId(), nextParagraphId, getSessionId()); return new ExecuteResult(paragraphResult); } @@ -366,12 +362,12 @@ public ExecuteResult submit(String subInterpreter, builder.append("\n" + code); String text = builder.toString(); - String nextParagraphId = zeppelinClient.nextSessionParagraph(noteId, maxStatement); - zeppelinClient.updateParagraph(noteId, nextParagraphId, "", text); + String nextParagraphId = zeppelinClient.nextSessionParagraph(getNoteId(), maxStatement); + zeppelinClient.updateParagraph(getNoteId(), nextParagraphId, "", text); if (messageHandler != null) { webSocketClient.addStatementMessageHandler(nextParagraphId, messageHandler); } - ParagraphResult paragraphResult = zeppelinClient.submitParagraph(noteId, nextParagraphId, sessionId); + ParagraphResult paragraphResult = zeppelinClient.submitParagraph(getNoteId(), nextParagraphId, getSessionId()); return new ExecuteResult(paragraphResult); } @@ -381,7 +377,7 @@ public ExecuteResult submit(String subInterpreter, * @throws Exception */ public void cancel(String statementId) throws Exception { - zeppelinClient.cancelParagraph(noteId, statementId); + zeppelinClient.cancelParagraph(getNoteId(), statementId); } /** @@ -391,7 +387,7 @@ public void cancel(String statementId) throws Exception { * @throws Exception */ public ExecuteResult queryStatement(String statementId) throws Exception { - ParagraphResult paragraphResult = zeppelinClient.queryParagraphResult(noteId, statementId); + ParagraphResult paragraphResult = zeppelinClient.queryParagraphResult(getNoteId(), statementId); return new ExecuteResult(paragraphResult); } @@ -402,7 +398,7 @@ public ExecuteResult queryStatement(String statementId) throws Exception { * @throws Exception */ public ExecuteResult waitUntilFinished(String statementId) throws Exception { - ParagraphResult paragraphResult = zeppelinClient.waitUtilParagraphFinish(noteId, statementId); + ParagraphResult paragraphResult = zeppelinClient.waitUtilParagraphFinish(getNoteId(), statementId); return new ExecuteResult(paragraphResult); } @@ -413,20 +409,32 @@ public ExecuteResult waitUntilFinished(String statementId) throws Exception { * @throws Exception */ public ExecuteResult waitUntilRunning(String statementId) throws Exception { - ParagraphResult paragraphResult = zeppelinClient.waitUtilParagraphRunning(noteId, statementId); + ParagraphResult paragraphResult = zeppelinClient.waitUtilParagraphRunning(getNoteId(), statementId); return new ExecuteResult(paragraphResult); } public String getNoteId() { - return noteId; + if (this.sessionInfo != null) { + return this.sessionInfo.getNoteId(); + } else { + return null; + } } public String getWeburl() { - return sessionInfo.getWeburl(); + if (this.sessionInfo != null) { + return sessionInfo.getWeburl(); + } else { + return null; + } } public String getSessionId() { - return sessionId; + if (this.sessionInfo != null) { + return this.sessionInfo.getSessionId(); + } else { + return null; + } } public String getInterpreter() { diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java index 6bef42c8843..9bb52659058 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java @@ -129,15 +129,15 @@ public String getVersion() throws Exception { * @return * @throws Exception */ - public String newSession(String interpreter) throws Exception { + public SessionInfo newSession(String interpreter) throws Exception { HttpResponse response = Unirest - .post("/session/{interpreter}") - .routeParam("interpreter", interpreter) + .post("/session") + .queryString("interpreter", interpreter) .asJson(); checkResponse(response); JsonNode jsonNode = response.getBody(); checkJsonNodeStatus(jsonNode); - return jsonNode.getObject().getString("message"); + return new SessionInfo(jsonNode.getObject().getJSONObject("body")); } /** @@ -176,30 +176,7 @@ public SessionInfo getSession(String sessionId) throws Exception { } /** - * Get the session weburl. It is spark ui url for spark interpreter, - * or flink web ui for flink interpreter, or may be null for the interpreter that has no weburl. - * - * @param sessionId - * @throws Exception - */ - public String getSessionWebUrl(String sessionId) throws Exception { - HttpResponse response = Unirest - .get("/session/{sessionId}") - .routeParam("sessionId", sessionId) - .asJson(); - checkResponse(response); - JsonNode jsonNode = response.getBody(); - checkJsonNodeStatus(jsonNode); - - JSONObject bodyObject = jsonNode.getObject().getJSONObject("body"); - if (bodyObject.has("weburl")) { - return bodyObject.getString("weburl"); - } else { - return null; - } - } - - /** + * List all the sessions. * * @return * @throws Exception @@ -209,6 +186,7 @@ public List listSessions() throws Exception { } /** + * List all the sessions for the provided interpreter. * * @param interpreter * @return @@ -343,6 +321,7 @@ public NoteResult queryNoteResult(String noteId) throws Exception { /** * Execute note with provided noteId, return until note execution is completed. + * Interpreter process will be stopped after note execution. * * @param noteId * @return @@ -354,6 +333,7 @@ public NoteResult executeNote(String noteId) throws Exception { /** * Execute note with provided noteId and parameters, return until note execution is completed. + * Interpreter process will be stopped after note execution. * * @param noteId * @param parameters @@ -361,23 +341,14 @@ public NoteResult executeNote(String noteId) throws Exception { * @throws Exception */ public NoteResult executeNote(String noteId, Map parameters) throws Exception { - JSONObject bodyObject = new JSONObject(); - bodyObject.put("params", parameters); - HttpResponse response = Unirest - .post("/notebook/job/{noteId}?blocking=true&isolated=true") - .routeParam("noteId", noteId) - .body(bodyObject.toString()) - .asJson(); - checkResponse(response); - JsonNode jsonNode = response.getBody(); - checkJsonNodeStatus(jsonNode); - - return queryNoteResult(noteId); + submitNote(noteId, parameters); + return waitUntilNoteFinished(noteId); } /** * Submit note to execute with provided noteId, return at once the submission is completed. * You need to query {@link NoteResult} by yourself afterwards until note execution is completed. + * Interpreter process will be stopped after note execution. * * @param noteId * @return @@ -390,6 +361,7 @@ public NoteResult submitNote(String noteId) throws Exception { /** * Submit note to execute with provided noteId and parameters, return at once the submission is completed. * You need to query {@link NoteResult} by yourself afterwards until note execution is completed. + * Interpreter process will be stopped after note execution. * * @param noteId * @param parameters @@ -401,13 +373,14 @@ public NoteResult submitNote(String noteId, Map parameters) thro bodyObject.put("params", parameters); // run note in non-blocking and isolated way. HttpResponse response = Unirest - .post("/notebook/job/{noteId}?blocking=false&isolated=true") + .post("/notebook/job/{noteId}") .routeParam("noteId", noteId) + .queryString("blocking", "false") + .queryString("isolated", "true") .body(bodyObject) .asJson(); checkResponse(response); JsonNode jsonNode = response.getBody(); - LOGGER.info("Start to run note: " + noteId); checkJsonNodeStatus(jsonNode); return queryNoteResult(noteId); } @@ -753,4 +726,22 @@ public ParagraphResult waitUtilParagraphRunning(String noteId, String paragraphI } } + /** + * This is equal to the restart operation in note page. + * + * @param noteId + * @param interpreter + */ + public void stopInterpreter(String noteId, String interpreter) throws Exception { + JSONObject bodyObject = new JSONObject(); + bodyObject.put("noteId", noteId); + HttpResponse response = Unirest + .put("/interpreter/setting/restart/{interpreter}") + .routeParam("interpreter", interpreter) + .body(bodyObject.toString()) + .asJson(); + checkResponse(response); + JsonNode jsonNode = response.getBody(); + checkJsonNodeStatus(jsonNode); + } } diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java index e3ee5af024b..ac9c5d7a312 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZSessionIntegrationTest.java @@ -56,6 +56,8 @@ public class ZSessionIntegrationTest extends AbstractTestRestApi { public static void setUp() throws Exception { System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HELIUM_REGISTRY.getVarName(), "helium"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ALLOWED_ORIGINS.getVarName(), "*"); + AbstractTestRestApi.startUp(ZSessionIntegrationTest.class.getSimpleName()); notebook = TestUtils.getInstance(Notebook.class); sparkHome = DownloadUtils.downloadSpark("2.4.4", "2.7"); @@ -76,7 +78,7 @@ public void testZSession_Shell() throws Exception { try { session.start(); - assertEquals("", session.getWeburl()); + assertNull(session.getWeburl()); assertNotNull(session.getNoteId()); Note note = notebook.getNote(session.getNoteId()); @@ -113,7 +115,7 @@ public void testZSession_Shell_Submit() throws Exception { try { session.start(); - assertEquals("", session.getWeburl()); + assertNull(session.getWeburl()); assertNotNull(session.getNoteId()); Note note = notebook.getNote(session.getNoteId()); @@ -181,7 +183,7 @@ public void testZSession_Spark() throws Exception { "| 1| a|\n" + "| 2| b|\n" + "+---+---+", result.getResults().get(0).getData().trim()); - assertEquals(3, result.getJobUrls().size()); + assertTrue(result.getJobUrls().size() > 0); // sparkr result = session.execute("r", "df <- as.DataFrame(faithful)\nhead(df)"); @@ -197,7 +199,7 @@ public void testZSession_Spark() throws Exception { assertEquals(1, result.getResults().size()); assertEquals("TABLE", result.getResults().get(0).getType()); assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("1\ta\n2\tb\n")); - assertEquals(3, result.getJobUrls().size()); + assertTrue(result.getJobUrls().size() > 0); // spark invalid sql result = session.execute("sql", "select * from unknown_table"); @@ -251,7 +253,7 @@ public void testZSession_Spark_Submit() throws Exception { "| 1| a|\n" + "| 2| b|\n" + "+---+---+", result.getResults().get(0).getData().trim()); - assertEquals(3, result.getJobUrls().size()); + assertTrue(result.getJobUrls().size() > 0); // sparkr result = session.submit("r", "df <- as.DataFrame(faithful)\nhead(df)"); @@ -269,7 +271,7 @@ public void testZSession_Spark_Submit() throws Exception { assertEquals(1, result.getResults().size()); assertEquals("TABLE", result.getResults().get(0).getType()); assertTrue(result.getResults().get(0).getData(), result.getResults().get(0).getData().contains("1\ta\n2\tb\n")); - assertEquals(3, result.getJobUrls().size()); + assertTrue(result.getJobUrls().size() > 0); // spark invalid sql result = session.submit("sql", "select * from unknown_table"); @@ -390,7 +392,7 @@ public void testZSession_Python() throws Exception { try { session.start(new SimpleMessageHandler()); - assertEquals("", session.getWeburl()); + assertNull(session.getWeburl()); assertNotNull(session.getNoteId()); // python @@ -420,7 +422,7 @@ public void testZSession_Python() throws Exception { } } - @Test + //@Test public void testZSession_Jdbc() throws Exception { Map intpProperties = new HashMap<>(); @@ -458,7 +460,7 @@ public void testZSession_Jdbc() throws Exception { } } - @Test + //@Test public void testZSession_Jdbc_Submit() throws Exception { Map intpProperties = new HashMap<>(); diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientIntegrationTest.java index d508c9edb25..820ebffc1a8 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientIntegrationTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientIntegrationTest.java @@ -48,6 +48,8 @@ public class ZeppelinClientIntegrationTest extends AbstractTestRestApi { public static void setUp() throws Exception { System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HELIUM_REGISTRY.getVarName(), "helium"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ALLOWED_ORIGINS.getVarName(), "*"); + AbstractTestRestApi.startUp(ZeppelinClientIntegrationTest.class.getSimpleName()); notebook = TestUtils.getInstance(Notebook.class); @@ -228,14 +230,6 @@ public void testSubmitParagraph() throws Exception { } catch (Exception e) { assertTrue(e.getMessage(), e.getMessage().contains("No such paragraph")); } - - // cancel paragraph - paragraphId = zeppelinClient.addParagraph(noteId, "run sh", "%sh sleep 10"); - zeppelinClient.submitParagraph(noteId, paragraphId); - zeppelinClient.waitUtilParagraphRunning(noteId, paragraphId); - zeppelinClient.cancelParagraph(noteId, paragraphId); - paragraphResult = zeppelinClient.waitUtilParagraphFinish(noteId, paragraphId); - assertEquals(Status.ABORT, paragraphResult.getStatus()); } @Test diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java index 16c139128a3..9b0fc93cae9 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinClientWithAuthIntegrationTest.java @@ -49,6 +49,8 @@ public class ZeppelinClientWithAuthIntegrationTest extends AbstractTestRestApi { public static void setUp() throws Exception { System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HELIUM_REGISTRY.getVarName(), "helium"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ALLOWED_ORIGINS.getVarName(), "*"); + AbstractTestRestApi.startUpWithAuthenticationEnable(ZeppelinClientWithAuthIntegrationTest.class.getSimpleName()); notebook = TestUtils.getInstance(Notebook.class); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContext.java index a1485e3f291..4800619d0e9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ExecutionContext.java @@ -18,8 +18,8 @@ package org.apache.zeppelin.interpreter; /** - * Context info about running interpreter. This is used for deciding which interpreter binding - * mode to use. + * Context info about running interpreter. This is used for deciding bind to + * which interpreter progress(InterpreterGroup) */ public class ExecutionContext { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 36ec1bd65f9..54769be62ed 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -69,7 +69,7 @@ @Singleton public class InterpreterRestApi { - private static final Logger logger = LoggerFactory.getLogger(InterpreterRestApi.class); + private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterRestApi.class); private final AuthenticationService authenticationService; private final AuthorizationService authorizationService; @@ -116,7 +116,7 @@ public Response getSetting(@PathParam("settingId") String settingId) { return new JsonResponse<>(Status.OK, "", setting).build(); } } catch (NullPointerException e) { - logger.error("Exception in InterpreterRestApi while creating ", e); + LOGGER.error("Exception in InterpreterRestApi while creating ", e); return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); } @@ -141,10 +141,10 @@ public Response newSettings(String message) { InterpreterSetting interpreterSetting = interpreterSettingManager .createNewSetting(request.getName(), request.getGroup(), request.getDependencies(), request.getOption(), request.getProperties()); - logger.info("new setting created with {}", interpreterSetting.getId()); + LOGGER.info("new setting created with {}", interpreterSetting.getId()); return new JsonResponse<>(Status.OK, "", interpreterSetting).build(); } catch (IOException e) { - logger.error("Exception in InterpreterRestApi while creating ", e); + LOGGER.error("Exception in InterpreterRestApi while creating ", e); return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)) .build(); } @@ -154,7 +154,7 @@ public Response newSettings(String message) { @Path("setting/{settingId}") @ZeppelinApi public Response updateSetting(String message, @PathParam("settingId") String settingId) { - logger.info("Update interpreterSetting {}", settingId); + LOGGER.info("Update interpreterSetting {}", settingId); try { UpdateInterpreterSettingRequest request = @@ -163,11 +163,11 @@ public Response updateSetting(String message, @PathParam("settingId") String set .setPropertyAndRestart(settingId, request.getOption(), request.getProperties(), request.getDependencies()); } catch (InterpreterException e) { - logger.error("Exception in InterpreterRestApi while updateSetting ", e); + LOGGER.error("Exception in InterpreterRestApi while updateSetting ", e); return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)) .build(); } catch (IOException e) { - logger.error("Exception in InterpreterRestApi while updateSetting ", e); + LOGGER.error("Exception in InterpreterRestApi while updateSetting ", e); return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); } @@ -185,7 +185,7 @@ public Response updateSetting(String message, @PathParam("settingId") String set @Path("setting/{settingId}") @ZeppelinApi public Response removeSetting(@PathParam("settingId") String settingId) throws IOException { - logger.info("Remove interpreterSetting {}", settingId); + LOGGER.info("Remove interpreterSetting {}", settingId); interpreterSettingManager.remove(settingId); return new JsonResponse(Status.OK).build(); } @@ -197,7 +197,7 @@ public Response removeSetting(@PathParam("settingId") String settingId) throws I @Path("setting/restart/{settingId}") @ZeppelinApi public Response restartSetting(String message, @PathParam("settingId") String settingId) { - logger.info("Restart interpreterSetting {}, msg={}", settingId, message); + LOGGER.info("Restart interpreterSetting {}, msg={}", settingId, message); InterpreterSetting setting = interpreterSettingManager.get(settingId); try { @@ -224,7 +224,7 @@ public Response restartSetting(String message, @PathParam("settingId") String se } } } catch (InterpreterException e) { - logger.error("Exception in InterpreterRestApi while restartSetting ", e); + LOGGER.error("Exception in InterpreterRestApi while restartSetting ", e); return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)) .build(); } @@ -268,9 +268,9 @@ public Response addRepository(String message) { Repository request = Repository.fromJson(message); interpreterSettingManager.addRepository(request.getId(), request.getUrl(), request.isSnapshot(), request.getAuthentication(), request.getProxy()); - logger.info("New repository {} added", request.getId()); + LOGGER.info("New repository {} added", request.getId()); } catch (Exception e) { - logger.error("Exception in InterpreterRestApi while adding repository ", e); + LOGGER.error("Exception in InterpreterRestApi while adding repository ", e); return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); } @@ -286,11 +286,11 @@ public Response addRepository(String message) { @Path("repository/{repoId}") @ZeppelinApi public Response removeRepository(@PathParam("repoId") String repoId) { - logger.info("Remove repository {}", repoId); + LOGGER.info("Remove repository {}", repoId); try { interpreterSettingManager.removeRepository(repoId); } catch (Exception e) { - logger.error("Exception in InterpreterRestApi while removing repository ", e); + LOGGER.error("Exception in InterpreterRestApi while removing repository ", e); return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); } @@ -311,7 +311,7 @@ public Response listInterpreterPropertyTypes() { @Path("install") @ZeppelinApi public Response installInterpreter(@NotNull String message) { - logger.info("Install interpreter: {}", message); + LOGGER.info("Install interpreter: {}", message); InterpreterInstallationRequest request = InterpreterInstallationRequest.fromJson(message); try { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java index 6ac48d4863e..c5088a9111b 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java @@ -22,24 +22,32 @@ import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.ManagedInterpreterGroup; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.rest.message.SessionInfo; +import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +/** + * + * Backend manager of ZSessions + */ public class SessionManager { private static final Logger LOGGER = LoggerFactory.getLogger(SessionManager.class); private static final int RETRY = 3; - private Set sessions = new HashSet<>(); + private Map sessions = new HashMap<>(); private InterpreterSettingManager interpreterSettingManager; private Notebook notebook; @@ -48,25 +56,44 @@ public SessionManager(Notebook notebook, InterpreterSettingManager interpreterSe this.interpreterSettingManager = interpreterSettingManager; } - public synchronized String newSession(String interpreter) throws Exception { + /** + * Create a new session, including allocate new session Id and create dedicated note for this session. + * + * @param interpreter + * @return + * @throws Exception + */ + public synchronized SessionInfo createSession(String interpreter) throws Exception { + String sessionId = null; int i = 0; while (i < RETRY) { - String sessionId = interpreter + "_" + System.currentTimeMillis(); - if (sessions.contains(sessionId)) { + sessionId = interpreter + "_" + System.currentTimeMillis(); + if (sessions.containsKey(sessionId)) { try { Thread.sleep(1); } catch (InterruptedException e) { e.printStackTrace(); } } else { - sessions.add(sessionId); - return sessionId; + break; } } - throw new Exception("Unable to generate session id"); + if (sessionId == null) { + throw new Exception("Unable to generate session id"); + } + + Note sessionNote = notebook.createNote("/_ZSession/" + sessionId, AuthenticationInfo.ANONYMOUS); + SessionInfo sessionInfo = new SessionInfo(sessionId, sessionNote.getId(), interpreter); + sessions.put(sessionId, sessionInfo); + return sessionInfo; } + /** + * Remove and stop this session. + * + * @param sessionId + */ public void removeSession(String sessionId) { this.sessions.remove(sessionId); InterpreterGroup interpreterGroup = this.interpreterSettingManager.getInterpreterGroupById(sessionId); @@ -77,45 +104,49 @@ public void removeSession(String sessionId) { ((ManagedInterpreterGroup) interpreterGroup).getInterpreterSetting().closeInterpreters(sessionId); } + /** + * Get the sessionInfo. + * It method will also update its state if these's associated interpreter process. + * + * @param sessionId + * @return + * @throws Exception + */ public SessionInfo getSession(String sessionId) throws Exception { + SessionInfo sessionInfo = sessions.get(sessionId); + if (sessionInfo == null) { + LOGGER.warn("No such session: " + sessionId); + return null; + } InterpreterGroup interpreterGroup = this.interpreterSettingManager.getInterpreterGroupById(sessionId); if (interpreterGroup != null) { RemoteInterpreterProcess remoteInterpreterProcess = ((ManagedInterpreterGroup) interpreterGroup).getRemoteInterpreterProcess(); - String state = ""; - String startTime = ""; if (remoteInterpreterProcess == null) { - state = "Ready"; + sessionInfo.setState("Ready"); } else if (remoteInterpreterProcess != null) { - startTime = remoteInterpreterProcess.getStartTime(); + sessionInfo.setStartTime(remoteInterpreterProcess.getStartTime()); + sessionInfo.setWeburl(interpreterGroup.getWebUrl()); if (remoteInterpreterProcess.isRunning()) { - state = "Running"; + sessionInfo.setState("Running"); } else { - state = "Stopped"; - } - } - String noteId = ""; - String interpreter = ((ManagedInterpreterGroup) interpreterGroup).getInterpreterSetting().getName(); - String notePath = "/_ZSession/" + interpreter + "/" + sessionId; - List notesInfo = notebook.getNotesInfo().stream() - .filter(e -> e.getPath().equals(notePath)) - .collect(Collectors.toList()); - if (notesInfo.size() != 0) { - noteId = notesInfo.get(0).getId(); - if (notesInfo.size() > 1) { - LOGGER.warn("Found more than 1 notes with path: " + notePath); + sessionInfo.setState("Stopped"); } } - return new SessionInfo(sessionId, noteId, interpreter, - state, interpreterGroup.getWebUrl(), startTime); } - LOGGER.warn("No such session: " + sessionId); - return null; + + return sessionInfo; } + /** + * Get all sessions. + * + * @return + * @throws Exception + */ public List getAllSessions() throws Exception { List sessionList = new ArrayList<>(); - for (String sessionId : sessions) { + for (String sessionId : sessions.keySet()) { SessionInfo session = getSession(sessionId); if (session != null) { sessionList.add(session); @@ -124,11 +155,18 @@ public List getAllSessions() throws Exception { return sessionList; } - public List getAllSessions(String interpreterGroup) throws Exception { + /** + * Get all sessions for provided interpreter. + * + * @param interpreter + * @return + * @throws Exception + */ + public List getAllSessions(String interpreter) throws Exception { List sessionList = new ArrayList<>(); - for (String sessionId : sessions) { + for (String sessionId : sessions.keySet()) { SessionInfo status = getSession(sessionId); - if (status != null && interpreterGroup.equals(status.getInterpreter())) { + if (status != null && interpreter.equals(status.getInterpreter())) { sessionList.add(status); } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java index 2ab53aecb81..7a1ebac54a4 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java @@ -49,14 +49,10 @@ public class SessionRestApi { private static final Logger LOGGER = LoggerFactory.getLogger(SessionRestApi.class); - private Notebook notebook; - private InterpreterSettingManager interpreterSettingManager; private SessionManager sessionManager; @Inject public SessionRestApi(Notebook notebook, InterpreterSettingManager interpreterSettingManager) { - this.notebook = notebook; - this.interpreterSettingManager = interpreterSettingManager; this.sessionManager = new SessionManager(notebook, interpreterSettingManager); } @@ -83,13 +79,12 @@ public Response listSessions(@QueryParam("interpreter") String interpreter) { } @POST - @Path("{interpreter}") - public Response newSession(@PathParam("interpreter") String interpreter) { - LOGGER.info("New session for interpreter: {}", interpreter); + @Path("/") + public Response createSession(@QueryParam("interpreter") String interpreter) { + LOGGER.info("Create new session for interpreter: {}", interpreter); try { - String sessionId = sessionManager.newSession(interpreter); - LOGGER.info("Allocate new session id: " + sessionId); - return new JsonResponse<>(Response.Status.OK, sessionId).build(); + SessionInfo sessionInfo = sessionManager.createSession(interpreter); + return new JsonResponse<>(Response.Status.OK, sessionInfo).build(); } catch (Exception e) { return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, "Fail to start session", ExceptionUtils.getStackTrace(e)).build(); @@ -110,7 +105,7 @@ public Response stopSession(@PathParam("sessionId") String sessionId) { } /** - * Get a session info. + * Get session info for provided sessionId. */ @GET @Path("{sessionId}") diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java index 50617be8970..7d6a3a6d4a8 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java @@ -26,11 +26,12 @@ * NewNoteRequest rest api request message. */ public class NewNoteRequest implements JsonSerializable { - private static final Gson gson = new Gson(); + private static final Gson GSON = new Gson(); - String name; - String defaultInterpreterGroup; - List paragraphs; + //TODO(zjffdu) rename it to be notePath instead of name + private String name; + private String defaultInterpreterGroup; + private List paragraphs; public NewNoteRequest (){ } @@ -48,10 +49,10 @@ public List getParagraphs() { } public String toJson() { - return gson.toJson(this); + return GSON.toJson(this); } public static NewNoteRequest fromJson(String json) { - return gson.fromJson(json, NewNoteRequest.class); + return GSON.fromJson(json, NewNoteRequest.class); } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/RestartInterpreterRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/RestartInterpreterRequest.java index 8a6d0d06a16..f82cb9e9da1 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/RestartInterpreterRequest.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/RestartInterpreterRequest.java @@ -24,9 +24,9 @@ * RestartInterpreter rest api request message. */ public class RestartInterpreterRequest implements JsonSerializable { - private static final Gson gson = new Gson(); + private static final Gson GSON = new Gson(); - String noteId; + private String noteId; public RestartInterpreterRequest() { } @@ -36,10 +36,10 @@ public String getNoteId() { } public String toJson() { - return gson.toJson(this); + return GSON.toJson(this); } public static RestartInterpreterRequest fromJson(String json) { - return gson.fromJson(json, RestartInterpreterRequest.class); + return GSON.fromJson(json, RestartInterpreterRequest.class); } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/SessionInfo.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/SessionInfo.java index 5ddf4913247..8e8f2abbc28 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/SessionInfo.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/SessionInfo.java @@ -17,6 +17,9 @@ package org.apache.zeppelin.rest.message; +/** + * (TODO) Move it to zeppelin-common, so that we don't need to have 2 copies. + */ public class SessionInfo { private String sessionId; @@ -26,6 +29,16 @@ public class SessionInfo { private String weburl; private String startTime; + public SessionInfo(String sessionId) { + this.sessionId = sessionId; + } + + public SessionInfo(String sessionId, String noteId, String interpreter) { + this.sessionId = sessionId; + this.noteId = noteId; + this.interpreter = interpreter; + } + public SessionInfo(String sessionId, String noteId, String interpreter, String state, String weburl, String startTime) { this.sessionId = sessionId; this.noteId = noteId; @@ -58,4 +71,16 @@ public String getWeburl() { public String getStartTime() { return startTime; } + + public void setState(String state) { + this.state = state; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public void setWeburl(String weburl) { + this.weburl = weburl; + } } From 8eddf3e09898b1af5df535c692cc637920f34e9a Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Wed, 2 Sep 2020 22:41:55 +0800 Subject: [PATCH 18/19] use ExceptionMapper --- .travis.yml | 4 +- .../apache/zeppelin/client/ClientConfig.java | 10 +- .../apache/zeppelin/rest/NotebookRestApi.java | 900 ++++++++---------- .../apache/zeppelin/rest/SessionManager.java | 2 +- .../apache/zeppelin/rest/SessionRestApi.java | 57 +- .../WebApplicationExceptionMapper.java | 12 +- .../message/gson/ExceptionSerializer.java | 3 + 7 files changed, 430 insertions(+), 558 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9ba1ef6e3e6..b7454e624a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -224,7 +224,7 @@ jobs: - bash -x ./testing/install_external_dependencies.sh - source ~/.environ install: - - mvn install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark/spark-dependencies,markdown,flink/interpreter,jdbc,shell -am + - mvn install -DskipTests -DskipRat -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark/spark-dependencies,markdown,flink/interpreter,jdbc,shell -am - mvn clean package -T 2C -pl zeppelin-plugins -amd -B before_script: - echo "export ZEPPELIN_HELIUM_REGISTRY=helium" >> conf/zeppelin-env.sh @@ -232,7 +232,7 @@ jobs: - export SPARK_PRINT_LAUNCH_COMMAND=true - tail conf/zeppelin-env.sh script: - - mvn test -DskipRat -pl zeppelin-interpreter-integration -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest + - mvn test -DskipRat -pl zeppelin-interpreter-integration -Pintegration -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest - name: "Test flink 1.10 & flink integration test" diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ClientConfig.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ClientConfig.java index aaf84fd9747..527e03302f8 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ClientConfig.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ClientConfig.java @@ -35,11 +35,19 @@ public ClientConfig(String zeppelinRestUrl, long queryInterval) { } public ClientConfig(String zeppelinRestUrl, long queryInterval, boolean useKnox) { - this.zeppelinRestUrl = zeppelinRestUrl; + this.zeppelinRestUrl = removeTrailingSlash(zeppelinRestUrl); this.queryInterval = queryInterval; this.useKnox = useKnox; } + private String removeTrailingSlash(String zeppelinRestUrl) { + if (zeppelinRestUrl.endsWith("/")) { + return zeppelinRestUrl.substring(0, zeppelinRestUrl.length() - 1); + } else { + return zeppelinRestUrl; + } + } + public String getZeppelinRestUrl() { return zeppelinRestUrl; } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 915f69f90e9..30096761ca3 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -127,19 +127,15 @@ public NotebookRestApi( @Path("{noteId}/permissions") @ZeppelinApi public Response getNotePermissions(@PathParam("noteId") String noteId) throws IOException { - try { - checkIfUserIsAnon(getBlockNotAuthenticatedUserErrorMsg()); - checkIfUserCanRead(noteId, - "Insufficient privileges you cannot get the list of permissions for this note"); - HashMap> permissionsMap = new HashMap<>(); - permissionsMap.put("owners", authorizationService.getOwners(noteId)); - permissionsMap.put("readers", authorizationService.getReaders(noteId)); - permissionsMap.put("writers", authorizationService.getWriters(noteId)); - permissionsMap.put("runners", authorizationService.getRunners(noteId)); - return new JsonResponse<>(Status.OK, "", permissionsMap).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + checkIfUserIsAnon(getBlockNotAuthenticatedUserErrorMsg()); + checkIfUserCanRead(noteId, + "Insufficient privileges you cannot get the list of permissions for this note"); + HashMap> permissionsMap = new HashMap<>(); + permissionsMap.put("owners", authorizationService.getOwners(noteId)); + permissionsMap.put("readers", authorizationService.getReaders(noteId)); + permissionsMap.put("writers", authorizationService.getWriters(noteId)); + permissionsMap.put("runners", authorizationService.getRunners(noteId)); + return new JsonResponse<>(Status.OK, "", permissionsMap).build(); } private String ownerPermissionError(Set current, Set allowed) { @@ -246,74 +242,71 @@ private void checkIfParagraphIsNotNull(Paragraph paragraph) { @ZeppelinApi public Response putNotePermissions(@PathParam("noteId") String noteId, String req) throws IOException { - try { - String principal = authenticationService.getPrincipal(); - Set roles = authenticationService.getAssociatedRoles(); - HashSet userAndRoles = new HashSet<>(); - userAndRoles.add(principal); - userAndRoles.addAll(roles); - - checkIfUserIsAnon(getBlockNotAuthenticatedUserErrorMsg()); - checkIfUserIsOwner(noteId, - ownerPermissionError(userAndRoles, authorizationService.getOwners(noteId))); - - HashMap> permMap = - GSON.fromJson(req, new TypeToken>>() { - }.getType()); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - - HashSet readers = permMap.get("readers"); - HashSet runners = permMap.get("runners"); - HashSet owners = permMap.get("owners"); - HashSet writers = permMap.get("writers"); - - LOGGER.info("Set permissions to note: {} with current user:{}, owners:{}, readers:{}, runners:{}, writers:{}", - noteId, principal, owners, readers, runners, writers); - - // Set readers, if runners, writers and owners is empty -> set to user requesting the change - if (readers != null && !readers.isEmpty()) { - if (runners.isEmpty()) { - runners = Sets.newHashSet(authenticationService.getPrincipal()); - } - if (writers.isEmpty()) { - writers = Sets.newHashSet(authenticationService.getPrincipal()); - } - if (owners.isEmpty()) { - owners = Sets.newHashSet(authenticationService.getPrincipal()); - } + + String principal = authenticationService.getPrincipal(); + Set roles = authenticationService.getAssociatedRoles(); + HashSet userAndRoles = new HashSet<>(); + userAndRoles.add(principal); + userAndRoles.addAll(roles); + + checkIfUserIsAnon(getBlockNotAuthenticatedUserErrorMsg()); + checkIfUserIsOwner(noteId, + ownerPermissionError(userAndRoles, authorizationService.getOwners(noteId))); + + HashMap> permMap = + GSON.fromJson(req, new TypeToken>>() { + }.getType()); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + + HashSet readers = permMap.get("readers"); + HashSet runners = permMap.get("runners"); + HashSet owners = permMap.get("owners"); + HashSet writers = permMap.get("writers"); + + LOGGER.info("Set permissions to note: {} with current user:{}, owners:{}, readers:{}, runners:{}, writers:{}", + noteId, principal, owners, readers, runners, writers); + + // Set readers, if runners, writers and owners is empty -> set to user requesting the change + if (readers != null && !readers.isEmpty()) { + if (runners.isEmpty()) { + runners = Sets.newHashSet(authenticationService.getPrincipal()); } - // Set runners, if writers and owners is empty -> set to user requesting the change - if (runners != null && !runners.isEmpty()) { - if (writers.isEmpty()) { - writers = Sets.newHashSet(authenticationService.getPrincipal()); - } - if (owners.isEmpty()) { - owners = Sets.newHashSet(authenticationService.getPrincipal()); - } + if (writers.isEmpty()) { + writers = Sets.newHashSet(authenticationService.getPrincipal()); } - // Set writers, if owners is empty -> set to user requesting the change - if (writers != null && !writers.isEmpty()) { - if (owners.isEmpty()) { - owners = Sets.newHashSet(authenticationService.getPrincipal()); - } + if (owners.isEmpty()) { + owners = Sets.newHashSet(authenticationService.getPrincipal()); + } + } + // Set runners, if writers and owners is empty -> set to user requesting the change + if (runners != null && !runners.isEmpty()) { + if (writers.isEmpty()) { + writers = Sets.newHashSet(authenticationService.getPrincipal()); + } + if (owners.isEmpty()) { + owners = Sets.newHashSet(authenticationService.getPrincipal()); } - - authorizationService.setReaders(noteId, readers); - authorizationService.setRunners(noteId, runners); - authorizationService.setWriters(noteId, writers); - authorizationService.setOwners(noteId, owners); - LOGGER.debug("After set permissions {} {} {} {}", authorizationService.getOwners(noteId), - authorizationService.getReaders(noteId), authorizationService.getRunners(noteId), - authorizationService.getWriters(noteId)); - AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); - authorizationService.saveNoteAuth(noteId, subject); - notebookServer.broadcastNote(note); - notebookServer.broadcastNoteList(subject, userAndRoles); - return new JsonResponse<>(Status.OK).build(); - } catch (Throwable e) { - return createResponseFromException(e); } + // Set writers, if owners is empty -> set to user requesting the change + if (writers != null && !writers.isEmpty()) { + if (owners.isEmpty()) { + owners = Sets.newHashSet(authenticationService.getPrincipal()); + } + } + + authorizationService.setReaders(noteId, readers); + authorizationService.setRunners(noteId, runners); + authorizationService.setWriters(noteId, writers); + authorizationService.setOwners(noteId, owners); + LOGGER.debug("After set permissions {} {} {} {}", authorizationService.getOwners(noteId), + authorizationService.getReaders(noteId), authorizationService.getRunners(noteId), + authorizationService.getWriters(noteId)); + AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); + authorizationService.saveNoteAuth(noteId, subject); + notebookServer.broadcastNote(note); + notebookServer.broadcastNoteList(subject, userAndRoles); + return new JsonResponse<>(Status.OK).build(); } /** @@ -325,13 +318,9 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re @GET @ZeppelinApi public Response getNoteList() throws IOException { - try { - List notesInfo = notebookService.listNotesInfo(false, getServiceContext(), - new RestServiceCallback>()); - return new JsonResponse<>(Status.OK, "", notesInfo).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + List notesInfo = notebookService.listNotesInfo(false, getServiceContext(), + new RestServiceCallback>()); + return new JsonResponse<>(Status.OK, "", notesInfo).build(); } /** @@ -345,13 +334,9 @@ public Response getNoteList() throws IOException { @Path("{noteId}") @ZeppelinApi public Response getNote(@PathParam("noteId") String noteId) throws IOException { - try { - Note note = - notebookService.getNote(noteId, getServiceContext(), new RestServiceCallback()); - return new JsonResponse<>(Status.OK, "", note).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + Note note = + notebookService.getNote(noteId, getServiceContext(), new RestServiceCallback()); + return new JsonResponse<>(Status.OK, "", note).build(); } /** @@ -365,13 +350,9 @@ public Response getNote(@PathParam("noteId") String noteId) throws IOException { @Path("export/{noteId}") @ZeppelinApi public Response exportNote(@PathParam("noteId") String noteId) throws IOException { - try { - checkIfUserCanRead(noteId, "Insufficient privileges you cannot export this note"); - String exportJson = notebook.exportNote(noteId); - return new JsonResponse<>(Status.OK, "", exportJson).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + checkIfUserCanRead(noteId, "Insufficient privileges you cannot export this note"); + String exportJson = notebook.exportNote(noteId); + return new JsonResponse<>(Status.OK, "", exportJson).build(); } /** @@ -386,13 +367,9 @@ public Response exportNote(@PathParam("noteId") String noteId) throws IOExceptio @Path("import") @ZeppelinApi public Response importNote(@QueryParam("notePath") String notePath, String noteJson) throws IOException { - try { - Note note = notebookService.importNote(notePath, noteJson, getServiceContext(), - new RestServiceCallback()); - return new JsonResponse<>(Status.OK, "", note.getId()).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + Note note = notebookService.importNote(notePath, noteJson, getServiceContext(), + new RestServiceCallback()); + return new JsonResponse<>(Status.OK, "", note.getId()).build(); } /** @@ -405,31 +382,27 @@ public Response importNote(@QueryParam("notePath") String notePath, String noteJ @POST @ZeppelinApi public Response createNote(String message) throws IOException { - try { - String user = authenticationService.getPrincipal(); - LOGGER.info("Creating new note by JSON {}", message); - NewNoteRequest request = NewNoteRequest.fromJson(message); - String defaultInterpreterGroup = request.getDefaultInterpreterGroup(); - if (StringUtils.isBlank(defaultInterpreterGroup)) { - defaultInterpreterGroup = zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_DEFAULT); - } - Note note = notebookService.createNote( - request.getName(), - defaultInterpreterGroup, - false, - getServiceContext(), - new RestServiceCallback<>()); - AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); - if (request.getParagraphs() != null) { - for (NewParagraphRequest paragraphRequest : request.getParagraphs()) { - Paragraph p = note.addNewParagraph(subject); - initParagraph(p, paragraphRequest, user); - } + String user = authenticationService.getPrincipal(); + LOGGER.info("Creating new note by JSON {}", message); + NewNoteRequest request = NewNoteRequest.fromJson(message); + String defaultInterpreterGroup = request.getDefaultInterpreterGroup(); + if (StringUtils.isBlank(defaultInterpreterGroup)) { + defaultInterpreterGroup = zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_DEFAULT); + } + Note note = notebookService.createNote( + request.getName(), + defaultInterpreterGroup, + false, + getServiceContext(), + new RestServiceCallback<>()); + AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); + if (request.getParagraphs() != null) { + for (NewParagraphRequest paragraphRequest : request.getParagraphs()) { + Paragraph p = note.addNewParagraph(subject); + initParagraph(p, paragraphRequest, user); } - return new JsonResponse(Status.OK, "", note.getId()).build(); - } catch (Throwable e) { - return createResponseFromException(e); } + return new JsonResponse(Status.OK, "", note.getId()).build(); } /** @@ -444,20 +417,16 @@ public Response createNote(String message) throws IOException { @ZeppelinApi public Response deleteNote(@PathParam("noteId") String noteId) throws IOException { LOGGER.info("Delete note {} ", noteId); - try { - notebookService.removeNote(noteId, - getServiceContext(), - new RestServiceCallback() { - @Override - public void onSuccess(String message, ServiceContext context) { - notebookServer.broadcastNoteList(context.getAutheInfo(), context.getUserAndRoles()); - } - }); - - return new JsonResponse<>(Status.OK, "").build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + notebookService.removeNote(noteId, + getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(String message, ServiceContext context) { + notebookServer.broadcastNoteList(context.getAutheInfo(), context.getUserAndRoles()); + } + }); + + return new JsonResponse<>(Status.OK, "").build(); } /** @@ -474,27 +443,24 @@ public void onSuccess(String message, ServiceContext context) { @ZeppelinApi public Response cloneNote(@PathParam("noteId") String noteId, String message) throws IOException, IllegalArgumentException { - try { - LOGGER.info("Clone note by JSON {}", message); - checkIfUserCanWrite(noteId, "Insufficient privileges you cannot clone this note"); - NewNoteRequest request = NewNoteRequest.fromJson(message); - String newNoteName = null; - if (request != null) { - newNoteName = request.getName(); - } - AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); - Note newNote = notebookService.cloneNote(noteId, newNoteName, getServiceContext(), - new RestServiceCallback() { - @Override - public void onSuccess(Note newNote, ServiceContext context) throws IOException { - notebookServer.broadcastNote(newNote); - notebookServer.broadcastNoteList(subject, context.getUserAndRoles()); - } - }); - return new JsonResponse<>(Status.OK, "", newNote.getId()).build(); - } catch (Throwable e) { - return createResponseFromException(e); + + LOGGER.info("Clone note by JSON {}", message); + checkIfUserCanWrite(noteId, "Insufficient privileges you cannot clone this note"); + NewNoteRequest request = NewNoteRequest.fromJson(message); + String newNoteName = null; + if (request != null) { + newNoteName = request.getName(); } + AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); + Note newNote = notebookService.cloneNote(noteId, newNoteName, getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(Note newNote, ServiceContext context) throws IOException { + notebookServer.broadcastNote(newNote); + notebookServer.broadcastNoteList(subject, context.getUserAndRoles()); + } + }); + return new JsonResponse<>(Status.OK, "", newNote.getId()).build(); } /** @@ -509,26 +475,23 @@ public void onSuccess(Note newNote, ServiceContext context) throws IOException { @ZeppelinApi public Response renameNote(@PathParam("noteId") String noteId, String message) throws IOException { - try { - LOGGER.info("Rename note by JSON {}", message); - RenameNoteRequest request = GSON.fromJson(message, RenameNoteRequest.class); - String newName = request.getName(); - if (newName.isEmpty()) { - LOGGER.warn("Trying to rename notebook {} with empty name parameter", noteId); - throw new BadRequestException("name can not be empty"); - } - notebookService.renameNote(noteId, request.getName(), false, getServiceContext(), - new RestServiceCallback() { - @Override - public void onSuccess(Note note, ServiceContext context) throws IOException { - notebookServer.broadcastNote(note); - notebookServer.broadcastNoteList(context.getAutheInfo(), context.getUserAndRoles()); - } - }); - return new JsonResponse(Status.OK, "").build(); - } catch (Throwable e) { - return createResponseFromException(e); + + LOGGER.info("Rename note by JSON {}", message); + RenameNoteRequest request = GSON.fromJson(message, RenameNoteRequest.class); + String newName = request.getName(); + if (newName.isEmpty()) { + LOGGER.warn("Trying to rename notebook {} with empty name parameter", noteId); + throw new BadRequestException("name can not be empty"); } + notebookService.renameNote(noteId, request.getName(), false, getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(Note note, ServiceContext context) throws IOException { + notebookServer.broadcastNote(note); + notebookServer.broadcastNoteList(context.getAutheInfo(), context.getUserAndRoles()); + } + }); + return new JsonResponse(Status.OK, "").build(); } /** @@ -543,30 +506,27 @@ public void onSuccess(Note note, ServiceContext context) throws IOException { @ZeppelinApi public Response insertParagraph(@PathParam("noteId") String noteId, String message) throws IOException { - try { - String user = authenticationService.getPrincipal(); - LOGGER.info("Insert paragraph {} {}", noteId, message); - - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanWrite(noteId, "Insufficient privileges you cannot add paragraph to this note"); - - NewParagraphRequest request = NewParagraphRequest.fromJson(message); - AuthenticationInfo subject = new AuthenticationInfo(user); - Paragraph p; - Double indexDouble = request.getIndex(); - if (indexDouble == null) { - p = note.addNewParagraph(subject); - } else { - p = note.insertNewParagraph(indexDouble.intValue(), subject); - } - initParagraph(p, request, user); - notebook.saveNote(note, subject); - notebookServer.broadcastNote(note); - return new JsonResponse<>(Status.OK, "", p.getId()).build(); - } catch (Throwable e) { - return createResponseFromException(e); + + String user = authenticationService.getPrincipal(); + LOGGER.info("Insert paragraph {} {}", noteId, message); + + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanWrite(noteId, "Insufficient privileges you cannot add paragraph to this note"); + + NewParagraphRequest request = NewParagraphRequest.fromJson(message); + AuthenticationInfo subject = new AuthenticationInfo(user); + Paragraph p; + Double indexDouble = request.getIndex(); + if (indexDouble == null) { + p = note.addNewParagraph(subject); + } else { + p = note.insertNewParagraph(indexDouble.intValue(), subject); } + initParagraph(p, request, user); + notebook.saveNote(note, subject); + notebookServer.broadcastNote(note); + return new JsonResponse<>(Status.OK, "", p.getId()).build(); } /** @@ -581,19 +541,15 @@ public Response insertParagraph(@PathParam("noteId") String noteId, String messa @ZeppelinApi public Response getParagraph(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId) throws IOException { - try { - LOGGER.info("Get paragraph {} {}", noteId, paragraphId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRead(noteId, "Insufficient privileges you cannot get this paragraph"); - Paragraph p = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(p); + LOGGER.info("Get paragraph {} {}", noteId, paragraphId); - return new JsonResponse<>(Status.OK, "", p).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRead(noteId, "Insufficient privileges you cannot get this paragraph"); + Paragraph p = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(p); + return new JsonResponse<>(Status.OK, "", p).build(); } /** @@ -608,29 +564,26 @@ public Response getParagraph(@PathParam("noteId") String noteId, public Response updateParagraph(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId, String message) throws IOException { - try { - String user = authenticationService.getPrincipal(); - LOGGER.info("{} will update paragraph {} {}", user, noteId, paragraphId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph"); - Paragraph p = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(p); - UpdateParagraphRequest updatedParagraph = GSON.fromJson(message, UpdateParagraphRequest.class); - p.setText(updatedParagraph.getText()); + String user = authenticationService.getPrincipal(); + LOGGER.info("{} will update paragraph {} {}", user, noteId, paragraphId); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph"); + Paragraph p = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(p); - if (updatedParagraph.getTitle() != null) { - p.setTitle(updatedParagraph.getTitle()); - } + UpdateParagraphRequest updatedParagraph = GSON.fromJson(message, UpdateParagraphRequest.class); + p.setText(updatedParagraph.getText()); - AuthenticationInfo subject = new AuthenticationInfo(user); - notebook.saveNote(note, subject); - notebookServer.broadcastParagraph(note, p, MSG_ID_NOT_DEFINED); - return new JsonResponse<>(Status.OK, "").build(); - } catch (Throwable e) { - return createResponseFromException(e); + if (updatedParagraph.getTitle() != null) { + p.setTitle(updatedParagraph.getTitle()); } + + AuthenticationInfo subject = new AuthenticationInfo(user); + notebook.saveNote(note, subject); + notebookServer.broadcastParagraph(note, p, MSG_ID_NOT_DEFINED); + return new JsonResponse<>(Status.OK, "").build(); } /** @@ -648,24 +601,21 @@ public Response updateParagraph(@PathParam("noteId") String noteId, public Response updateParagraphConfig(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId, String message) throws IOException { - try { - String user = authenticationService.getPrincipal(); - LOGGER.info("{} will update paragraph config {} {}", user, noteId, paragraphId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph config"); - Paragraph p = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(p); - - Map newConfig = GSON.fromJson(message, HashMap.class); - configureParagraph(p, newConfig, user); - AuthenticationInfo subject = new AuthenticationInfo(user); - notebook.saveNote(note, subject); - return new JsonResponse<>(Status.OK, "", p).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + String user = authenticationService.getPrincipal(); + LOGGER.info("{} will update paragraph config {} {}", user, noteId, paragraphId); + + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph config"); + Paragraph p = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(p); + + Map newConfig = GSON.fromJson(message, HashMap.class); + configureParagraph(p, newConfig, user); + AuthenticationInfo subject = new AuthenticationInfo(user); + notebook.saveNote(note, subject); + return new JsonResponse<>(Status.OK, "", p).build(); } /** @@ -683,20 +633,16 @@ public Response moveParagraph(@PathParam("noteId") String noteId, @PathParam("newIndex") String newIndex) throws IOException { - try { - LOGGER.info("Move paragraph {} {} {}", noteId, paragraphId, newIndex); - notebookService.moveParagraph(noteId, paragraphId, Integer.parseInt(newIndex), - getServiceContext(), - new RestServiceCallback() { - @Override - public void onSuccess(Paragraph result, ServiceContext context) throws IOException { - notebookServer.broadcastNote(result.getNote()); - } - }); - return new JsonResponse(Status.OK, "").build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + LOGGER.info("Move paragraph {} {} {}", noteId, paragraphId, newIndex); + notebookService.moveParagraph(noteId, paragraphId, Integer.parseInt(newIndex), + getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(Paragraph result, ServiceContext context) throws IOException { + notebookServer.broadcastNote(result.getNote()); + } + }); + return new JsonResponse(Status.OK, "").build(); } /** @@ -711,34 +657,28 @@ public void onSuccess(Paragraph result, ServiceContext context) throws IOExcepti @ZeppelinApi public Response deleteParagraph(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId) throws IOException { - try { - LOGGER.info("Delete paragraph {} {}", noteId, paragraphId); - notebookService.removeParagraph(noteId, paragraphId, getServiceContext(), - new RestServiceCallback() { - @Override - public void onSuccess(Paragraph p, ServiceContext context) throws IOException { - notebookServer.broadcastNote(p.getNote()); - } - }); - - return new JsonResponse(Status.OK, "").build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + + LOGGER.info("Delete paragraph {} {}", noteId, paragraphId); + notebookService.removeParagraph(noteId, paragraphId, getServiceContext(), + new RestServiceCallback() { + @Override + public void onSuccess(Paragraph p, ServiceContext context) throws IOException { + notebookServer.broadcastNote(p.getNote()); + } + }); + + return new JsonResponse(Status.OK, "").build(); } @POST @Path("{noteId}/paragraph/next") public Response nextSessionParagraph(@PathParam("noteId") String noteId, - @QueryParam("maxParagraph") int maxParagraph) { - try { - Paragraph p = notebookService.getNextSessionParagraph(noteId, maxParagraph, - getServiceContext(), - new RestServiceCallback<>()); - return new JsonResponse(Status.OK, p.getId()).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + @QueryParam("maxParagraph") int maxParagraph) throws IOException { + + Paragraph p = notebookService.getNextSessionParagraph(noteId, maxParagraph, + getServiceContext(), + new RestServiceCallback<>()); + return new JsonResponse(Status.OK, p.getId()).build(); } /** @@ -752,14 +692,10 @@ public Response nextSessionParagraph(@PathParam("noteId") String noteId, @ZeppelinApi public Response clearAllParagraphOutput(@PathParam("noteId") String noteId) throws IOException { - try { - LOGGER.info("Clear all paragraph output of note {}", noteId); - notebookService.clearAllParagraphOutput(noteId, getServiceContext(), - new RestServiceCallback<>()); - return new JsonResponse(Status.OK, "").build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + LOGGER.info("Clear all paragraph output of note {}", noteId); + notebookService.clearAllParagraphOutput(noteId, getServiceContext(), + new RestServiceCallback<>()); + return new JsonResponse(Status.OK, "").build(); } /** @@ -780,36 +716,31 @@ public Response runNoteJobs(@PathParam("noteId") String noteId, @QueryParam("blocking") Boolean blocking, @QueryParam("isolated") Boolean isolated, String message) - throws IOException, IllegalArgumentException { - - try { - if (blocking == null) { - blocking = false; - } - if (isolated == null) { - isolated = false; - } - Map params = new HashMap<>(); - if (!StringUtils.isEmpty(message)) { - ParametersRequest request = - ParametersRequest.fromJson(message); - params = request.getParams(); - } + throws Exception, IllegalArgumentException { - LOGGER.info("Run note jobs, noteId: {}, blocking: {}, isolated: {}, params: {}", noteId, blocking, isolated, params); - Note note = notebook.getNote(noteId); - AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); - subject.setRoles(new LinkedList<>(authenticationService.getAssociatedRoles())); - checkIfNoteIsNotNull(note); - checkIfUserCanRun(noteId, "Insufficient privileges you cannot run job for this note"); - - //TODO(zjffdu), can we run a note via rest api when cron is enabled ? - note.runAll(subject, blocking, isolated, params); - return new JsonResponse<>(Status.OK).build(); - } catch (Throwable e) { - LOGGER.error("Exception from run", e); - return createResponseFromException(e); + if (blocking == null) { + blocking = false; + } + if (isolated == null) { + isolated = false; } + Map params = new HashMap<>(); + if (!StringUtils.isEmpty(message)) { + ParametersRequest request = + ParametersRequest.fromJson(message); + params = request.getParams(); + } + + LOGGER.info("Run note jobs, noteId: {}, blocking: {}, isolated: {}, params: {}", noteId, blocking, isolated, params); + Note note = notebook.getNote(noteId); + AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); + subject.setRoles(new LinkedList<>(authenticationService.getAssociatedRoles())); + checkIfNoteIsNotNull(note); + checkIfUserCanRun(noteId, "Insufficient privileges you cannot run job for this note"); + + //TODO(zjffdu), can we run a note via rest api when cron is enabled ? + note.runAll(subject, blocking, isolated, params); + return new JsonResponse<>(Status.OK).build(); } /** @@ -825,21 +756,18 @@ public Response runNoteJobs(@PathParam("noteId") String noteId, @ZeppelinApi public Response stopNoteJobs(@PathParam("noteId") String noteId) throws IOException, IllegalArgumentException { - try { - LOGGER.info("Stop note jobs {} ", noteId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRun(noteId, "Insufficient privileges you cannot stop this job for this note"); - - for (Paragraph p : note.getParagraphs()) { - if (!p.isTerminated()) { - p.abort(); - } + + LOGGER.info("Stop note jobs {} ", noteId); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRun(noteId, "Insufficient privileges you cannot stop this job for this note"); + + for (Paragraph p : note.getParagraphs()) { + if (!p.isTerminated()) { + p.abort(); } - return new JsonResponse<>(Status.OK).build(); - } catch (Throwable e) { - return createResponseFromException(e); } + return new JsonResponse<>(Status.OK).build(); } /** @@ -855,16 +783,13 @@ public Response stopNoteJobs(@PathParam("noteId") String noteId) @ZeppelinApi public Response getNoteJobStatus(@PathParam("noteId") String noteId) throws IOException, IllegalArgumentException { - try { - LOGGER.info("Get note job status."); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status"); - - return new JsonResponse<>(Status.OK, null, new NoteJobStatus(note)).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + + LOGGER.info("Get note job status."); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status"); + + return new JsonResponse<>(Status.OK, null, new NoteJobStatus(note)).build(); } /** @@ -882,19 +807,16 @@ public Response getNoteJobStatus(@PathParam("noteId") String noteId) public Response getNoteParagraphJobStatus(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException { - try { - LOGGER.info("Get note paragraph job status."); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status"); - - Paragraph paragraph = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(paragraph); - - return new JsonResponse<>(Status.OK, null, new ParagraphJobStatus(paragraph)).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + + LOGGER.info("Get note paragraph job status."); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status"); + + Paragraph paragraph = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(paragraph); + + return new JsonResponse<>(Status.OK, null, new ParagraphJobStatus(paragraph)).build(); } /** @@ -915,27 +837,23 @@ public Response runParagraph(@PathParam("noteId") String noteId, String message) throws IOException, IllegalArgumentException { - try { - LOGGER.info("Run paragraph job asynchronously {} {} {}", noteId, paragraphId, message); + LOGGER.info("Run paragraph job asynchronously {} {} {}", noteId, paragraphId, message); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - Paragraph paragraph = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(paragraph); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + Paragraph paragraph = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(paragraph); - Map params = new HashMap<>(); - if (!StringUtils.isEmpty(message)) { - ParametersRequest request = - ParametersRequest.fromJson(message); - params = request.getParams(); - } - notebookService.runParagraph(noteId, paragraphId, paragraph.getTitle(), - paragraph.getText(), params, new HashMap<>(), sessionId, - false, false, getServiceContext(), new RestServiceCallback<>()); - return new JsonResponse<>(Status.OK).build(); - } catch (Throwable e) { - return createResponseFromException(e); + Map params = new HashMap<>(); + if (!StringUtils.isEmpty(message)) { + ParametersRequest request = + ParametersRequest.fromJson(message); + params = request.getParams(); } + notebookService.runParagraph(noteId, paragraphId, paragraph.getTitle(), + paragraph.getText(), params, new HashMap<>(), sessionId, + false, false, getServiceContext(), new RestServiceCallback<>()); + return new JsonResponse<>(Status.OK).build(); } /** @@ -959,31 +877,27 @@ public Response runParagraphSynchronously(@PathParam("noteId") String noteId, throws IOException, IllegalArgumentException { LOGGER.info("Run paragraph synchronously {} {} {}", noteId, paragraphId, message); - try { - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - Paragraph paragraph = note.getParagraph(paragraphId); - checkIfParagraphIsNotNull(paragraph); - - Map params = new HashMap<>(); - if (!StringUtils.isEmpty(message)) { - ParametersRequest request = - ParametersRequest.fromJson(message); - params = request.getParams(); - } + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + Paragraph paragraph = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(paragraph); - if (notebookService.runParagraph(noteId, paragraphId, paragraph.getTitle(), - paragraph.getText(), params, - new HashMap<>(), sessionId, false, true, getServiceContext(), new RestServiceCallback<>())) { - note = notebookService.getNote(noteId, getServiceContext(), new RestServiceCallback<>()); - Paragraph p = note.getParagraph(paragraphId); - InterpreterResult result = p.getReturn(); - return new JsonResponse<>(Status.OK, result).build(); - } else { - return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, "Fail to run paragraph").build(); - } - } catch (Throwable e) { - return createResponseFromException(e); + Map params = new HashMap<>(); + if (!StringUtils.isEmpty(message)) { + ParametersRequest request = + ParametersRequest.fromJson(message); + params = request.getParams(); + } + + if (notebookService.runParagraph(noteId, paragraphId, paragraph.getTitle(), + paragraph.getText(), params, + new HashMap<>(), sessionId, false, true, getServiceContext(), new RestServiceCallback<>())) { + note = notebookService.getNote(noteId, getServiceContext(), new RestServiceCallback<>()); + Paragraph p = note.getParagraph(paragraphId); + InterpreterResult result = p.getReturn(); + return new JsonResponse<>(Status.OK, result).build(); + } else { + return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, "Fail to run paragraph").build(); } } @@ -1002,14 +916,10 @@ public Response runParagraphSynchronously(@PathParam("noteId") String noteId, public Response cancelParagraph(@PathParam("noteId") String noteId, @PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException { - try { - LOGGER.info("stop paragraph job {} ", noteId); - notebookService.cancelParagraph(noteId, paragraphId, getServiceContext(), - new RestServiceCallback()); - return new JsonResponse<>(Status.OK).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + LOGGER.info("stop paragraph job {} ", noteId); + notebookService.cancelParagraph(noteId, paragraphId, getServiceContext(), + new RestServiceCallback()); + return new JsonResponse<>(Status.OK).build(); } /** @@ -1026,30 +936,26 @@ public Response cancelParagraph(@PathParam("noteId") String noteId, public Response registerCronJob(@PathParam("noteId") String noteId, String message) throws IOException, IllegalArgumentException { - try { - LOGGER.info("Register cron job note={} request cron msg={}", noteId, message); + LOGGER.info("Register cron job note={} request cron msg={}", noteId, message); - CronRequest request = CronRequest.fromJson(message); + CronRequest request = CronRequest.fromJson(message); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRun(noteId, "Insufficient privileges you cannot set a cron job for this note"); - checkIfNoteSupportsCron(note); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRun(noteId, "Insufficient privileges you cannot set a cron job for this note"); + checkIfNoteSupportsCron(note); - if (!CronExpression.isValidExpression(request.getCronString())) { - return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build(); - } + if (!CronExpression.isValidExpression(request.getCronString())) { + return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build(); + } - Map config = note.getConfig(); - config.put("cron", request.getCronString()); - config.put("releaseresource", request.getReleaseResource()); - note.setConfig(config); - schedulerService.refreshCron(note.getId()); + Map config = note.getConfig(); + config.put("cron", request.getCronString()); + config.put("releaseresource", request.getReleaseResource()); + note.setConfig(config); + schedulerService.refreshCron(note.getId()); - return new JsonResponse<>(Status.OK).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + return new JsonResponse<>(Status.OK).build(); } /** @@ -1066,25 +972,21 @@ public Response registerCronJob(@PathParam("noteId") String noteId, String messa public Response removeCronJob(@PathParam("noteId") String noteId) throws IOException, IllegalArgumentException { - try { - LOGGER.info("Remove cron job note {}", noteId); + LOGGER.info("Remove cron job note {}", noteId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserIsOwner(noteId, - "Insufficient privileges you cannot remove this cron job from this note"); - checkIfNoteSupportsCron(note); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserIsOwner(noteId, + "Insufficient privileges you cannot remove this cron job from this note"); + checkIfNoteSupportsCron(note); - Map config = note.getConfig(); - config.remove("cron"); - config.remove("releaseresource"); - note.setConfig(config); - schedulerService.refreshCron(note.getId()); + Map config = note.getConfig(); + config.remove("cron"); + config.remove("releaseresource"); + note.setConfig(config); + schedulerService.refreshCron(note.getId()); - return new JsonResponse<>(Status.OK).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + return new JsonResponse<>(Status.OK).build(); } /** @@ -1101,21 +1003,17 @@ public Response removeCronJob(@PathParam("noteId") String noteId) public Response getCronJob(@PathParam("noteId") String noteId) throws IOException, IllegalArgumentException { - try { - LOGGER.info("Get cron job note {}", noteId); + LOGGER.info("Get cron job note {}", noteId); - Note note = notebook.getNote(noteId); - checkIfNoteIsNotNull(note); - checkIfUserCanRead(noteId, "Insufficient privileges you cannot get cron information"); - checkIfNoteSupportsCron(note); - Map response = new HashMap<>(); - response.put("cron", note.getConfig().get("cron")); - response.put("releaseResource", note.getConfig().get("releaseresource")); + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanRead(noteId, "Insufficient privileges you cannot get cron information"); + checkIfNoteSupportsCron(note); + Map response = new HashMap<>(); + response.put("cron", note.getConfig().get("cron")); + response.put("releaseResource", note.getConfig().get("releaseresource")); - return new JsonResponse<>(Status.OK, response).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + return new JsonResponse<>(Status.OK, response).build(); } /** @@ -1129,17 +1027,13 @@ public Response getCronJob(@PathParam("noteId") String noteId) @Path("jobmanager/") @ZeppelinApi public Response getJobListforNote() throws IOException, IllegalArgumentException { - try { - LOGGER.info("Get note jobs for job manager"); - List noteJobs = jobManagerService - .getNoteJobInfoByUnixTime(0, getServiceContext(), new RestServiceCallback<>()); - Map response = new HashMap<>(); - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", noteJobs); - return new JsonResponse<>(Status.OK, response).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + LOGGER.info("Get note jobs for job manager"); + List noteJobs = jobManagerService + .getNoteJobInfoByUnixTime(0, getServiceContext(), new RestServiceCallback<>()); + Map response = new HashMap<>(); + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", noteJobs); + return new JsonResponse<>(Status.OK, response).build(); } /** @@ -1156,18 +1050,14 @@ public Response getJobListforNote() throws IOException, IllegalArgumentException @ZeppelinApi public Response getUpdatedJobListforNote(@PathParam("lastUpdateUnixtime") long lastUpdateUnixTime) throws IOException, IllegalArgumentException { - try { - LOGGER.info("Get updated note jobs lastUpdateTime {}", lastUpdateUnixTime); - List noteJobs = - jobManagerService.getNoteJobInfoByUnixTime(lastUpdateUnixTime, getServiceContext(), - new RestServiceCallback<>()); - Map response = new HashMap<>(); - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", noteJobs); - return new JsonResponse<>(Status.OK, response).build(); - } catch (Throwable e) { - return createResponseFromException(e); - } + LOGGER.info("Get updated note jobs lastUpdateTime {}", lastUpdateUnixTime); + List noteJobs = + jobManagerService.getNoteJobInfoByUnixTime(lastUpdateUnixTime, getServiceContext(), + new RestServiceCallback<>()); + Map response = new HashMap<>(); + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", noteJobs); + return new JsonResponse<>(Status.OK, response).build(); } /** @@ -1177,30 +1067,26 @@ public Response getUpdatedJobListforNote(@PathParam("lastUpdateUnixtime") long l @Path("search") @ZeppelinApi public Response search(@QueryParam("q") String queryTerm) { - try { - LOGGER.info("Searching notes for: {}", queryTerm); - String principal = authenticationService.getPrincipal(); - Set roles = authenticationService.getAssociatedRoles(); - HashSet userAndRoles = new HashSet<>(); - userAndRoles.add(principal); - userAndRoles.addAll(roles); - List> notesFound = noteSearchService.query(queryTerm); - for (int i = 0; i < notesFound.size(); i++) { - String[] ids = notesFound.get(i).get("id").split("/", 2); - String noteId = ids[0]; - if (!authorizationService.isOwner(noteId, userAndRoles) && - !authorizationService.isReader(noteId, userAndRoles) && - !authorizationService.isWriter(noteId, userAndRoles) && - !authorizationService.isRunner(noteId, userAndRoles)) { - notesFound.remove(i); - i--; - } + LOGGER.info("Searching notes for: {}", queryTerm); + String principal = authenticationService.getPrincipal(); + Set roles = authenticationService.getAssociatedRoles(); + HashSet userAndRoles = new HashSet<>(); + userAndRoles.add(principal); + userAndRoles.addAll(roles); + List> notesFound = noteSearchService.query(queryTerm); + for (int i = 0; i < notesFound.size(); i++) { + String[] ids = notesFound.get(i).get("id").split("/", 2); + String noteId = ids[0]; + if (!authorizationService.isOwner(noteId, userAndRoles) && + !authorizationService.isReader(noteId, userAndRoles) && + !authorizationService.isWriter(noteId, userAndRoles) && + !authorizationService.isRunner(noteId, userAndRoles)) { + notesFound.remove(i); + i--; } - LOGGER.info("{} notes found", notesFound.size()); - return new JsonResponse<>(Status.OK, notesFound).build(); - } catch (Throwable e) { - return createResponseFromException(e); } + LOGGER.info("{} notes found", notesFound.size()); + return new JsonResponse<>(Status.OK, notesFound).build(); } @@ -1244,12 +1130,4 @@ private void configureParagraph(Paragraph p, Map newConfig, Stri p.setConfig(origConfig); } - - private Response createResponseFromException(Throwable e) { - if (e instanceof WebApplicationException) { - return ((WebApplicationException) e).getResponse(); - } else { - return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, ExceptionUtils.getStackTrace(e)).build(); - } - } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java index c5088a9111b..342374f1a0a 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionManager.java @@ -83,7 +83,7 @@ public synchronized SessionInfo createSession(String interpreter) throws Excepti throw new Exception("Unable to generate session id"); } - Note sessionNote = notebook.createNote("/_ZSession/" + sessionId, AuthenticationInfo.ANONYMOUS); + Note sessionNote = notebook.createNote("/_ZSession/" + interpreter + "/" + sessionId, AuthenticationInfo.ANONYMOUS); SessionInfo sessionInfo = new SessionInfo(sessionId, sessionNote.getId(), interpreter); sessions.put(sessionId, sessionInfo); return sessionInfo; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java index 7a1ebac54a4..8c5b164255f 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SessionRestApi.java @@ -58,50 +58,35 @@ public SessionRestApi(Notebook notebook, InterpreterSettingManager interpreterSe @GET @Path("/") - public Response listSessions(@QueryParam("interpreter") String interpreter) { + public Response listSessions(@QueryParam("interpreter") String interpreter) throws Exception { if (StringUtils.isBlank(interpreter)) { LOGGER.info("List all sessions"); } else { LOGGER.info("List all sessions for interpreter: " + interpreter); } - try { - List sessionList = null; - if (StringUtils.isBlank(interpreter)) { - sessionList = sessionManager.getAllSessions(); - } else { - sessionList = sessionManager.getAllSessions(interpreter); - } - return new JsonResponse<>(Response.Status.OK, sessionList).build(); - } catch (Exception e) { - return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, - "Fail to list session", ExceptionUtils.getStackTrace(e)).build(); + List sessionList = null; + if (StringUtils.isBlank(interpreter)) { + sessionList = sessionManager.getAllSessions(); + } else { + sessionList = sessionManager.getAllSessions(interpreter); } + return new JsonResponse<>(Response.Status.OK, sessionList).build(); } @POST @Path("/") - public Response createSession(@QueryParam("interpreter") String interpreter) { + public Response createSession(@QueryParam("interpreter") String interpreter) throws Exception { LOGGER.info("Create new session for interpreter: {}", interpreter); - try { - SessionInfo sessionInfo = sessionManager.createSession(interpreter); - return new JsonResponse<>(Response.Status.OK, sessionInfo).build(); - } catch (Exception e) { - return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, - "Fail to start session", ExceptionUtils.getStackTrace(e)).build(); - } + SessionInfo sessionInfo = sessionManager.createSession(interpreter); + return new JsonResponse<>(Response.Status.OK, sessionInfo).build(); } @DELETE @Path("{sessionId}") public Response stopSession(@PathParam("sessionId") String sessionId) { LOGGER.info("Stop session: {}", sessionId); - try { - sessionManager.removeSession(sessionId); - return new JsonResponse<>(Response.Status.OK, sessionId).build(); - } catch (Exception e) { - return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, - "Fail to stop session", ExceptionUtils.getStackTrace(e)).build(); - } + sessionManager.removeSession(sessionId); + return new JsonResponse<>(Response.Status.OK, sessionId).build(); } /** @@ -110,18 +95,12 @@ public Response stopSession(@PathParam("sessionId") String sessionId) { @GET @Path("{sessionId}") @ZeppelinApi - public Response getSession(@PathParam("sessionId") String sessionId) { - try { - SessionInfo session = sessionManager.getSession(sessionId); - if (session == null) { - return new JsonResponse<>(Response.Status.NOT_FOUND).build(); - } else { - return new JsonResponse<>(Response.Status.OK, "", session).build(); - } - } catch (Throwable e) { - LOGGER.error("Exception in SessionRestApi while getSession ", e); - return new JsonResponse<>(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage(), - ExceptionUtils.getStackTrace(e)).build(); + public Response getSession(@PathParam("sessionId") String sessionId) throws Exception { + SessionInfo session = sessionManager.getSession(sessionId); + if (session == null) { + return new JsonResponse<>(Response.Status.NOT_FOUND).build(); + } else { + return new JsonResponse<>(Response.Status.OK, "", session).build(); } } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/exception/WebApplicationExceptionMapper.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/exception/WebApplicationExceptionMapper.java index 79e3fe0e585..5615f875b72 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/exception/WebApplicationExceptionMapper.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/exception/WebApplicationExceptionMapper.java @@ -23,10 +23,11 @@ import javax.ws.rs.core.Response; import javax.ws.rs.ext.ExceptionMapper; import javax.ws.rs.ext.Provider; + import org.apache.zeppelin.rest.message.gson.ExceptionSerializer; @Provider -public class WebApplicationExceptionMapper implements ExceptionMapper { +public class WebApplicationExceptionMapper implements ExceptionMapper { private final Gson gson; public WebApplicationExceptionMapper() { @@ -37,8 +38,11 @@ public WebApplicationExceptionMapper() { } @Override - public Response toResponse(WebApplicationException exception) { - return Response.status(exception.getResponse().getStatus()) - .entity(gson.toJson(exception)).build(); + public Response toResponse(Throwable exception) { + if (exception instanceof WebApplicationException) { + return ((WebApplicationException) exception).getResponse(); + } else { + return Response.status(500).entity(gson.toJson(exception)).build(); + } } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/gson/ExceptionSerializer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/gson/ExceptionSerializer.java index ac654c097ad..78e61011441 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/gson/ExceptionSerializer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/gson/ExceptionSerializer.java @@ -21,6 +21,8 @@ import com.google.gson.JsonObject; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import org.apache.commons.lang3.exception.ExceptionUtils; + import java.lang.reflect.Type; import javax.ws.rs.WebApplicationException; @@ -32,6 +34,7 @@ public JsonElement serialize( JsonObject jsonObject = new JsonObject(); jsonObject.addProperty("exception", e.getClass().getSimpleName()); jsonObject.addProperty("message", e.getMessage()); + jsonObject.addProperty("stacktrace", ExceptionUtils.getStackTrace(e)); if (e instanceof WebApplicationException) { jsonObject.addProperty("status", ((WebApplicationException) e).getResponse().getStatus()); From 9b8ff424402fae059f933cae1e7f3c70c480b02b Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Tue, 8 Sep 2020 15:18:28 +0800 Subject: [PATCH 19/19] update travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b7454e624a3..7c50f9b430b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -220,6 +220,7 @@ jobs: dist: xenial before_install: - export PYTHON=3 + - export R=true - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc - bash -x ./testing/install_external_dependencies.sh - source ~/.environ