From 429516845bf9137c8d1d6e908a0e521894bae529 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:14:57 +0200 Subject: [PATCH 01/71] Added shirt dependencies --- pom.xml | 11 +++++++++++ zeppelin-server/pom.xml | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/pom.xml b/pom.xml index 81fd00ecb40..2eed0ea84fc 100644 --- a/pom.xml +++ b/pom.xml @@ -867,6 +867,17 @@ ${json4s.version} + + + org.apache.shiro + shiro-core + 1.2.3 + + + org.apache.shiro + shiro-web + 1.2.3 + diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 4b2b4d9264d..d9833b1b39c 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -284,6 +284,15 @@ test + + + org.apache.shiro + shiro-core + + + org.apache.shiro + shiro-web + From e52f4fb2a5997bf41189b20d7b12222497445148 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:16:40 +0200 Subject: [PATCH 02/71] default shiro configuration file --- zeppelin-server/src/main/resources/shiro.ini | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 zeppelin-server/src/main/resources/shiro.ini diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini new file mode 100644 index 00000000000..f619315f6b6 --- /dev/null +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -0,0 +1,17 @@ +[users] +admin = password + +[main] + +# Let's use some in-memory caching to reduce the number of runtime lookups against Stormpath. +# A real application might want to use a more robust caching solution (e.g. ehcache or a +# distributed cache). When using such caches, be aware of your cache TTL settings: too high +# a TTL and the cache won't reflect any potential changes in Stormpath fast enough. Too low +# and the cache could evict too often, reducing performance. +cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager +securityManager.cacheManager = $cacheManager + + +[urls] +/** = anon +#/** = authcBasic From 6e07f10e5aab247a18df4dad7e10c719373ccee2 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:17:33 +0200 Subject: [PATCH 03/71] The connected or anonymous user should get a ticket first from this API --- .../apache/zeppelin/rest/SecurityRestApi.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java new file mode 100644 index 00000000000..c8fad87b97e --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java @@ -0,0 +1,53 @@ +package org.apache.zeppelin.rest; + +import org.apache.zeppelin.server.JsonResponse; +import org.apache.zeppelin.ticket.TicketContainer; +import org.apache.shiro.SecurityUtils; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; +import java.util.HashMap; +import java.util.Map; + +/** + * Zeppelin root rest api endpoint. + * + * @author anthonycorbacho + * @since 0.3.4 + */ +@Path("/security") +@Produces("application/json") +public class SecurityRestApi { + /** + * Required by Swagger. + */ + public SecurityRestApi() { + super(); + } + + /** + * Get ticket + * Returns username & ticket + * for anonymous access, username is always anonymous. + * After getting this ticket, access through websockets become safe + * @return 200 response + */ + @GET + @Path("ticket") + public Response ticket() { + Object oprincipal = SecurityUtils.getSubject().getPrincipal(); + String principal; + if (oprincipal == null) + principal = "anonymous"; + else + principal = oprincipal.toString(); + String ticket = TicketContainer.instance.getTicket(principal); + Map data = new HashMap<>(); + data.put("principal", principal); + data.put("ticket", ticket); + + return new JsonResponse(Response.Status.OK, "", data).build(); + } +} From 8178c2cb43783694fe2070f950c5bfd35f73e1ae Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:17:54 +0200 Subject: [PATCH 04/71] Shiro filter to protect access to the API. No need to proceed web resources here. --- .../org/apache/zeppelin/server/ZeppelinServer.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index bd55b2d6c32..161ccefdbde 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -34,6 +34,7 @@ import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.rest.InterpreterRestApi; import org.apache.zeppelin.rest.NotebookRestApi; +import org.apache.zeppelin.rest.SecurityRestApi; import org.apache.zeppelin.rest.ZeppelinRestApi; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.socket.NotebookServer; @@ -215,7 +216,11 @@ private static ServletContextHandler setupRestApiContextHandler() { cxfContext.addServlet(cxfServletHolder, "/*"); cxfContext.addFilter(new FilterHolder(CorsFilter.class), "/*", - EnumSet.allOf(DispatcherType.class)); + EnumSet.allOf(DispatcherType.class)); + + cxfContext.addFilter(org.apache.shiro.web.servlet.ShiroFilter.class, "/*", + EnumSet.allOf(DispatcherType.class)); + cxfContext.addEventListener(new org.apache.shiro.web.env.EnvironmentLoaderListener()); return cxfContext; } @@ -309,7 +314,7 @@ public Set> getClasses() { } @Override - public java.util.Set getSingletons() { + public Set getSingletons() { Set singletons = new HashSet(); /** Rest-api root endpoint */ @@ -322,6 +327,9 @@ public java.util.Set getSingletons() { InterpreterRestApi interpreterApi = new InterpreterRestApi(replFactory); singletons.add(interpreterApi); + SecurityRestApi securityApi = new SecurityRestApi(); + singletons.add(securityApi); + return singletons; } } From 774ada93f2e4a2747fc28e50afb71548a3d7e9f1 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:18:13 +0200 Subject: [PATCH 05/71] Very simple ticket container No cleanup is done, since the same user accross different devices share the same ticket The Map size is at most the number of different user names having access to a Zeppelin instance --- .../zeppelin/ticket/TicketContainer.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java new file mode 100644 index 00000000000..c1c4b62684a --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java @@ -0,0 +1,51 @@ +package org.apache.zeppelin.ticket; + +import java.util.Calendar; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Created by hayssams on 24/04/15. + * Very simple ticket container + * No cleanup is done, since the same user accross different devices share the same ticket + * The Map size is at most the number of different user names having access to a Zeppelin instance + */ + + +public class TicketContainer { + private static class Entry { + public final String ticket; + // lastAccessTime still unused + public final long lastAccessTime; + Entry(String ticket) { + this.ticket = ticket; + this.lastAccessTime = Calendar.getInstance().getTimeInMillis(); + } + } + private Map sessions = new ConcurrentHashMap<>(); + + public static final TicketContainer instance = new TicketContainer(); + + public boolean isValid(String principal, String ticket) { + Entry entry = sessions.get(principal); + return entry != null && entry.ticket.equals(ticket); + } + + public synchronized String getTicket(String principal) { + Entry entry = sessions.get(principal); + String ticket; + if (entry == null) { + if (principal.equals("anonymous")) + ticket = "anonymous"; // enable testing on anonymous when ticket is required in the url + else + ticket = UUID.randomUUID().toString(); + } + else { + ticket = entry.ticket; + } + entry = new Entry(ticket); + sessions.put(principal, entry); + return ticket; + } +} From c2b3cf325244ddf52e46723190d191cb93824900 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:18:28 +0200 Subject: [PATCH 06/71] User name & ticket are now required to access the notebook api since each note is now attached to a user --- .../apache/zeppelin/rest/NotebookRestApi.java | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 8a933f70397..a3483727b98 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -33,6 +33,7 @@ import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.rest.message.InterpreterSettingListForNoteBind; import org.apache.zeppelin.server.JsonResponse; +import org.apache.zeppelin.ticket.TicketContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,10 +61,15 @@ public NotebookRestApi(Notebook notebook) { * @throws IOException */ @PUT - @Path("interpreter/bind/{noteId}") - public Response bind(@PathParam("noteId") String noteId, String req) throws IOException { + @Path("interpreter/bind/{noteId}/{principal}/{ticket}") + public Response bind(@PathParam("noteId") String noteId, + @PathParam("principal") String principal, + @PathParam("ticket") String ticket, String req) throws Exception { + if (!TicketContainer.instance.isValid(principal, ticket)) + throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); + List settingIdList = gson.fromJson(req, new TypeToken>(){}.getType()); - notebook.bindInterpretersToNote(noteId, settingIdList); + notebook.bindInterpretersToNote(noteId, settingIdList, principal); return new JsonResponse(Status.OK).build(); } @@ -71,12 +77,17 @@ public Response bind(@PathParam("noteId") String noteId, String req) throws IOEx * list binded setting */ @GET - @Path("interpreter/bind/{noteId}") - public Response bind(@PathParam("noteId") String noteId) { - List settingList - = new LinkedList(); + @Path("interpreter/bind/{noteId}/{principal}/{ticket}") + public Response bind(@PathParam("noteId") String noteId, + @PathParam("principal") String principal, + @PathParam("ticket") String ticket) throws Exception { + if (!TicketContainer.instance.isValid(principal, ticket)) + throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); + + List settingList = new LinkedList<>(); - List selectedSettings = notebook.getBindedInterpreterSettings(noteId); + List selectedSettings = + notebook.getBindedInterpreterSettings(noteId, principal); for (InterpreterSetting setting : selectedSettings) { settingList.add(new InterpreterSettingListForNoteBind( setting.id(), From 16f0ace0e7df6546faac6fb33bd5d689d7e7b6d4 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:18:51 +0200 Subject: [PATCH 07/71] Notes are now loaded from and stored in a subdirectory named after the username. --- .../org/apache/zeppelin/notebook/Note.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index b5e68a42bb3..0fc1ea19003 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -57,6 +57,7 @@ public class Note implements Serializable, JobListener { List paragraphs = new LinkedList(); private String name; private String id; + private String owner; Map> angularObjects = new HashMap>(); @@ -81,10 +82,13 @@ public class Note implements Serializable, JobListener { public Note() {} public Note(ZeppelinConfiguration conf, NoteInterpreterLoader replLoader, - JobListenerFactory jobListenerFactory, org.quartz.Scheduler quartzSched) { + JobListenerFactory jobListenerFactory, + org.quartz.Scheduler quartzSched, + String owner) { this.conf = conf; this.replLoader = replLoader; this.jobListenerFactory = jobListenerFactory; + this.owner = owner; generateId(); } @@ -96,6 +100,8 @@ public String id() { return id; } + public String owner() { return this.owner; } + public String getName() { return name; } @@ -298,14 +304,14 @@ public void persist() throws IOException { gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.create(); - File dir = new File(conf.getNotebookDir() + "/" + id); + File dir = new File(conf.getNotebookDir() + File.separator + owner + File.separator + id); if (!dir.exists()) { dir.mkdirs(); } else if (dir.isFile()) { throw new RuntimeException("File already exists" + dir.toString()); } - File file = new File(conf.getNotebookDir() + "/" + id + "/note.json"); + File file = new File(dir, "note.json"); logger().info("Persist note {} into {}", id, file.getAbsolutePath()); snapshotAngularObjectRegistry(); @@ -316,19 +322,21 @@ public void persist() throws IOException { } public void unpersist() throws IOException { - File dir = new File(conf.getNotebookDir() + "/" + id); + File dir = new File(conf.getNotebookDir() + File.separator + owner + File.separator + id); FileUtils.deleteDirectory(dir); } public static Note load(String id, ZeppelinConfiguration conf, NoteInterpreterLoader replLoader, - Scheduler scheduler, JobListenerFactory jobListenerFactory, org.quartz.Scheduler quartzSched) + Scheduler scheduler, JobListenerFactory jobListenerFactory, + org.quartz.Scheduler quartzSched, String principal) throws IOException { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.create(); - File file = new File(conf.getNotebookDir() + "/" + id + "/note.json"); + File file = new File(conf.getNotebookDir() + File.separator + principal + + File.separator + id + File.separator + "note.json"); logger().info("Load note {} from {}", id, file.getAbsolutePath()); if (!file.isFile()) { From a8eb2cc17b3a30f3e8838291ef3cab3b9b4bbae2 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:19:08 +0200 Subject: [PATCH 08/71] Web socket handlers require the username & ticket got through the initial HTTP call to the SecurityRestApi. This proves that web socket access is done by the user who provided the credentials through Shiro on the HTTP channel. --- .../zeppelin/socket/NotebookServer.java | 115 ++++++++++-------- 1 file changed, 67 insertions(+), 48 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 69d62d8e4fe..8b2ce699161 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -19,11 +19,7 @@ import java.io.IOException; import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; @@ -39,6 +35,7 @@ import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.server.ZeppelinServer; import org.apache.zeppelin.socket.Message.OP; +import org.apache.zeppelin.ticket.TicketContainer; import org.java_websocket.WebSocket; import org.java_websocket.handshake.ClientHandshake; import org.java_websocket.server.WebSocketServer; @@ -67,7 +64,7 @@ private static void creatingwebSocketServerLog(String address, int port) { Gson gson = new Gson(); Map> noteSocketMap = new HashMap>(); - List connectedSockets = new LinkedList(); + Map> userSocketMap = new HashMap<>(); public NotebookServer() { super(new InetSocketAddress(DEFAULT_ADDR, DEFAULT_PORT)); @@ -86,10 +83,7 @@ private Notebook notebook() { @Override public void onOpen(WebSocket conn, ClientHandshake handshake) { LOG.info("New connection from {} : {}", conn.getRemoteSocketAddress().getHostName(), conn - .getRemoteSocketAddress().getPort()); - synchronized (connectedSockets) { - connectedSockets.add(conn); - } + .getRemoteSocketAddress().getPort()); } @Override @@ -97,17 +91,22 @@ public void onMessage(WebSocket conn, String msg) { Notebook notebook = notebook(); try { Message messagereceived = deserializeMessage(msg); - LOG.info("RECEIVE << " + messagereceived.op); + LOG.info("RECEIVE OP << " + messagereceived.op); + LOG.info("RECEIVE PRINCIPAL << " + messagereceived.principal); + LOG.info("RECEIVE TICKET << " + messagereceived.ticket); + String ticket = TicketContainer.instance.getTicket(messagereceived.principal); + if (ticket != null && !ticket.equals(messagereceived.ticket)) + throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); /** Lets be elegant here */ switch (messagereceived.op) { case LIST_NOTES: - broadcastNoteList(); + broadcastNoteList(conn, messagereceived); break; case GET_NOTE: sendNote(conn, notebook, messagereceived); break; case NEW_NOTE: - createNote(conn, notebook); + createNote(conn, notebook, messagereceived); break; case DEL_NOTE: removeNote(conn, notebook, messagereceived); @@ -140,7 +139,7 @@ public void onMessage(WebSocket conn, String msg) { angularObjectUpdated(conn, notebook, messagereceived); break; default: - broadcastNoteList(); + broadcastNoteList(conn, messagereceived); break; } } catch (Exception e) { @@ -153,16 +152,22 @@ public void onClose(WebSocket conn, int code, String reason, boolean remote) { LOG.info("Closed connection to {} : {}", conn.getRemoteSocketAddress().getHostName(), conn .getRemoteSocketAddress().getPort()); removeConnectionFromAllNote(conn); - synchronized (connectedSockets) { - connectedSockets.remove(conn); + synchronized (userSocketMap) { + Collection> allSockets = userSocketMap.values(); + for (List userList : allSockets) { + userList.remove(conn); + } } } @Override public void onError(WebSocket conn, Exception message) { removeConnectionFromAllNote(conn); - synchronized (connectedSockets) { - connectedSockets.remove(conn); + synchronized (userSocketMap) { + Collection> allSockets = userSocketMap.values(); + for (List userList : allSockets) { + userList.remove(conn); + } } } @@ -230,7 +235,7 @@ private String getOpenNoteId(WebSocket socket) { private void broadcastToNoteBindedInterpreter(String interpreterGroupId, Message m) { Notebook notebook = notebook(); - List notes = notebook.getAllNotes(); + List notes = notebook.getAllNotes(m.principal); for (Note note : notes) { List ids = note.getNoteReplLoader().getInterpreters(); for (String id : ids) { @@ -256,10 +261,21 @@ private void broadcast(String noteId, Message m) { } } - private void broadcastAll(Message m) { - synchronized (connectedSockets) { - for (WebSocket conn : connectedSockets) { - conn.send(serializeMessage(m)); + private void broadcastAll(WebSocket conn, Message m) { + synchronized (userSocketMap) { + //conn.send(serializeMessage(m)); + List> notesInfo = (List>) m.get("notes"); + for ( Map info : notesInfo) { + String principal = info.get("principal"); + List conns = userSocketMap.get(principal); + if (conns == null) { + conns = new LinkedList<>(); + conns.add(conn); + userSocketMap.put(principal, conns); + } + for (WebSocket theconn : conns) { + theconn.send(serializeMessage(m)); + } } } } @@ -268,17 +284,18 @@ private void broadcastNote(Note note) { broadcast(note.id(), new Message(OP.NOTE).put("note", note)); } - private void broadcastNoteList() { + private void broadcastNoteList(WebSocket conn, Message fromMessage) { Notebook notebook = notebook(); - List notes = notebook.getAllNotes(); + List notes = notebook.getAllNotes(fromMessage.principal); List> notesInfo = new LinkedList>(); for (Note note : notes) { - Map info = new HashMap(); + Map info = new HashMap<>(); info.put("id", note.id()); info.put("name", note.getName()); + info.put("principal", fromMessage.principal); notesInfo.add(info); } - broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); + broadcastAll(conn, new Message(OP.NOTES_INFO).put("notes", notesInfo)); } private void sendNote(WebSocket conn, Notebook notebook, Message fromMessage) { @@ -286,7 +303,7 @@ private void sendNote(WebSocket conn, Notebook notebook, Message fromMessage) { if (noteId == null) { return; } - Note note = notebook.getNote(noteId); + Note note = notebook.getNote(noteId, fromMessage.principal); if (note != null) { addConnectionToNote(note.id(), conn); @@ -306,19 +323,19 @@ private void updateNote(WebSocket conn, Notebook notebook, Message fromMessage) if (config == null) { return; } - Note note = notebook.getNote(noteId); + Note note = notebook.getNote(noteId, fromMessage.principal); if (note != null) { boolean cronUpdated = isCronUpdated(config, note.getConfig()); note.setName(name); note.setConfig(config); if (cronUpdated) { - notebook.refreshCron(note.id()); + notebook.refreshCron(note.id(), fromMessage.principal); } note.persist(); broadcastNote(note); - broadcastNoteList(); + broadcastNoteList(conn, fromMessage); } } @@ -335,12 +352,12 @@ private boolean isCronUpdated(Map configA, Map c return cronUpdated; } - private void createNote(WebSocket conn, Notebook notebook) throws IOException { - Note note = notebook.createNote(); + private void createNote(WebSocket conn, Notebook notebook, Message fromMsg) throws IOException { + Note note = notebook.createNote(fromMsg.principal); note.addParagraph(); // it's an empty note. so add one paragraph note.persist(); broadcastNote(note); - broadcastNoteList(); + broadcastNoteList(conn, fromMsg); } private void removeNote(WebSocket conn, Notebook notebook, Message fromMessage) @@ -349,11 +366,11 @@ private void removeNote(WebSocket conn, Notebook notebook, Message fromMessage) if (noteId == null) { return; } - Note note = notebook.getNote(noteId); + Note note = notebook.getNote(noteId, fromMessage.principal); note.unpersist(); - notebook.removeNote(noteId); + notebook.removeNote(noteId, fromMessage.principal); removeNote(noteId); - broadcastNoteList(); + broadcastNoteList(conn, fromMessage); } private void updateParagraph(WebSocket conn, Notebook notebook, Message fromMessage) @@ -364,7 +381,7 @@ private void updateParagraph(WebSocket conn, Notebook notebook, Message fromMess } Map params = (Map) fromMessage.get("params"); Map config = (Map) fromMessage.get("config"); - final Note note = notebook.getNote(getOpenNoteId(conn)); + final Note note = notebook.getNote(getOpenNoteId(conn), fromMessage.principal); Paragraph p = note.getParagraph(paragraphId); p.settings.setParams(params); p.setConfig(config); @@ -380,7 +397,7 @@ private void removeParagraph(WebSocket conn, Notebook notebook, Message fromMess if (paragraphId == null) { return; } - final Note note = notebook.getNote(getOpenNoteId(conn)); + final Note note = notebook.getNote(getOpenNoteId(conn), fromMessage.principal); /** We dont want to remove the last paragraph */ if (!note.isLastParagraph(paragraphId)) { note.removeParagraph(paragraphId); @@ -400,7 +417,7 @@ private void completion(WebSocket conn, Notebook notebook, Message fromMessage) return; } - final Note note = notebook.getNote(getOpenNoteId(conn)); + final Note note = notebook.getNote(getOpenNoteId(conn), fromMessage.principal); List candidates = note.completion(paragraphId, buffer, cursor); resp.put("completions", candidates); conn.send(serializeMessage(resp)); @@ -418,9 +435,10 @@ private void angularObjectUpdated(WebSocket conn, Notebook notebook, String interpreterGroupId = (String) fromMessage.get("interpreterGroupId"); String varName = (String) fromMessage.get("name"); Object varValue = fromMessage.get("value"); + String principal = fromMessage.principal; // propagate change to (Remote) AngularObjectRegistry - Note note = notebook.getNote(noteId); + Note note = notebook.getNote(noteId, fromMessage.principal); if (note != null) { List settings = note.getNoteReplLoader().getInterpreterSettings(); for (InterpreterSetting setting : settings) { @@ -437,6 +455,7 @@ private void angularObjectUpdated(WebSocket conn, Notebook notebook, } else { // path from client -> server ao.set(varValue, false); + ao.setPrincipal(principal); } break; @@ -445,7 +464,7 @@ private void angularObjectUpdated(WebSocket conn, Notebook notebook, } // broadcast change to all web session that uses related interpreter. - for (Note n : notebook.getAllNotes()) { + for (Note n : notebook.getAllNotes(fromMessage.principal)) { List settings = note.getNoteReplLoader().getInterpreterSettings(); for (InterpreterSetting setting : settings) { if (setting.getInterpreterGroup() == null) { @@ -474,7 +493,7 @@ private void moveParagraph(WebSocket conn, Notebook notebook, Message fromMessag } final int newIndex = (int) Double.parseDouble(fromMessage.get("index").toString()); - final Note note = notebook.getNote(getOpenNoteId(conn)); + final Note note = notebook.getNote(getOpenNoteId(conn), fromMessage.principal); note.moveParagraph(paragraphId, newIndex); note.persist(); broadcastNote(note); @@ -484,7 +503,7 @@ private void insertParagraph(WebSocket conn, Notebook notebook, Message fromMess throws IOException { final int index = (int) Double.parseDouble(fromMessage.get("index").toString()); - final Note note = notebook.getNote(getOpenNoteId(conn)); + final Note note = notebook.getNote(getOpenNoteId(conn), fromMessage.principal); note.insertParagraph(index); note.persist(); broadcastNote(note); @@ -498,7 +517,7 @@ private void cancelParagraph(WebSocket conn, Notebook notebook, Message fromMess return; } - final Note note = notebook.getNote(getOpenNoteId(conn)); + final Note note = notebook.getNote(getOpenNoteId(conn), fromMessage.principal); Paragraph p = note.getParagraph(paragraphId); p.abort(); } @@ -509,7 +528,7 @@ private void runParagraph(WebSocket conn, Notebook notebook, Message fromMessage if (paragraphId == null) { return; } - final Note note = notebook.getNote(getOpenNoteId(conn)); + final Note note = notebook.getNote(getOpenNoteId(conn), fromMessage.principal); Paragraph p = note.getParagraph(paragraphId); String text = (String) fromMessage.get("paragraph"); p.setText(text); @@ -611,7 +630,7 @@ public void onAdd(String interpreterGroupId, AngularObject object) { public void onUpdate(String interpreterGroupId, AngularObject object) { Notebook notebook = notebook(); - List notes = notebook.getAllNotes(); + List notes = notebook.getAllNotes(object.getPrincipal()); for (Note note : notes) { List intpSettings = note.getNoteReplLoader() .getInterpreterSettings(); @@ -632,7 +651,7 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { @Override public void onRemove(String interpreterGroupId, AngularObject object) { Notebook notebook = notebook(); - List notes = notebook.getAllNotes(); + List notes = notebook.getAllNotes(object.getPrincipal()); for (Note note : notes) { List ids = note.getNoteReplLoader().getInterpreters(); for (String id : ids) { From f81aaa5f7cbd579b489938ff9339b6134e66e940 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:19:26 +0200 Subject: [PATCH 09/71] Notebooks are now user dependent. We still load all note in memory. Tests are updated accordingly by using the anonymous user. --- .../apache/zeppelin/notebook/Notebook.java | 206 +++++++++++------- .../zeppelin/notebook/NotebookTest.java | 12 +- 2 files changed, 133 insertions(+), 85 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 844763f7576..841c2e42806 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -19,15 +19,7 @@ import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; @@ -59,7 +51,7 @@ public class Notebook { private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; /** Keep the order. */ - Map notes = new LinkedHashMap(); + Map> notes = new LinkedHashMap<>(); private ZeppelinConfiguration conf; private StdSchedulerFactory quertzSchedFact; private org.quartz.Scheduler quartzSched; @@ -72,7 +64,7 @@ public Notebook(ZeppelinConfiguration conf, SchedulerFactory schedulerFactory, this.schedulerFactory = schedulerFactory; this.replFactory = replFactory; this.jobListenerFactory = jobListenerFactory; - quertzSchedFact = new org.quartz.impl.StdSchedulerFactory(); + quertzSchedFact = new StdSchedulerFactory(); quartzSched = quertzSchedFact.getScheduler(); quartzSched.start(); CronJob.notebook = this; @@ -86,45 +78,57 @@ public Notebook(ZeppelinConfiguration conf, SchedulerFactory schedulerFactory, * @return * @throws IOException */ - public Note createNote() throws IOException { + public Note createNote(String principal) throws IOException { if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) { - return createNote(replFactory.getDefaultInterpreterSettingList()); + return createNote(replFactory.getDefaultInterpreterSettingList(), principal); } else { - return createNote(null); + return createNote(null, principal); } } + private Map getUserNotes(String principal) { + synchronized (notes) { + Map userNotes = notes.get(principal); + if (userNotes == null) + userNotes = new HashMap<>(); + notes.put(principal, userNotes); + return userNotes; + } + + + } + /** * Create new note. * * @return * @throws IOException */ - public Note createNote(List interpreterIds) throws IOException { + public Note createNote(List interpreterIds, String principal) throws IOException { NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory); - Note note = new Note(conf, intpLoader, jobListenerFactory, quartzSched); + Note note = new Note(conf, intpLoader, jobListenerFactory, quartzSched, principal); intpLoader.setNoteId(note.id()); synchronized (notes) { - notes.put(note.id(), note); + getUserNotes(principal).put(note.id(), note); } if (interpreterIds != null) { - bindInterpretersToNote(note.id(), interpreterIds); + bindInterpretersToNote(note.id(), interpreterIds, principal); } return note; } public void bindInterpretersToNote(String id, - List interpreterSettingIds) throws IOException { - Note note = getNote(id); + List interpreterSettingIds, String principal) throws IOException { + Note note = getNote(id, principal); if (note != null) { note.getNoteReplLoader().setInterpreters(interpreterSettingIds); replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); } } - public List getBindedInterpreterSettingsIds(String id) { - Note note = getNote(id); + public List getBindedInterpreterSettingsIds(String id, String principal) { + Note note = getNote(id, principal); if (note != null) { return note.getNoteReplLoader().getInterpreters(); } else { @@ -132,8 +136,8 @@ public List getBindedInterpreterSettingsIds(String id) { } } - public List getBindedInterpreterSettings(String id) { - Note note = getNote(id); + public List getBindedInterpreterSettings(String id, String principal) { + Note note = getNote(id, principal); if (note != null) { return note.getNoteReplLoader().getInterpreterSettings(); } else { @@ -141,16 +145,16 @@ public List getBindedInterpreterSettings(String id) { } } - public Note getNote(String id) { + public Note getNote(String id, String principal) { synchronized (notes) { - return notes.get(id); + return getUserNotes(principal).get(id); } } - public void removeNote(String id) { + public void removeNote(String id, String principal) { Note note; synchronized (notes) { - note = notes.remove(id); + note = getUserNotes(principal).remove(id); } try { note.unpersist(); @@ -161,61 +165,73 @@ public void removeNote(String id) { private void loadAllNotes() throws IOException { File notebookDir = new File(conf.getNotebookDir()); - File[] dirs = notebookDir.listFiles(); - if (dirs == null) { + logger.info("Notebook Directory " + notebookDir.getAbsolutePath()); + File[] userdirs = notebookDir.listFiles(); + if (userdirs == null) { return; } Map angularObjectSnapshot = new HashMap(); - for (File f : dirs) { - boolean isHidden = f.getName().startsWith("."); - if (f.isDirectory() && !isHidden) { - Scheduler scheduler = - schedulerFactory.createOrGetFIFOScheduler("note_" + System.currentTimeMillis()); - logger.info("Loading note from " + f.getName()); - NoteInterpreterLoader noteInterpreterLoader = new NoteInterpreterLoader(replFactory); - Note note = Note.load(f.getName(), - conf, - noteInterpreterLoader, - scheduler, - jobListenerFactory, quartzSched); - noteInterpreterLoader.setNoteId(note.id()); - - // restore angular object -------------- - Date lastUpdatedDate = new Date(0); - for (Paragraph p : note.getParagraphs()) { - if (p.getDateFinished() != null && - lastUpdatedDate.before(p.getDateFinished())) { - lastUpdatedDate = p.getDateFinished(); - } - } + for (File fuser : userdirs) { + String principal = fuser.getName(); + boolean isHidden = principal.startsWith("."); + if (fuser.isDirectory() && !isHidden) { + File[] dirs = fuser.listFiles(); + if (dirs != null) { + for (File f : dirs) { + boolean isHiddenFile = f.getName().startsWith("."); + if (f.isDirectory() && !isHiddenFile) { + Scheduler scheduler = + schedulerFactory.createOrGetFIFOScheduler("note_" + System.currentTimeMillis()); + logger.info("Loading note from " + f.getName()); + NoteInterpreterLoader noteInterpreterLoader = new NoteInterpreterLoader(replFactory); + Note note = Note.load(f.getName(), + conf, + noteInterpreterLoader, + scheduler, + jobListenerFactory, quartzSched, fuser.getName()); + logger.info("Loading note from " + f.getName()); + noteInterpreterLoader.setNoteId(note.id()); + + // restore angular object -------------- + Date lastUpdatedDate = new Date(0); + for (Paragraph p : note.getParagraphs()) { + if (p.getDateFinished() != null && + lastUpdatedDate.before(p.getDateFinished())) { + lastUpdatedDate = p.getDateFinished(); + } + } - Map> savedObjects = note.getAngularObjects(); - - if (savedObjects != null) { - for (String intpGroupName : savedObjects.keySet()) { - List objectList = savedObjects.get(intpGroupName); - - for (AngularObject savedObject : objectList) { - SnapshotAngularObject snapshot = angularObjectSnapshot.get(savedObject.getName()); - if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) { - angularObjectSnapshot.put( - savedObject.getName(), - new SnapshotAngularObject( - intpGroupName, - savedObject, - lastUpdatedDate)); + Map> savedObjects = note.getAngularObjects(); + + if (savedObjects != null) { + for (String intpGroupName : savedObjects.keySet()) { + List objectList = savedObjects.get(intpGroupName); + + for (AngularObject savedObject : objectList) { + SnapshotAngularObject snapshot = + angularObjectSnapshot.get(savedObject.getName()); + if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) { + angularObjectSnapshot.put( + savedObject.getName(), + new SnapshotAngularObject( + intpGroupName, + savedObject, + lastUpdatedDate)); + } + } + } + } + + synchronized (notes) { + getUserNotes(principal).put(note.id(), note); + refreshCron(note.id(), principal); } } } } - - synchronized (notes) { - notes.put(note.id(), note); - refreshCron(note.id()); - } } } @@ -258,9 +274,38 @@ public Date getLastUpdate() { } } + public List getAllNotes(String principal) { + synchronized (notes) { + List noteList = new ArrayList<>(getUserNotes(principal).values()); + Collections.sort(noteList, new Comparator() { + @Override + public int compare(Object one, Object two) { + Note note1 = (Note) one; + Note note2 = (Note) two; + + String name1 = note1.id(); + if (note1.getName() != null) { + name1 = note1.getName(); + } + String name2 = note2.id(); + if (note2.getName() != null) { + name2 = note2.getName(); + } + ((Note) one).getName(); + return name1.compareTo(name2); + } + }); + return noteList; + } + } + public List getAllNotes() { synchronized (notes) { - List noteList = new ArrayList(notes.values()); + Collection> usersNotes = notes.values(); + List noteList = new ArrayList<>(); + for (Map userNotes : usersNotes) { + noteList.addAll(userNotes.values()); + } Collections.sort(noteList, new Comparator() { @Override public int compare(Object one, Object two) { @@ -302,18 +347,18 @@ public static class CronJob implements org.quartz.Job { @Override public void execute(JobExecutionContext context) throws JobExecutionException { - + String principal = context.getJobDetail().getJobDataMap().getString("principal"); String noteId = context.getJobDetail().getJobDataMap().getString("noteId"); - Note note = notebook.getNote(noteId); + Note note = notebook.getNote(noteId, principal); note.runAll(); } } - public void refreshCron(String id) { + public void refreshCron(String id, String principal) { removeCron(id); synchronized (notes) { - Note note = notes.get(id); + Note note = getUserNotes(principal).get(id); if (note == null) { return; } @@ -329,8 +374,11 @@ public void refreshCron(String id) { JobDetail newJob = - JobBuilder.newJob(CronJob.class).withIdentity(id, "note").usingJobData("noteId", id) - .build(); + JobBuilder.newJob(CronJob.class) + .withIdentity(id, "note") + .usingJobData("noteId", id) + .usingJobData("principal", principal) + .build(); Map info = note.getInfo(); info.put("cron", null); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 8d2c65ad114..515133910d7 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -81,7 +81,7 @@ public void tearDown() throws Exception { @Test public void testSelectingReplImplementation() throws IOException { - Note note = notebook.createNote(); + Note note = notebook.createNote("anonymous"); note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); // run with defatul repl @@ -101,7 +101,7 @@ public void testSelectingReplImplementation() throws IOException { @Test public void testPersist() throws IOException, SchedulerException{ - Note note = notebook.createNote(); + Note note = notebook.createNote("anonymous"); // run with default repl Paragraph p1 = note.addParagraph(); @@ -114,7 +114,7 @@ public void testPersist() throws IOException, SchedulerException{ @Test public void testRunAll() throws IOException { - Note note = notebook.createNote(); + Note note = notebook.createNote("anonymous"); note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); Paragraph p1 = note.addParagraph(); @@ -131,7 +131,7 @@ public void testRunAll() throws IOException { @Test public void testSchedule() throws InterruptedException, IOException{ // create a note and a paragraph - Note note = notebook.createNote(); + Note note = notebook.createNote("anonymous"); note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); Paragraph p = note.addParagraph(); @@ -143,7 +143,7 @@ public void testSchedule() throws InterruptedException, IOException{ Map config = note.getConfig(); config.put("cron", "* * * * * ?"); note.setConfig(config); - notebook.refreshCron(note.id()); + notebook.refreshCron(note.id(),"anonymous"); Thread.sleep(1*1000); dateFinished = p.getDateFinished(); assertNotNull(dateFinished); @@ -151,7 +151,7 @@ public void testSchedule() throws InterruptedException, IOException{ // remove cron scheduler. config.put("cron", null); note.setConfig(config); - notebook.refreshCron(note.id()); + notebook.refreshCron(note.id(),"anonymous"); Thread.sleep(1*1000); assertEquals(dateFinished, p.getDateFinished()); } From 3f96384f62c4f5d2f907063d9e3d0582daa93cb8 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:19:44 +0200 Subject: [PATCH 10/71] Get ticket on startup --- zeppelin-web/app/scripts/controllers/nav.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/zeppelin-web/app/scripts/controllers/nav.js b/zeppelin-web/app/scripts/controllers/nav.js index 39258452dc5..c5fbef416dd 100644 --- a/zeppelin-web/app/scripts/controllers/nav.js +++ b/zeppelin-web/app/scripts/controllers/nav.js @@ -23,7 +23,7 @@ * * @author anthonycorbacho */ -angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootScope, $routeParams) { +angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootScope, $routeParams, $http) { /** Current list of notes (ids) */ $scope.notes = []; $('#notebook-list').perfectScrollbar({suppressScrollX: true}); @@ -36,7 +36,18 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco var loadNotes = function() { $rootScope.$emit('sendNewEvent', {op: 'LIST_NOTES'}); }; - loadNotes(); + /** ask for a ticket for websocket access + * Shiro will require credentials here + * */ + $http.get('/api/security/ticket'). + success(function(ticket, status, headers, config) { + $rootScope.ticket = angular.fromJson(ticket).body; + loadNotes(); + }). + error(function(data, status, headers, config) { + console.log("Could not get ticket"); + }); + /** Create a new note */ $scope.createNewNote = function() { From baf311c4d6173af47581df95e3b727d067bf224a Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:20:01 +0200 Subject: [PATCH 11/71] Submit username & ticket on web socket requests --- zeppelin-web/app/scripts/controllers/main.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/zeppelin-web/app/scripts/controllers/main.js b/zeppelin-web/app/scripts/controllers/main.js index 4948b49659b..1d99309ed11 100644 --- a/zeppelin-web/app/scripts/controllers/main.js +++ b/zeppelin-web/app/scripts/controllers/main.js @@ -24,7 +24,7 @@ */ angular.module('zeppelinWebApp') .controller('MainCtrl', function($scope, WebSocket, $rootScope, $window) { - $rootScope.compiledScope = $scope.$new(true, $rootScope); + $rootScope.compiledScope = $scope.$new(true, $rootScope); $scope.WebSocketWaitingList = []; $scope.connected = false; $scope.looknfeel = 'default'; @@ -80,17 +80,20 @@ angular.module('zeppelinWebApp') $scope.connected = false; }); - /** Send info to the websocket server */ + /** Send info to the websocket server + * including the username & ticket + */ var send = function(data) { + data.principal = $scope.ticket.principal; + data.ticket = $scope.ticket.ticket; if (WebSocket.currentState() !== 'OPEN') { $scope.WebSocketWaitingList.push(data); } else { - console.log('Send >> %o, %o', data.op, data); + console.log('Send >> %o, %o, %o, %o', data.op, data.principal, data.ticket, data); WebSocket.send(JSON.stringify(data)); } }; - /** get the childs event and sebd to the websocket server */ $rootScope.$on('sendNewEvent', function(event, data) { if (!event.defaultPrevented) { From 4c9167eb943cd368955681665fd151f30a0a1d0f Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:20:23 +0200 Subject: [PATCH 12/71] add user & ticket to API calls --- zeppelin-web/app/scripts/controllers/notebook.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/zeppelin-web/app/scripts/controllers/notebook.js b/zeppelin-web/app/scripts/controllers/notebook.js index 9a9fcd7ce99..0d775972888 100644 --- a/zeppelin-web/app/scripts/controllers/notebook.js +++ b/zeppelin-web/app/scripts/controllers/notebook.js @@ -350,8 +350,9 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro } }; + /** user dependent*/ var getInterpreterBindings = function(callback) { - $http.get(getRestApiBase()+ '/notebook/interpreter/bind/' +$scope.note.id). + $http.get(getRestApiBase()+ '/notebook/interpreter/bind/' +$scope.note.id + "/" + $rootScope.ticket.principal+ "/" + $rootScope.ticket.ticket). success(function(data, status, headers, config) { $scope.interpreterBindings = data.body; $scope.interpreterBindingsOrig = jQuery.extend(true, [], $scope.interpreterBindings); // to check dirty @@ -409,6 +410,7 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro $scope.showSetting = false; }; + /** settings are user dependent */ $scope.saveSetting = function() { var selectedSettingIds = []; for (var no in $scope.interpreterBindings) { @@ -418,8 +420,8 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro } } - $http.put(getRestApiBase() + '/notebook/interpreter/bind/' + $scope.note.id, - selectedSettingIds). + $http.put(getRestApiBase() + '/notebook/interpreter/bind/' + $scope.note.id + "/" + $rootScope.ticket.principal+ "/" + $rootScope.ticket.ticket, + selectedSettingIds). success(function(data, status, headers, config) { console.log('Interpreter binding %o saved', selectedSettingIds); $scope.showSetting = false; @@ -483,7 +485,7 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro } scope[varName] = data.angularObject.object; } - + }); var isFunction = function(functionToCheck) { From f9244c3b4e644e9e5a39ee9ca1341697f1232345 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:22:02 +0200 Subject: [PATCH 13/71] Upgrade tests --- .../java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 0d4e1588262..f8319e21e8c 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -26,7 +26,6 @@ import java.util.Map; import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.server.ZeppelinServer; import org.junit.AfterClass; @@ -99,10 +98,10 @@ public void getSettings() throws IOException { @Test public void testInterpreterAutoBinding() throws IOException { // create note - Note note = ZeppelinServer.notebook.createNote(); + Note note = ZeppelinServer.notebook.createNote("anonymous"); // check interpreter is bindded - GetMethod get = httpGet("/notebook/interpreter/bind/"+note.id()); + GetMethod get = httpGet("/notebook/interpreter/bind/"+note.id()+"/anonymous/anonymous"); assertThat(get, isAllowed()); Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>(){}.getType()); List> body = (List>) resp.get("body"); From ba1da1429f043e4b26afd59f1b59dc84418b431e Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:22:21 +0200 Subject: [PATCH 14/71] Message hold the username & ticket now --- .../src/main/java/org/apache/zeppelin/socket/Message.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java index e4626bf7721..fadbcdf0369 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java @@ -98,6 +98,8 @@ public static enum OP { } public OP op; + public String ticket; + public String principal; public Map data = new HashMap(); public Message(OP op) { From 7ab1b235d83cd330012904ab1060b5d62192102f Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:22:44 +0200 Subject: [PATCH 15/71] Angular objects are attached specific to a user --- .../main/java/org/apache/zeppelin/display/AngularObject.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java index bbfcd1b184d..b0fbe272d55 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java @@ -33,6 +33,7 @@ public class AngularObject { private String name; private T object; + private String principal; private transient AngularObjectListener listener; private transient List watchers = new LinkedList(); @@ -48,6 +49,10 @@ public String getName() { return name; } + public String getPrincipal() { return principal; } + + public void setPrincipal(String principal) { this.principal = principal; } + @Override public boolean equals(Object o) { if (o instanceof AngularObject) { From 284e2ea7bb4ea9729c468472385a21bb117b5f7f Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:56:53 +0200 Subject: [PATCH 16/71] For an anonymous user, set ticket to anonymous to allow testing --- .../java/org/apache/zeppelin/rest/SecurityRestApi.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java index c8fad87b97e..d448500be23 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java @@ -43,7 +43,14 @@ public Response ticket() { principal = "anonymous"; else principal = oprincipal.toString(); - String ticket = TicketContainer.instance.getTicket(principal); + + // ticket set to anonymous for anonymous user. Simplify testing. + String ticket; + if ("anonymous".equals(principal)) + ticket = "anonymous"; + else + ticket = TicketContainer.instance.getTicket(principal); + Map data = new HashMap<>(); data.put("principal", principal); data.put("ticket", ticket); From cd05b62a011e8fb421319183a12245c6edd59c86 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 12:44:34 +0200 Subject: [PATCH 17/71] Add missing import --- .../test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index f8319e21e8c..d7c64a912ae 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -26,6 +26,7 @@ import java.util.Map; import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.server.ZeppelinServer; import org.junit.AfterClass; From 5960221e8f14ac1d25142f4379d3fedf3e4ce2e2 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 12:53:04 +0200 Subject: [PATCH 18/71] Add Apache Licence --- .../apache/zeppelin/rest/SecurityRestApi.java | 23 +++++++++++++++---- .../zeppelin/ticket/TicketContainer.java | 17 ++++++++++++++ zeppelin-server/src/main/resources/shiro.ini | 17 ++++++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java index d448500be23..e9ae622f3ae 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.rest; import org.apache.zeppelin.server.JsonResponse; @@ -12,11 +29,9 @@ import java.util.Map; /** - * Zeppelin root rest api endpoint. - * - * @author anthonycorbacho - * @since 0.3.4 + * Created by hayssams on 24/04/15. */ + @Path("/security") @Produces("application/json") public class SecurityRestApi { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java index c1c4b62684a..40662501c54 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.ticket; import java.util.Calendar; diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini index f619315f6b6..408c491e8b7 100644 --- a/zeppelin-server/src/main/resources/shiro.ini +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + [users] admin = password From 059b0fd327d5409d0f63bee1739df9b951e0d220 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 15:27:14 +0200 Subject: [PATCH 19/71] anonymous/anonymous is always a valid username/ticket combination --- .../main/java/org/apache/zeppelin/ticket/TicketContainer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java index 40662501c54..2bed45e9006 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java @@ -45,6 +45,9 @@ private static class Entry { public static final TicketContainer instance = new TicketContainer(); public boolean isValid(String principal, String ticket) { + // Always accept anonymous with ticket anonymous + if ("anonymous".equals(principal) && "anonymous".equals(ticket)) + return true; Entry entry = sessions.get(principal); return entry != null && entry.ticket.equals(ticket); } From aa4199ec405b5b614a11227603e166dc9c4b3eaa Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 11:03:09 +0200 Subject: [PATCH 20/71] Use HTTP Header for Auth --- .../apache/zeppelin/rest/NotebookRestApi.java | 18 +++++++----------- .../app/scripts/controllers/notebook.js | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index a3483727b98..4732fde7b6c 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -21,11 +21,7 @@ import java.util.LinkedList; import java.util.List; -import javax.ws.rs.GET; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; +import javax.ws.rs.*; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; @@ -61,10 +57,10 @@ public NotebookRestApi(Notebook notebook) { * @throws IOException */ @PUT - @Path("interpreter/bind/{noteId}/{principal}/{ticket}") + @Path("interpreter/bind/{noteId}") public Response bind(@PathParam("noteId") String noteId, - @PathParam("principal") String principal, - @PathParam("ticket") String ticket, String req) throws Exception { + @HeaderParam("X-Principal") String principal, + @HeaderParam("X-Ticket") String ticket, String req) throws Exception { if (!TicketContainer.instance.isValid(principal, ticket)) throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); @@ -77,10 +73,10 @@ public Response bind(@PathParam("noteId") String noteId, * list binded setting */ @GET - @Path("interpreter/bind/{noteId}/{principal}/{ticket}") + @Path("interpreter/bind/{noteId}") public Response bind(@PathParam("noteId") String noteId, - @PathParam("principal") String principal, - @PathParam("ticket") String ticket) throws Exception { + @HeaderParam("X-Principal") String principal, + @HeaderParam("X-Ticket") String ticket) throws Exception { if (!TicketContainer.instance.isValid(principal, ticket)) throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); diff --git a/zeppelin-web/app/scripts/controllers/notebook.js b/zeppelin-web/app/scripts/controllers/notebook.js index 0d775972888..897089b91fe 100644 --- a/zeppelin-web/app/scripts/controllers/notebook.js +++ b/zeppelin-web/app/scripts/controllers/notebook.js @@ -350,9 +350,19 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro } }; + /** Config header */ + var getTicketHeader = function(principal, ticket) { + return { + headers: { + 'X-Principal': principal, + 'X-Ticket': ticket + } + }; + } + /** user dependent*/ var getInterpreterBindings = function(callback) { - $http.get(getRestApiBase()+ '/notebook/interpreter/bind/' +$scope.note.id + "/" + $rootScope.ticket.principal+ "/" + $rootScope.ticket.ticket). + $http.get(getRestApiBase()+ '/notebook/interpreter/bind/' +$scope.note.id, getTicketHeader($rootScope.ticket.principal, $rootScope.ticket.ticket)). success(function(data, status, headers, config) { $scope.interpreterBindings = data.body; $scope.interpreterBindingsOrig = jQuery.extend(true, [], $scope.interpreterBindings); // to check dirty @@ -420,8 +430,8 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro } } - $http.put(getRestApiBase() + '/notebook/interpreter/bind/' + $scope.note.id + "/" + $rootScope.ticket.principal+ "/" + $rootScope.ticket.ticket, - selectedSettingIds). + $http.put(getRestApiBase() + '/notebook/interpreter/bind/' + $scope.note.id, selectedSettingIds, + getTicketHeader($rootScope.ticket.principal, $rootScope.ticket.ticket)). success(function(data, status, headers, config) { console.log('Interpreter binding %o saved', selectedSettingIds); $scope.showSetting = false; From 9c5316c5e44edbb90c88e95efda8cf03888665f1 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 11:03:28 +0200 Subject: [PATCH 21/71] Remove reference to stormpath --- zeppelin-server/src/main/resources/shiro.ini | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini index 408c491e8b7..19693e1085c 100644 --- a/zeppelin-server/src/main/resources/shiro.ini +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -20,15 +20,10 @@ admin = password [main] -# Let's use some in-memory caching to reduce the number of runtime lookups against Stormpath. -# A real application might want to use a more robust caching solution (e.g. ehcache or a -# distributed cache). When using such caches, be aware of your cache TTL settings: too high -# a TTL and the cache won't reflect any potential changes in Stormpath fast enough. Too low -# and the cache could evict too often, reducing performance. cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager securityManager.cacheManager = $cacheManager [urls] -/** = anon -#/** = authcBasic +#/** = anon +/** = authcBasic From 6118b0742e72d79c880757f963fcbed51284b4a2 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 11:40:39 +0200 Subject: [PATCH 22/71] Merge with Zeppelin 26 #47 --- .../org/apache/zeppelin/notebook/Note.java | 104 ++++--------- .../apache/zeppelin/notebook/NoteInfo.java | 13 +- .../apache/zeppelin/notebook/Notebook.java | 146 ++++++++---------- .../zeppelin/notebook/repo/NotebookRepo.java | 5 +- .../notebook/repo/VFSNotebookRepo.java | 64 ++++++-- 5 files changed, 160 insertions(+), 172 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 0fc1ea19003..4aec9b8c183 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -17,9 +17,6 @@ package org.apache.zeppelin.notebook; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.Serializable; import java.util.HashMap; @@ -28,27 +25,21 @@ import java.util.Map; import java.util.Random; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.utility.IdHashes; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; -import org.apache.zeppelin.scheduler.Scheduler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - /** * Binded interpreters for a note */ @@ -57,13 +48,14 @@ public class Note implements Serializable, JobListener { List paragraphs = new LinkedList(); private String name; private String id; - private String owner; + private String owner = "anonymous"; Map> angularObjects = new HashMap>(); private transient NoteInterpreterLoader replLoader; private transient ZeppelinConfiguration conf; private transient JobListenerFactory jobListenerFactory; + private transient NotebookRepo repo; /** * note configurations. @@ -79,13 +71,14 @@ public class Note implements Serializable, JobListener { */ private Map info = new HashMap(); + public Note() {} - public Note(ZeppelinConfiguration conf, NoteInterpreterLoader replLoader, - JobListenerFactory jobListenerFactory, - org.quartz.Scheduler quartzSched, - String owner) { - this.conf = conf; + public Note(NotebookRepo repo, + NoteInterpreterLoader replLoader, + JobListenerFactory jobListenerFactory, + String owner) { + this.repo = repo; this.replLoader = replLoader; this.jobListenerFactory = jobListenerFactory; this.owner = owner; @@ -100,7 +93,9 @@ public String id() { return id; } - public String owner() { return this.owner; } + public String getOwner() { return this.owner; } + + public String setOwner() { return this.owner; } public String getName() { return name; @@ -118,8 +113,20 @@ public void setReplLoader(NoteInterpreterLoader replLoader) { this.replLoader = replLoader; } - public void setZeppelinConfiguration(ZeppelinConfiguration conf) { - this.conf = conf; + public JobListenerFactory getJobListenerFactory() { + return jobListenerFactory; + } + + public void setJobListenerFactory(JobListenerFactory jobListenerFactory) { + this.jobListenerFactory = jobListenerFactory; + } + + public NotebookRepo getNotebookRepo() { + return repo; + } + + public void setNotebookRepo(NotebookRepo repo) { + this.repo = repo; } public Map> getAngularObjects() { @@ -129,7 +136,6 @@ public Map> getAngularObjects() { /** * Add paragraph last. * - * @param p */ public Paragraph addParagraph() { Paragraph p = new Paragraph(this, this, replLoader); @@ -143,7 +149,6 @@ public Paragraph addParagraph() { * Insert paragraph in given index. * * @param index - * @param p */ public Paragraph insertParagraph(int index) { Paragraph p = new Paragraph(this, this, replLoader); @@ -242,7 +247,6 @@ public Paragraph getLastParagraph() { /** * Run all paragraphs sequentially. * - * @param jobListener */ public void runAll() { synchronized (paragraphs) { @@ -300,63 +304,11 @@ private void snapshotAngularObjectRegistry() { } public void persist() throws IOException { - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.setPrettyPrinting(); - Gson gson = gsonBuilder.create(); - - File dir = new File(conf.getNotebookDir() + File.separator + owner + File.separator + id); - if (!dir.exists()) { - dir.mkdirs(); - } else if (dir.isFile()) { - throw new RuntimeException("File already exists" + dir.toString()); - } - - File file = new File(dir, "note.json"); - logger().info("Persist note {} into {}", id, file.getAbsolutePath()); - - snapshotAngularObjectRegistry(); - String json = gson.toJson(this); - FileOutputStream out = new FileOutputStream(file); - out.write(json.getBytes(conf.getString(ConfVars.ZEPPELIN_ENCODING))); - out.close(); + repo.save(this); } public void unpersist() throws IOException { - File dir = new File(conf.getNotebookDir() + File.separator + owner + File.separator + id); - - FileUtils.deleteDirectory(dir); - } - - public static Note load(String id, ZeppelinConfiguration conf, NoteInterpreterLoader replLoader, - Scheduler scheduler, JobListenerFactory jobListenerFactory, - org.quartz.Scheduler quartzSched, String principal) - throws IOException { - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.setPrettyPrinting(); - Gson gson = gsonBuilder.create(); - - File file = new File(conf.getNotebookDir() + File.separator + principal + - File.separator + id + File.separator + "note.json"); - logger().info("Load note {} from {}", id, file.getAbsolutePath()); - - if (!file.isFile()) { - return null; - } - - FileInputStream ins = new FileInputStream(file); - String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); - Note note = gson.fromJson(json, Note.class); - note.setZeppelinConfiguration(conf); - note.setReplLoader(replLoader); - note.jobListenerFactory = jobListenerFactory; - for (Paragraph p : note.paragraphs) { - p.setNote(note); - - if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { - p.setStatus(Status.ABORT); - } - } - return note; + repo.remove(this.id, this.owner); } public Map getConfig() { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java index db07e50e088..77dfd3a953b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java @@ -26,18 +26,21 @@ public class NoteInfo { String id; String name; + String owner; private Map config = new HashMap(); - public NoteInfo(String id, String name, Map config) { + public NoteInfo(String id, String name, String owner, Map config) { super(); this.id = id; this.name = name; + this.owner = owner; this.config = config; } public NoteInfo(Note note) { id = note.id(); name = note.getName(); + owner = note.getOwner(); config = note.getConfig(); } @@ -57,6 +60,14 @@ public void setName(String name) { this.name = name; } + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + public Map getConfig() { return config; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 841c2e42806..171aa51802a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -17,18 +17,15 @@ package org.apache.zeppelin.notebook; -import java.io.File; import java.io.IOException; import java.util.*; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObject; -import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.InterpreterFactory; -import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterSetting; -import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.quartz.CronScheduleBuilder; import org.quartz.CronTrigger; @@ -56,11 +53,14 @@ public class Notebook { private StdSchedulerFactory quertzSchedFact; private org.quartz.Scheduler quartzSched; private JobListenerFactory jobListenerFactory; + private NotebookRepo notebookRepo; - public Notebook(ZeppelinConfiguration conf, SchedulerFactory schedulerFactory, + public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, + SchedulerFactory schedulerFactory, InterpreterFactory replFactory, JobListenerFactory jobListenerFactory) throws IOException, SchedulerException { this.conf = conf; + this.notebookRepo = notebookRepo; this.schedulerFactory = schedulerFactory; this.replFactory = replFactory; this.jobListenerFactory = jobListenerFactory; @@ -106,7 +106,7 @@ private Map getUserNotes(String principal) { */ public Note createNote(List interpreterIds, String principal) throws IOException { NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory); - Note note = new Note(conf, intpLoader, jobListenerFactory, quartzSched, principal); + Note note = new Note(notebookRepo, intpLoader, jobListenerFactory, principal); intpLoader.setNoteId(note.id()); synchronized (notes) { getUserNotes(principal).put(note.id(), note); @@ -115,6 +115,7 @@ public Note createNote(List interpreterIds, String principal) throws IOE bindInterpretersToNote(note.id(), interpreterIds, principal); } + note.persist(); return note; } @@ -163,91 +164,74 @@ public void removeNote(String id, String principal) { } } - private void loadAllNotes() throws IOException { - File notebookDir = new File(conf.getNotebookDir()); - logger.info("Notebook Directory " + notebookDir.getAbsolutePath()); - File[] userdirs = notebookDir.listFiles(); - if (userdirs == null) { - return; + private Note loadNoteFromRepo(String id, String owner) { + Note note = null; + try { + note = notebookRepo.get(id, owner); + } catch (IOException e) { + logger.error("Failed to load " + id, e); } + if (note == null) { + return null; + } + + // set NoteInterpreterLoader + NoteInterpreterLoader noteInterpreterLoader = new NoteInterpreterLoader( + replFactory); + note.setReplLoader(noteInterpreterLoader); + noteInterpreterLoader.setNoteId(note.id()); + // set JobListenerFactory + note.setJobListenerFactory(jobListenerFactory); + + // set notebookRepo + note.setNotebookRepo(notebookRepo); Map angularObjectSnapshot = - new HashMap(); - - for (File fuser : userdirs) { - String principal = fuser.getName(); - boolean isHidden = principal.startsWith("."); - if (fuser.isDirectory() && !isHidden) { - File[] dirs = fuser.listFiles(); - if (dirs != null) { - for (File f : dirs) { - boolean isHiddenFile = f.getName().startsWith("."); - if (f.isDirectory() && !isHiddenFile) { - Scheduler scheduler = - schedulerFactory.createOrGetFIFOScheduler("note_" + System.currentTimeMillis()); - logger.info("Loading note from " + f.getName()); - NoteInterpreterLoader noteInterpreterLoader = new NoteInterpreterLoader(replFactory); - Note note = Note.load(f.getName(), - conf, - noteInterpreterLoader, - scheduler, - jobListenerFactory, quartzSched, fuser.getName()); - logger.info("Loading note from " + f.getName()); - noteInterpreterLoader.setNoteId(note.id()); - - // restore angular object -------------- - Date lastUpdatedDate = new Date(0); - for (Paragraph p : note.getParagraphs()) { - if (p.getDateFinished() != null && - lastUpdatedDate.before(p.getDateFinished())) { - lastUpdatedDate = p.getDateFinished(); - } - } - - Map> savedObjects = note.getAngularObjects(); - - if (savedObjects != null) { - for (String intpGroupName : savedObjects.keySet()) { - List objectList = savedObjects.get(intpGroupName); - - for (AngularObject savedObject : objectList) { - SnapshotAngularObject snapshot = - angularObjectSnapshot.get(savedObject.getName()); - if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) { - angularObjectSnapshot.put( - savedObject.getName(), - new SnapshotAngularObject( - intpGroupName, - savedObject, - lastUpdatedDate)); - } - } - } - } - - synchronized (notes) { - getUserNotes(principal).put(note.id(), note); - refreshCron(note.id(), principal); - } - } - } - } + new HashMap(); + + // restore angular object -------------- + Date lastUpdatedDate = new Date(0); + for (Paragraph p : note.getParagraphs()) { + p.setNote(note); + if (p.getDateFinished() != null && + lastUpdatedDate.before(p.getDateFinished())) { + lastUpdatedDate = p.getDateFinished(); } } - for (String name : angularObjectSnapshot.keySet()) { - SnapshotAngularObject snapshot = angularObjectSnapshot.get(name); - List settings = replFactory.get(); - for (InterpreterSetting setting : settings) { - InterpreterGroup intpGroup = setting.getInterpreterGroup(); - if (intpGroup.getId().equals(snapshot.getIntpGroupId())) { - AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry(); - if (registry.get(name) == null) { - registry.add(name, snapshot.getAngularObject().get(), false); + Map> savedObjects = note.getAngularObjects(); + + if (savedObjects != null) { + for (String intpGroupName : savedObjects.keySet()) { + List objectList = savedObjects.get(intpGroupName); + + for (AngularObject savedObject : objectList) { + SnapshotAngularObject snapshot = angularObjectSnapshot.get(savedObject.getName()); + if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) { + angularObjectSnapshot.put( + savedObject.getName(), + new SnapshotAngularObject( + intpGroupName, + savedObject, + lastUpdatedDate)); } } } } + + synchronized (notes) { + getUserNotes(owner).put(note.id(), note); + refreshCron(note.id(), owner); + } + return note; + } + + private void loadAllNotes() throws IOException { + List noteInfos = notebookRepo.list(); + + for (NoteInfo info : noteInfos) { + loadNoteFromRepo(info.getId(), info.getOwner()); + } } class SnapshotAngularObject { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java index 07e08758ee4..0605c7fae34 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java @@ -28,7 +28,8 @@ */ public interface NotebookRepo { public List list() throws IOException; - public Note get(String noteId) throws IOException; + public List list(String owner) throws IOException; + public Note get(String noteId, String owner) throws IOException; public void save(Note note) throws IOException; - public void remove(String noteId) throws IOException; + public void remove(String noteId, String owner) throws IOException; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 3039f80ff7d..2135902457b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -100,8 +100,8 @@ private boolean isDirectory(FileObject fo) throws IOException { } @Override - public List list() throws IOException { - FileObject rootDir = getRootDir(); + public List list(String owner) throws IOException { + FileObject rootDir = getRootDir(owner); FileObject[] children = rootDir.getChildren(); @@ -109,9 +109,9 @@ public List list() throws IOException { for (FileObject f : children) { String fileName = f.getName().getBaseName(); if (f.isHidden() - || fileName.startsWith(".") - || fileName.startsWith("#") - || fileName.startsWith("~")) { + || fileName.startsWith(".") + || fileName.startsWith("#") + || fileName.startsWith("~")) { // skip hidden, temporary files continue; } @@ -137,6 +137,43 @@ public List list() throws IOException { return infos; } + @Override + public List list() throws IOException { + FileObject rootDir = getRootDir(); + logger.info(rootDir.getName().getPath()); + FileObject[] children = rootDir.getChildren(); + + List infos = new LinkedList<>(); + for (FileObject f : children) { + String owner = f.getName().getBaseName(); + logger.info("OWNER=" + owner); + if (f.isHidden() + || owner.startsWith(".") + || owner.startsWith("#") + || owner.startsWith("~")) { + // skip hidden, temporary files + continue; + } + + if (!isDirectory(f)) { + // currently one directory per user saved like, [OWNER]/[NOTE_ID]/note.json. + // so it must be a directory + continue; + } + + try { + List ownerInfos = list(owner); + if (ownerInfos != null) { + infos.addAll(ownerInfos); + } + } catch (IOException e) { + logger.error("Can't read note " + f.getName().toString(), e); + } + } + + return infos; + } + private Note getNote(FileObject noteDir) throws IOException { if (!isDirectory(noteDir)) { throw new IOException(noteDir.getName().toString() + " is not a directory"); @@ -175,15 +212,15 @@ private NoteInfo getNoteInfo(FileObject noteDir) throws IOException { } @Override - public Note get(String noteId) throws IOException { - FileObject rootDir = fsManager.resolveFile(getPath("/")); + public Note get(String noteId, String owner) throws IOException { + FileObject rootDir = fsManager.resolveFile(getPath("/" + owner)); FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD); return getNote(noteDir); } - private FileObject getRootDir() throws IOException { - FileObject rootDir = fsManager.resolveFile(getPath("/")); + private FileObject getRootDir(String owner) throws IOException { + FileObject rootDir = fsManager.resolveFile(getPath(owner != null ? "/" + owner : "/")); if (!rootDir.exists()) { throw new IOException("Root path does not exists"); @@ -195,6 +232,9 @@ private FileObject getRootDir() throws IOException { return rootDir; } + private FileObject getRootDir() throws IOException { + return getRootDir(null); + } @Override public void save(Note note) throws IOException { @@ -203,7 +243,7 @@ public void save(Note note) throws IOException { Gson gson = gsonBuilder.create(); String json = gson.toJson(note); - FileObject rootDir = getRootDir(); + FileObject rootDir = getRootDir(note.getOwner()); FileObject noteDir = rootDir.resolveFile(note.id(), NameScope.CHILD); @@ -222,8 +262,8 @@ public void save(Note note) throws IOException { } @Override - public void remove(String noteId) throws IOException { - FileObject rootDir = fsManager.resolveFile(getPath("/")); + public void remove(String noteId, String owner) throws IOException { + FileObject rootDir = fsManager.resolveFile(getPath("/" + owner)); FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD); if (!noteDir.exists()) { From a471ded8eddee8ff1b47927275ebed2c6231fafc Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 11:40:59 +0200 Subject: [PATCH 23/71] Update Test --- .../test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 3379753cd4b..7a2f050c672 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -117,7 +117,7 @@ public void testInterpreterAutoBinding() throws IOException { @Test public void testInterpreterRestart() throws IOException, InterruptedException { // create new note - Note note = ZeppelinServer.notebook.createNote(); + Note note = ZeppelinServer.notebook.createNote("anonymous"); note.addParagraph(); Paragraph p = note.getLastParagraph(); From 869663b4601b3d90883232b08a4e9c681d8a45db Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 11:41:32 +0200 Subject: [PATCH 24/71] Handle correctly same user multiple devices refresh --- .../java/org/apache/zeppelin/socket/NotebookServer.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 8b2ce699161..87472d1139a 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -270,11 +270,13 @@ private void broadcastAll(WebSocket conn, Message m) { List conns = userSocketMap.get(principal); if (conns == null) { conns = new LinkedList<>(); - conns.add(conn); userSocketMap.put(principal, conns); } + if (!conns.contains(conn)) { + conns.add(conn); + } for (WebSocket theconn : conns) { - theconn.send(serializeMessage(m)); + theconn.send(serializeMessage(m)); } } } From 18e26ee0a9214dbd2c95e615bc75791fa508e2bb Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 11:55:52 +0200 Subject: [PATCH 25/71] Correct checktyle violation --- .../main/java/org/apache/zeppelin/socket/NotebookServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 87472d1139a..58213881a86 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -276,7 +276,7 @@ private void broadcastAll(WebSocket conn, Message m) { conns.add(conn); } for (WebSocket theconn : conns) { - theconn.send(serializeMessage(m)); + theconn.send(serializeMessage(m)); } } } From 97e6f8e60a2339b4cdea59cae9a24b81f3d03702 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 14:30:55 +0200 Subject: [PATCH 26/71] Make sure user folder exists before saving --- .../apache/zeppelin/notebook/repo/VFSNotebookRepo.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 2135902457b..3321f14c1f7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -242,10 +242,16 @@ public void save(Note note) throws IOException { gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.create(); String json = gson.toJson(note); + FileObject rootDir = getRootDir(); + + rootDir.createFolder(); + + FileObject ownerDir = fsManager.resolveFile(rootDir, note.getOwner()); - FileObject rootDir = getRootDir(note.getOwner()); + if (!ownerDir.exists()) + ownerDir.createFolder(); - FileObject noteDir = rootDir.resolveFile(note.id(), NameScope.CHILD); + FileObject noteDir = ownerDir.resolveFile(note.id(), NameScope.CHILD); if (!noteDir.exists()) { noteDir.createFolder(); From 167df710474a23725002140f746d9762eebcdc02 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 16:03:28 +0200 Subject: [PATCH 27/71] disable auth in tests --- zeppelin-server/src/main/resources/shiro.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini index 19693e1085c..84c90f6297b 100644 --- a/zeppelin-server/src/main/resources/shiro.ini +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -25,5 +25,5 @@ securityManager.cacheManager = $cacheManager [urls] -#/** = anon -/** = authcBasic +/** = anon +#/** = authcBasic From b0855371bde14e1397e5f7dd91ba8e6a22fa8d0a Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 18:54:26 +0200 Subject: [PATCH 28/71] Remove credentials from url in JUnit tests --- .../test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 7a2f050c672..3e28cc84c44 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -105,7 +105,7 @@ public void testInterpreterAutoBinding() throws IOException { Note note = ZeppelinServer.notebook.createNote("anonymous"); // check interpreter is bindded - GetMethod get = httpGet("/notebook/interpreter/bind/"+note.id()+"/anonymous/anonymous"); + GetMethod get = httpGet("/notebook/interpreter/bind/"+note.id()); assertThat(get, isAllowed()); Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>(){}.getType()); List> body = (List>) resp.get("body"); From 7913fe6dfa12c0d46101e35828c6347668bb97c8 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 19:18:17 +0200 Subject: [PATCH 29/71] Add auth Header to JUnit test --- .../java/org/apache/zeppelin/rest/AbstractTestRestApi.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index ac40dda07f2..aadb66104f5 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -135,6 +135,8 @@ protected static GetMethod httpGet(String path) throws IOException { LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); GetMethod getMethod = new GetMethod(url + path); + getMethod.addRequestHeader("X-Principal", "anonymous"); + getMethod.addRequestHeader("X-Ticket", "anonymous"); httpClient.executeMethod(getMethod); LOG.info("{} - {}", getMethod.getStatusCode(), getMethod.getStatusText()); return getMethod; @@ -144,6 +146,8 @@ protected static PostMethod httpPost(String path, String body) throws IOExceptio LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); PostMethod postMethod = new PostMethod(url + path); + postMethod.addRequestHeader("X-Principal", "anonymous"); + postMethod.addRequestHeader("X-Ticket", "anonymous"); RequestEntity entity = new ByteArrayRequestEntity(body.getBytes("UTF-8")); postMethod.setRequestEntity(entity); httpClient.executeMethod(postMethod); From 6cd9c9d278e752dfcb9a8d1a985cbef7114ff866 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 7 May 2015 19:36:04 +0200 Subject: [PATCH 30/71] Display currently logged user name on main screen --- zeppelin-web/app/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-web/app/index.html b/zeppelin-web/app/index.html index 2ad0e972034..6579aa1218c 100644 --- a/zeppelin-web/app/index.html +++ b/zeppelin-web/app/index.html @@ -84,7 +84,7 @@
  • - Connected + {{ticket.principal}} Connected Disconnected
From 80ca373e10baf993c9a03e94b6d1d932a506a193 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 8 May 2015 03:10:27 +0200 Subject: [PATCH 31/71] merge principal & ticket into a single header --- .../org/apache/zeppelin/rest/NotebookRestApi.java | 15 ++++++++++----- .../apache/zeppelin/rest/AbstractTestRestApi.java | 6 ++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 4732fde7b6c..1d4b543eb35 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -58,9 +58,12 @@ public NotebookRestApi(Notebook notebook) { */ @PUT @Path("interpreter/bind/{noteId}") - public Response bind(@PathParam("noteId") String noteId, - @HeaderParam("X-Principal") String principal, - @HeaderParam("X-Ticket") String ticket, String req) throws Exception { + public Response bind( + @PathParam("noteId") String noteId, + @HeaderParam("X-Ticket") String principalAndTicket, String req) throws Exception { + String[] tokens = principalAndTicket.split(":"); + String principal = tokens[0]; + String ticket = tokens[1]; if (!TicketContainer.instance.isValid(principal, ticket)) throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); @@ -75,8 +78,10 @@ public Response bind(@PathParam("noteId") String noteId, @GET @Path("interpreter/bind/{noteId}") public Response bind(@PathParam("noteId") String noteId, - @HeaderParam("X-Principal") String principal, - @HeaderParam("X-Ticket") String ticket) throws Exception { + @HeaderParam("X-Ticket") String principalAndTicket) throws Exception { + String[] tokens = principalAndTicket.split(":"); + String principal = tokens[0]; + String ticket = tokens[1]; if (!TicketContainer.instance.isValid(principal, ticket)) throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index aadb66104f5..36f574e9deb 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -135,8 +135,7 @@ protected static GetMethod httpGet(String path) throws IOException { LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); GetMethod getMethod = new GetMethod(url + path); - getMethod.addRequestHeader("X-Principal", "anonymous"); - getMethod.addRequestHeader("X-Ticket", "anonymous"); + getMethod.addRequestHeader("X-Ticket", "anonymous:anonymous"); httpClient.executeMethod(getMethod); LOG.info("{} - {}", getMethod.getStatusCode(), getMethod.getStatusText()); return getMethod; @@ -146,8 +145,7 @@ protected static PostMethod httpPost(String path, String body) throws IOExceptio LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); PostMethod postMethod = new PostMethod(url + path); - postMethod.addRequestHeader("X-Principal", "anonymous"); - postMethod.addRequestHeader("X-Ticket", "anonymous"); + postMethod.addRequestHeader("X-Ticket", "anonymous:anonymous"); RequestEntity entity = new ByteArrayRequestEntity(body.getBytes("UTF-8")); postMethod.setRequestEntity(entity); httpClient.executeMethod(postMethod); From 4ceb507a4fbafd1aa6afce6b5748eb94fd4f8e6b Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 8 May 2015 04:41:15 +0200 Subject: [PATCH 32/71] Use principal from shiro context --- .../apache/zeppelin/rest/NotebookRestApi.java | 30 +++++++++---------- .../zeppelin/rest/AbstractTestRestApi.java | 2 -- .../app/scripts/controllers/notebook.js | 15 ++-------- 3 files changed, 17 insertions(+), 30 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 1d4b543eb35..14cbac00d81 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -25,6 +25,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.shiro.SecurityUtils; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.rest.message.InterpreterSettingListForNoteBind; @@ -59,13 +60,13 @@ public NotebookRestApi(Notebook notebook) { @PUT @Path("interpreter/bind/{noteId}") public Response bind( - @PathParam("noteId") String noteId, - @HeaderParam("X-Ticket") String principalAndTicket, String req) throws Exception { - String[] tokens = principalAndTicket.split(":"); - String principal = tokens[0]; - String ticket = tokens[1]; - if (!TicketContainer.instance.isValid(principal, ticket)) - throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); + @PathParam("noteId") String noteId, String req) throws Exception { + Object oprincipal = SecurityUtils.getSubject().getPrincipal(); + String principal; + if (oprincipal == null) + principal = "anonymous"; + else + principal = oprincipal.toString(); List settingIdList = gson.fromJson(req, new TypeToken>(){}.getType()); notebook.bindInterpretersToNote(noteId, settingIdList, principal); @@ -77,14 +78,13 @@ public Response bind( */ @GET @Path("interpreter/bind/{noteId}") - public Response bind(@PathParam("noteId") String noteId, - @HeaderParam("X-Ticket") String principalAndTicket) throws Exception { - String[] tokens = principalAndTicket.split(":"); - String principal = tokens[0]; - String ticket = tokens[1]; - if (!TicketContainer.instance.isValid(principal, ticket)) - throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); - + public Response bind(@PathParam("noteId") String noteId) throws Exception { + Object oprincipal = SecurityUtils.getSubject().getPrincipal(); + String principal; + if (oprincipal == null) + principal = "anonymous"; + else + principal = oprincipal.toString(); List settingList = new LinkedList<>(); List selectedSettings = diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 36f574e9deb..ac40dda07f2 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -135,7 +135,6 @@ protected static GetMethod httpGet(String path) throws IOException { LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); GetMethod getMethod = new GetMethod(url + path); - getMethod.addRequestHeader("X-Ticket", "anonymous:anonymous"); httpClient.executeMethod(getMethod); LOG.info("{} - {}", getMethod.getStatusCode(), getMethod.getStatusText()); return getMethod; @@ -145,7 +144,6 @@ protected static PostMethod httpPost(String path, String body) throws IOExceptio LOG.info("Connecting to {}", url + path); HttpClient httpClient = new HttpClient(); PostMethod postMethod = new PostMethod(url + path); - postMethod.addRequestHeader("X-Ticket", "anonymous:anonymous"); RequestEntity entity = new ByteArrayRequestEntity(body.getBytes("UTF-8")); postMethod.setRequestEntity(entity); httpClient.executeMethod(postMethod); diff --git a/zeppelin-web/app/scripts/controllers/notebook.js b/zeppelin-web/app/scripts/controllers/notebook.js index 897089b91fe..4c0996fd2aa 100644 --- a/zeppelin-web/app/scripts/controllers/notebook.js +++ b/zeppelin-web/app/scripts/controllers/notebook.js @@ -350,19 +350,9 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro } }; - /** Config header */ - var getTicketHeader = function(principal, ticket) { - return { - headers: { - 'X-Principal': principal, - 'X-Ticket': ticket - } - }; - } - /** user dependent*/ var getInterpreterBindings = function(callback) { - $http.get(getRestApiBase()+ '/notebook/interpreter/bind/' +$scope.note.id, getTicketHeader($rootScope.ticket.principal, $rootScope.ticket.ticket)). + $http.get(getRestApiBase()+ '/notebook/interpreter/bind/' +$scope.note.id). success(function(data, status, headers, config) { $scope.interpreterBindings = data.body; $scope.interpreterBindingsOrig = jQuery.extend(true, [], $scope.interpreterBindings); // to check dirty @@ -430,8 +420,7 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro } } - $http.put(getRestApiBase() + '/notebook/interpreter/bind/' + $scope.note.id, selectedSettingIds, - getTicketHeader($rootScope.ticket.principal, $rootScope.ticket.ticket)). + $http.put(getRestApiBase() + '/notebook/interpreter/bind/' + $scope.note.id, selectedSettingIds). success(function(data, status, headers, config) { console.log('Interpreter binding %o saved', selectedSettingIds); $scope.showSetting = false; From 5c36dc3cb9ba6842ae340509cb9a6efd89a5e751 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 12 Jul 2015 15:19:04 +0200 Subject: [PATCH 33/71] Apache licence --- zeppelin-server/src/main/resources/shiro.ini | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini index a08a6fe9498..408c491e8b7 100644 --- a/zeppelin-server/src/main/resources/shiro.ini +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -1,19 +1,19 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# [users] admin = password From 2103a7114061020f12329806f24ffe820fdc8224 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 20 Sep 2015 21:28:52 +0200 Subject: [PATCH 34/71] syntax error (missing tag) --- zeppelin-server/pom.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 2f8e6b4ffd2..9fc27781955 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -257,6 +257,12 @@ test + + org.mockito + mockito-all + 1.9.0 + + org.apache.shiro @@ -265,12 +271,6 @@ org.apache.shiro shiro-web - - - org.mockito - mockito-all - 1.9.0 - From b5c1e00f657cf0f20dab84a208c5a9ad6694ab9d Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 20 Sep 2015 23:18:47 +0200 Subject: [PATCH 35/71] remove unused import --- .../src/main/java/org/apache/zeppelin/server/ZeppelinServer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index a0f2ab1b301..c5f7d276621 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -41,7 +41,6 @@ import org.apache.zeppelin.rest.ZeppelinRestApi; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.socket.NotebookServer; -import org.apache.zeppelin.socket.SslWebSocketServerFactory; import org.apache.zeppelin.rest.SecurityRestApi; import org.eclipse.jetty.server.AbstractConnector; import org.eclipse.jetty.server.Handler; From 3d3498eec982f109508ab6d207c0d2f0c2a09c79 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 20 Sep 2015 23:19:29 +0200 Subject: [PATCH 36/71] Make Notebook sync work with user specific notes --- .../notebook/repo/NotebookRepoSync.java | 75 ++++++++++--------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java index 288cff97671..f6ebe53231c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java @@ -74,20 +74,28 @@ public NotebookRepoSync(ZeppelinConfiguration conf) throws Exception { public List list() throws IOException { return getRepo(0).list(); } - - /* list from specific repo (for tests) */ + + public List list(String owner) throws IOException { + return getRepo(0).list(owner); + } + + /* list from specific repo (for tests) */ List list(int repoIndex) throws IOException { return getRepo(repoIndex).list(); } - /* by default returns from first repository */ - public Note get(String noteId) throws IOException { - return getRepo(0).get(noteId); + List list(int repoIndex, String owner) throws IOException { + return getRepo(repoIndex).list(owner); + } + + /* by default returns from first repository */ + public Note get(String noteId, String owner) throws IOException { + return getRepo(0).get(noteId, owner); } /* get note from specific repo (for tests) */ - Note get(int repoIndex, String noteId) throws IOException { - return getRepo(repoIndex).get(noteId); + Note get(int repoIndex, String noteId, String owner) throws IOException { + return getRepo(repoIndex).get(noteId, owner); } /* by default saves to all repos */ @@ -108,9 +116,9 @@ void save(int repoIndex, Note note) throws IOException { getRepo(repoIndex).save(note); } - public void remove(String noteId) throws IOException { + public void remove(String noteId, String owner) throws IOException { for (NotebookRepo repo : repos) { - repo.remove(noteId); + repo.remove(noteId, owner); } /* TODO(khalid): handle case when removing from secondary storage fails */ } @@ -126,28 +134,28 @@ public void sync(int sourceRepoIndex, int destRepoIndex) throws IOException { List sourceNotes = sourceRepo.list(); List destNotes = destRepo.list(); - Map> noteIDs = notesCheckDiff(sourceNotes, + Map> noteInfos = notesCheckDiff(sourceNotes, sourceRepo, destNotes, destRepo); - List pushNoteIDs = noteIDs.get(pushKey); - List pullNoteIDs = noteIDs.get(pullKey); - if (!pushNoteIDs.isEmpty()) { + List pushNoteInfos = noteInfos.get(pushKey); + List pullNoteInfos = noteInfos.get(pullKey); + if (!pushNoteInfos.isEmpty()) { LOG.info("Notes with the following IDs will be pushed"); - for (String id : pushNoteIDs) { - LOG.info("ID : " + id); + for (NoteInfo noteInfo : pushNoteInfos) { + LOG.info("ID : " + noteInfo.getId()); } - pushNotes(pushNoteIDs, sourceRepo, destRepo); + pushNotes(pushNoteInfos, sourceRepo, destRepo); } else { LOG.info("Nothing to push"); } - if (!pullNoteIDs.isEmpty()) { + if (!pullNoteInfos.isEmpty()) { LOG.info("Notes with the following IDs will be pulled"); - for (String id : pullNoteIDs) { - LOG.info("ID : " + id); + for (NoteInfo noteInfo : pullNoteInfos) { + LOG.info("ID : " + noteInfo.getId()); } - pushNotes(pullNoteIDs, destRepo, sourceRepo); + pushNotes(pullNoteInfos, destRepo, sourceRepo); } else { LOG.info("Nothing to pull"); } @@ -159,10 +167,10 @@ public void sync() throws IOException { sync(0, 1); } - private void pushNotes(List ids, NotebookRepo localRepo, + private void pushNotes(List noteInfos, NotebookRepo localRepo, NotebookRepo remoteRepo) throws IOException { - for (String id : ids) { - remoteRepo.save(localRepo.get(id)); + for (NoteInfo noteInfo: noteInfos) { + remoteRepo.save(localRepo.get(noteInfo.getId(), noteInfo.getOwner())); } } @@ -181,12 +189,12 @@ private NotebookRepo getRepo(int repoIndex) throws IOException { return repos.get(repoIndex); } - private Map> notesCheckDiff(List sourceNotes, + private Map> notesCheckDiff(List sourceNotes, NotebookRepo sourceRepo, List destNotes, NotebookRepo destRepo) throws IOException { - List pushIDs = new ArrayList(); - List pullIDs = new ArrayList(); + List pushIDs = new ArrayList(); + List pullIDs = new ArrayList(); NoteInfo dnote; Date sdate, ddate; @@ -194,22 +202,22 @@ private Map> notesCheckDiff(List sourceNotes, dnote = containsID(destNotes, snote.getId()); if (dnote != null) { /* note exists in source and destination storage systems */ - sdate = lastModificationDate(sourceRepo.get(snote.getId())); - ddate = lastModificationDate(destRepo.get(dnote.getId())); + sdate = lastModificationDate(sourceRepo.get(snote.getId(), snote.getOwner())); + ddate = lastModificationDate(destRepo.get(dnote.getId(), snote.getOwner())); if (sdate.after(ddate)) { /* source contains more up to date note - push */ - pushIDs.add(snote.getId()); + pushIDs.add(snote); LOG.info("Modified note is added to push list : " + sdate); } else if (sdate.compareTo(ddate) != 0) { /* destination contains more up to date note - pull */ LOG.info("Modified note is added to pull list : " + ddate); - pullIDs.add(snote.getId()); + pullIDs.add(snote); } } else { /* note exists in source storage, and absent in destination * view source as up to date - push * (another scenario : note was deleted from destination - not considered)*/ - pushIDs.add(snote.getId()); + pushIDs.add(snote); } } @@ -217,11 +225,11 @@ private Map> notesCheckDiff(List sourceNotes, dnote = containsID(sourceNotes, note.getId()); if (dnote == null) { /* note exists in destination storage, and absent in source - pull*/ - pullIDs.add(note.getId()); + pullIDs.add(note); } } - Map> map = new HashMap>(); + Map> map = new HashMap>(); map.put(pushKey, pushIDs); map.put(pullKey, pullIDs); return map; @@ -293,5 +301,4 @@ private void printNoteInfo(NoteInfo note) { entry.getValue().toString() + "of class " + entry.getClass()); } } - } From 39066d10f6f01903a87bdb1e79d416060862ca01 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 20 Sep 2015 23:19:49 +0200 Subject: [PATCH 37/71] make S3 work with user specific notes --- .../notebook/repo/S3NotebookRepo.java | 29 ++++++++++++------- .../zeppelin/notebook/NotebookTest.java | 8 ++--- .../notebook/repo/NotebookRepoSyncTest.java | 22 +++++++------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index bb9e5d1571d..cff939eccdd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -90,17 +90,23 @@ public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException { @Override public List list() throws IOException { + return list(null); + } + + @Override + public List list(String owner) throws IOException { List infos = new LinkedList(); NoteInfo info = null; + String ownerPath = owner == null ? "" : "/" + owner; try { ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) - .withPrefix(user + "/" + "notebook"); - ObjectListing objectListing; + .withPrefix(user + "/" + "notebook" + ownerPath); + ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); - - for (S3ObjectSummary objectSummary : + + for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { if (objectSummary.getKey().contains("note.json")) { try { @@ -113,11 +119,11 @@ public List list() throws IOException { } } } - + listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException ase) { - + } catch (AmazonClientException ace) { logger.info("Caught an AmazonClientException, " + "which means the client encountered " + @@ -156,8 +162,8 @@ private NoteInfo getNoteInfo(String key) throws IOException { } @Override - public Note get(String noteId) throws IOException { - return getNote(user + "/" + "notebook" + "/" + noteId + "/" + "note.json"); + public Note get(String noteId, String owner) throws IOException { + return getNote(user + "/" + "notebook" + "/" + owner + "/" + noteId + "/" + "note.json"); } @Override @@ -166,7 +172,8 @@ public void save(Note note) throws IOException { gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.create(); String json = gson.toJson(note); - String key = user + "/" + "notebook" + "/" + note.id() + "/" + "note.json"; + String key = user + "/" + "notebook" + "/" + + note.getOwner() + "/" + note.id() + "/" + "note.json"; File file = File.createTempFile("note", "json"); file.deleteOnExit(); @@ -179,9 +186,9 @@ public void save(Note note) throws IOException { } @Override - public void remove(String noteId) throws IOException { + public void remove(String noteId, String owner) throws IOException { - String key = user + "/" + "notebook" + "/" + noteId; + String key = user + "/" + "notebook" + "/" + owner + "/" + noteId; final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName).withPrefix(key); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index a0c10309a59..deb00c01cf8 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -164,7 +164,7 @@ public void testSchedule() throws InterruptedException, IOException{ public void testAngularObjectRemovalOnNotebookRemove() throws InterruptedException, IOException { // create a note and a paragraph - Note note = notebook.createNote(); + Note note = notebook.createNote("anonymous"); note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); AngularObjectRegistry registry = note.getNoteReplLoader() @@ -177,7 +177,7 @@ public void testAngularObjectRemovalOnNotebookRemove() throws InterruptedExcepti registry.add("o2", "object2", null); // remove notebook - notebook.removeNote(note.id()); + notebook.removeNote(note.id(), "anonymous"); // local object should be removed assertNull(registry.get("o1", note.id())); @@ -189,7 +189,7 @@ public void testAngularObjectRemovalOnNotebookRemove() throws InterruptedExcepti public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedException, IOException { // create a note and a paragraph - Note note = notebook.createNote(); + Note note = notebook.createNote("anonymous"); note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); AngularObjectRegistry registry = note.getNoteReplLoader() @@ -210,7 +210,7 @@ public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedExc // local and global scope object should be removed assertNull(registry.get("o1", note.id())); assertNull(registry.get("o2", null)); - notebook.removeNote(note.id()); + notebook.removeNote(note.id(), "anonymous"); } private void delete(File file){ diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 981ed6e2451..94ca9a1d059 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -110,7 +110,7 @@ public void testSyncOnCreate() throws IOException { assertEquals(0, notebookRepoSync.list(1).size()); /* create note */ - Note note = notebookSync.createNote(); + Note note = notebookSync.createNote("anonymous"); // check that automatically saved on both storages assertEquals(1, notebookRepoSync.list(0).size()); @@ -126,7 +126,7 @@ public void testSyncOnDelete() throws IOException { assertEquals(0, notebookRepoSync.list(0).size()); assertEquals(0, notebookRepoSync.list(1).size()); - Note note = notebookSync.createNote(); + Note note = notebookSync.createNote("anonymous"); /* check that created in both storage systems */ assertEquals(1, notebookRepoSync.list(0).size()); @@ -134,7 +134,7 @@ public void testSyncOnDelete() throws IOException { assertEquals(notebookRepoSync.list(0).get(0).getId(),notebookRepoSync.list(1).get(0).getId()); /* remove Note */ - notebookSync.removeNote(notebookRepoSync.list(0).get(0).getId()); + notebookSync.removeNote(notebookRepoSync.list(0).get(0).getId(), "anonymous"); /* check that deleted in both storages */ assertEquals(0, notebookRepoSync.list(0).size()); @@ -146,7 +146,7 @@ public void testSyncOnDelete() throws IOException { public void testSyncUpdateMain() throws IOException { /* create note */ - Note note = notebookSync.createNote(); + Note note = notebookSync.createNote("anonymous"); Paragraph p1 = note.addParagraph(); p1.setText("hello world"); @@ -155,29 +155,29 @@ public void testSyncUpdateMain() throws IOException { /* new paragraph not yet saved into storages */ assertEquals(0, notebookRepoSync.get(0, - notebookRepoSync.list(0).get(0).getId()).getParagraphs().size()); + notebookRepoSync.list(0).get(0).getId(), "anonymous").getParagraphs().size()); assertEquals(0, notebookRepoSync.get(1, - notebookRepoSync.list(1).get(0).getId()).getParagraphs().size()); + notebookRepoSync.list(1).get(0).getId(), "anonymous").getParagraphs().size()); /* save to storage under index 0 (first storage) */ notebookRepoSync.save(0, note); /* check paragraph saved to first storage */ assertEquals(1, notebookRepoSync.get(0, - notebookRepoSync.list(0).get(0).getId()).getParagraphs().size()); + notebookRepoSync.list(0).get(0).getId(), "anonymous").getParagraphs().size()); /* check paragraph isn't saved to second storage */ assertEquals(0, notebookRepoSync.get(1, - notebookRepoSync.list(1).get(0).getId()).getParagraphs().size()); + notebookRepoSync.list(1).get(0).getId(), "anonymous").getParagraphs().size()); /* apply sync */ notebookRepoSync.sync(); /* check whether added to second storage */ assertEquals(1, notebookRepoSync.get(1, - notebookRepoSync.list(1).get(0).getId()).getParagraphs().size()); + notebookRepoSync.list(1).get(0).getId(), "anonymous").getParagraphs().size()); /* check whether same paragraph id */ assertEquals(p1.getId(), notebookRepoSync.get(0, - notebookRepoSync.list(0).get(0).getId()).getLastParagraph().getId()); + notebookRepoSync.list(0).get(0).getId(), "anonymous").getLastParagraph().getId()); assertEquals(p1.getId(), notebookRepoSync.get(1, - notebookRepoSync.list(1).get(0).getId()).getLastParagraph().getId()); + notebookRepoSync.list(1).get(0).getId(), "anonymous").getLastParagraph().getId()); } private void delete(File file){ From c70f41c4183aa8177d7d8e5f1e766815f0cec89c Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 20 Sep 2015 23:29:32 +0200 Subject: [PATCH 38/71] use single quote in JS log message --- zeppelin-web/src/components/navbar/navbar.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 0a64960ba70..01d6c0e3a81 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -50,7 +50,7 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco vm.loadNotes(); }). error(function(data, status, headers, config) { - console.log("Could not get ticket"); + console.log('Could not get ticket'); }); function isActive(noteId) { From 5f0b496cac542dc396f93af3e8e59bf47a429384 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 25 Sep 2015 08:17:29 +0200 Subject: [PATCH 39/71] minor code updates to cope with changes on upstream --- .../display/AngularObjectRegistry.java | 2 +- .../AngularObjectRegistryListener.java | 2 +- .../remote/RemoteInterpreterServer.java | 7 +- .../display/AngularObjectRegistryTest.java | 2 +- .../remote/RemoteAngularObjectTest.java | 2 +- .../zeppelin/socket/NotebookServer.java | 77 ++++++++++--------- .../socket/NotebookSocketListener.java | 1 + .../rest/ZeppelinSparkClusterTest.java | 4 +- zeppelin-web/src/app/home/home.controller.js | 6 +- .../components/navbar/navbar.controller.js | 4 + 10 files changed, 58 insertions(+), 49 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java index d6bab7b732c..77980397138 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java @@ -119,7 +119,7 @@ public AngularObject remove(String name, String noteId, boolean emit) { Map r = getRegistryForKey(noteId); AngularObject o = r.remove(name); if (listener != null && emit) { - listener.onRemove(interpreterId, name, noteId);; + listener.onRemove(interpreterId, o); } return o; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java index 3ba57d7b1af..3f08efae4a2 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java @@ -24,5 +24,5 @@ public interface AngularObjectRegistryListener { public void onAdd(String interpreterGroupId, AngularObject object); public void onUpdate(String interpreterGroupId, AngularObject object); - public void onRemove(String interpreterGroupId, String name, String noteId); + public void onRemove(String interpreterGroupId, AngularObject object); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 16b188394d0..888826f5111 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -397,10 +397,10 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { } @Override - public void onRemove(String interpreterGroupId, String name, String noteId) { + public void onRemove(String interpreterGroupId, AngularObject object) { Map removeObject = new HashMap(); - removeObject.put("name", name); - removeObject.put("noteId", noteId); + removeObject.put("name", object.getName()); + removeObject.put("noteId", object.getNoteId()); sendEvent(new RemoteInterpreterEvent( RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE, gson.toJson(removeObject))); @@ -433,7 +433,6 @@ public RemoteInterpreterEvent getEvent() throws TException { /** * called when object is updated in client (web) side. - * @param className * @param name * @param noteId noteId where the update issues * @param object diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java index 43aca62c497..b693e6a4f42 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java @@ -45,7 +45,7 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { } @Override - public void onRemove(String interpreterGroupId, String name, String noteId) { + public void onRemove(String interpreterGroupId, AngularObject object) { onRemove.incrementAndGet(); } }); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index 29a1fb11972..fcc2f6c1fe9 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -184,7 +184,7 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { } @Override - public void onRemove(String interpreterGroupId, String name, String noteId) { + public void onRemove(String interpreterGroupId, AngularObject object) { onRemove.incrementAndGet(); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index d8c73bdbf2f..0dcd91e0f3a 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -16,10 +16,8 @@ */ package org.apache.zeppelin.socket; import java.io.IOException; -import java.net.InetSocketAddress; import java.util.*; -import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; import java.util.HashMap; @@ -45,9 +43,6 @@ import org.apache.zeppelin.server.ZeppelinServer; import org.apache.zeppelin.socket.Message.OP; import org.apache.zeppelin.ticket.TicketContainer; -import org.java_websocket.WebSocket; -import org.java_websocket.handshake.ClientHandshake; -import org.java_websocket.server.WebSocketServer; import org.apache.zeppelin.utils.SecurityUtils; import org.eclipse.jetty.websocket.WebSocket; import org.eclipse.jetty.websocket.WebSocketServlet; @@ -66,9 +61,7 @@ public class NotebookServer extends WebSocketServlet implements private static final Logger LOG = LoggerFactory .getLogger(NotebookServer.class); Gson gson = new Gson(); - Map> noteSocketMap = new HashMap>(); - Map> userSocketMap = new HashMap<>(); - List connectedSockets = new LinkedList(); + Map> userSocketMap = new HashMap<>(); final Map> noteSocketMap = new HashMap<>(); final List connectedSockets = new LinkedList<>(); @@ -175,24 +168,24 @@ public void onMessage(NotebookSocket conn, String msg) { } @Override - public void onClose(WebSocket conn, int code, String reason, boolean remote) { - LOG.info("Closed connection to {} : {}", conn.getRemoteSocketAddress().getHostName(), conn - .getRemoteSocketAddress().getPort()); + public void onClose(NotebookSocket conn, int code, String reason) { + LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() + .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); removeConnectionFromAllNote(conn); synchronized (userSocketMap) { - Collection> allSockets = userSocketMap.values(); - for (List userList : allSockets) { + Collection> allSockets = userSocketMap.values(); + for (List userList : allSockets) { userList.remove(conn); } } } @Override - public void onError(WebSocket conn, Exception message) { + public void onError(NotebookSocket conn, Exception message) { removeConnectionFromAllNote(conn); synchronized (userSocketMap) { - Collection> allSockets = userSocketMap.values(); - for (List userList : allSockets) { + Collection> allSockets = userSocketMap.values(); + for (List userList : allSockets) { userList.remove(conn); } } @@ -291,12 +284,12 @@ private void broadcast(String noteId, Message m) { } } - private void broadcastAll(WebSocket conn, Message m) { + private void broadcastAll(NotebookSocket conn, Message m) { synchronized (userSocketMap) { List> notesInfo = (List>) m.get("notes"); for ( Map info : notesInfo) { String principal = info.get("principal"); - List conns = userSocketMap.get(principal); + List conns = userSocketMap.get(principal); if (conns == null) { conns = new LinkedList<>(); userSocketMap.put(principal, conns); @@ -304,8 +297,12 @@ private void broadcastAll(WebSocket conn, Message m) { if (!conns.contains(conn)) { conns.add(conn); } - for (WebSocket theconn : conns) { - theconn.send(serializeMessage(m)); + for (NotebookSocket theconn : conns) { + try { + theconn.send(serializeMessage(m)); + } catch (IOException e) { + e.printStackTrace(); + } } } } @@ -315,7 +312,7 @@ private void broadcastNote(Note note) { broadcast(note.id(), new Message(OP.NOTE).put("note", note)); } - private void broadcastNoteList(WebSocket conn, Message fromMessage) { + private void broadcastNoteList(NotebookSocket conn, Message fromMessage) { Notebook notebook = notebook(); ZeppelinConfiguration conf = notebook.getConf(); String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); @@ -351,7 +348,7 @@ private void sendNote(NotebookSocket conn, Notebook notebook, } } - private void sendHomeNote(NotebookSocket conn, Notebook notebook + private void sendHomeNote(NotebookSocket conn, Notebook notebook, Message fromMessage) throws IOException { String noteId = notebook.getConf().getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); @@ -369,7 +366,7 @@ private void sendHomeNote(NotebookSocket conn, Notebook notebook } } - private void updateNote(WebSocket conn, Notebook notebook, Message fromMessage) + private void updateNote(NotebookSocket conn, Notebook notebook, Message fromMessage) throws SchedulerException, IOException { String noteId = (String) fromMessage.get("id"); String name = (String) fromMessage.get("name"); @@ -411,11 +408,12 @@ private boolean isCronUpdated(Map configA, return cronUpdated; } - private void createNote(WebSocket conn, Notebook notebook, Message fromMsg) throws IOException { + private void createNote(NotebookSocket conn, Notebook notebook, Message fromMsg) + throws IOException { Note note = notebook.createNote(fromMsg.principal); note.addParagraph(); // it's an empty note. so add one paragraph - if (message != null) { - String noteName = (String) message.get("name"); + if (fromMsg != null) { + String noteName = (String) fromMsg.get("name"); if (noteName == null || noteName.isEmpty()){ noteName = "Note " + note.getId(); } @@ -427,7 +425,7 @@ private void createNote(WebSocket conn, Notebook notebook, Message fromMsg) thro broadcastNoteList(conn, fromMsg); } - private void removeNote(WebSocket conn, Notebook notebook, Message fromMessage) + private void removeNote(NotebookSocket conn, Notebook notebook, Message fromMessage) throws IOException { String noteId = (String) fromMessage.get("id"); if (noteId == null) { @@ -462,15 +460,18 @@ private void cloneNote(NotebookSocket conn, Notebook notebook, Message fromMessa throws IOException, CloneNotSupportedException { String noteId = getOpenNoteId(conn); String name = (String) fromMessage.get("name"); - Note sourceNote = notebook.getNote(noteId); - Note newNote = notebook.createNote(); + Note sourceNote = notebook.getNote(noteId, fromMessage.principal); + Note newNote = notebook.createNote(fromMessage.principal); if (name != null) { newNote.setName(name); } // Copy the interpreter bindings List boundInterpreterSettingsIds = notebook - .getBindedInterpreterSettingsIds(sourceNote.id()); - notebook.bindInterpretersToNote(newNote.id(), boundInterpreterSettingsIds); + .getBindedInterpreterSettingsIds(sourceNote.id(), fromMessage.principal); + notebook.bindInterpretersToNote( + newNote.id(), + boundInterpreterSettingsIds, + fromMessage.principal); List paragraphs = sourceNote.getParagraphs(); for (Paragraph para : paragraphs) { @@ -479,7 +480,7 @@ private void cloneNote(NotebookSocket conn, Notebook notebook, Message fromMessa } newNote.persist(); broadcastNote(newNote); - broadcastNoteList(); + broadcastNoteList(conn, fromMessage); } private void removeParagraph(NotebookSocket conn, Notebook notebook, @@ -489,7 +490,7 @@ private void removeParagraph(NotebookSocket conn, Notebook notebook, return; } final Note note = notebook.getNote(getOpenNoteId(conn), fromMessage.principal); - /** We dont want to remove the last paragraph */ + /** We don't want to remove the last paragraph */ if (!note.isLastParagraph(paragraphId)) { note.removeParagraph(paragraphId); note.persist(); @@ -606,7 +607,7 @@ private void moveParagraph(NotebookSocket conn, Notebook notebook, broadcastNote(note); } - private void insertParagraph(WebSocket conn, Notebook notebook, Message fromMessage) + private void insertParagraph(NotebookSocket conn, Notebook notebook, Message fromMessage) throws IOException { final int index = (int) Double.parseDouble(fromMessage.get("index").toString()); final Note note = notebook.getNote(getOpenNoteId(conn), fromMessage.principal); @@ -773,11 +774,11 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { } @Override - public void onRemove(String interpreterGroupId, String name, String noteId) { + public void onRemove(String interpreterGroupId, AngularObject object) { Notebook notebook = notebook(); List notes = notebook.getAllNotes(object.getPrincipal()); for (Note note : notes) { - if (noteId != null && !note.id().equals(noteId)) { + if (object.getNoteId() != null && !note.id().equals(object.getNoteId())) { continue; } @@ -786,8 +787,8 @@ public void onRemove(String interpreterGroupId, String name, String noteId) { if (id.equals(interpreterGroupId)) { broadcast( note.id(), - new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", name).put( - "noteId", noteId)); + new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", object.getName()).put( + "noteId", object.getNoteId())); } } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocketListener.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocketListener.java index 77fed6ed7b1..475a21e4205 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocketListener.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocketListener.java @@ -22,5 +22,6 @@ public interface NotebookSocketListener { public void onClose(NotebookSocket socket, int code, String message); public void onOpen(NotebookSocket socket); + public void onError(NotebookSocket conn, Exception message); public void onMessage(NotebookSocket socket, String message); } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index 7fd537d21a4..4efd3fef0f7 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -136,12 +136,12 @@ public void zRunTest() throws IOException { @Test public void pySparkDepLoaderTest() throws IOException { // create new note - Note note = ZeppelinServer.notebook.createNote(); + Note note = ZeppelinServer.notebook.createNote("anonymous"); if (isPyspark() && getSparkVersionNumber(note) >= 14) { // restart spark interpreter List settings = - ZeppelinServer.notebook.getBindedInterpreterSettings(note.id()); + ZeppelinServer.notebook.getBindedInterpreterSettings(note.id(), "anonymous"); for (InterpreterSetting setting : settings) { if (setting.getGroup().equals("spark")) { diff --git a/zeppelin-web/src/app/home/home.controller.js b/zeppelin-web/src/app/home/home.controller.js index 64ff8801557..1a202194412 100644 --- a/zeppelin-web/src/app/home/home.controller.js +++ b/zeppelin-web/src/app/home/home.controller.js @@ -14,7 +14,11 @@ 'use strict'; angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, notebookListDataFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv) { - + $rootScope.ticket = { + 'principal':'anonymous', + 'ticket':'anonymous' + }; + var vm = this; vm.notes = notebookListDataFactory; vm.websocketMsgSrv = websocketMsgSrv; diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 01d6c0e3a81..73be946c14c 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -16,6 +16,10 @@ 'use strict'; angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootScope, $routeParams, notebookListDataFactory, websocketMsgSrv, arrayOrderingSrv, $http) { + $rootScope.ticket = { + 'principal':'anonymous', + 'ticket':'anonymous' + }; /** Current list of notes (ids) */ var vm = this; From 401af7b0cf9100e90e2cbd83a537b6220ab205ef Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:16:40 +0200 Subject: [PATCH 40/71] default shiro configuration file --- zeppelin-server/src/main/resources/shiro.ini | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 zeppelin-server/src/main/resources/shiro.ini diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini new file mode 100644 index 00000000000..f619315f6b6 --- /dev/null +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -0,0 +1,17 @@ +[users] +admin = password + +[main] + +# Let's use some in-memory caching to reduce the number of runtime lookups against Stormpath. +# A real application might want to use a more robust caching solution (e.g. ehcache or a +# distributed cache). When using such caches, be aware of your cache TTL settings: too high +# a TTL and the cache won't reflect any potential changes in Stormpath fast enough. Too low +# and the cache could evict too often, reducing performance. +cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager +securityManager.cacheManager = $cacheManager + + +[urls] +/** = anon +#/** = authcBasic From aabe114479d3a0863969d32035fb71b3ca25558c Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:17:33 +0200 Subject: [PATCH 41/71] The connected or anonymous user should get a ticket first from this API --- .../apache/zeppelin/rest/SecurityRestApi.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java new file mode 100644 index 00000000000..c8fad87b97e --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java @@ -0,0 +1,53 @@ +package org.apache.zeppelin.rest; + +import org.apache.zeppelin.server.JsonResponse; +import org.apache.zeppelin.ticket.TicketContainer; +import org.apache.shiro.SecurityUtils; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; +import java.util.HashMap; +import java.util.Map; + +/** + * Zeppelin root rest api endpoint. + * + * @author anthonycorbacho + * @since 0.3.4 + */ +@Path("/security") +@Produces("application/json") +public class SecurityRestApi { + /** + * Required by Swagger. + */ + public SecurityRestApi() { + super(); + } + + /** + * Get ticket + * Returns username & ticket + * for anonymous access, username is always anonymous. + * After getting this ticket, access through websockets become safe + * @return 200 response + */ + @GET + @Path("ticket") + public Response ticket() { + Object oprincipal = SecurityUtils.getSubject().getPrincipal(); + String principal; + if (oprincipal == null) + principal = "anonymous"; + else + principal = oprincipal.toString(); + String ticket = TicketContainer.instance.getTicket(principal); + Map data = new HashMap<>(); + data.put("principal", principal); + data.put("ticket", ticket); + + return new JsonResponse(Response.Status.OK, "", data).build(); + } +} From 30e2d3949ee8ca3f5aa191d229d56c1e7fcd2271 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:17:54 +0200 Subject: [PATCH 42/71] Shiro filter to protect access to the API. No need to proceed web resources here. --- .../org/apache/zeppelin/server/ZeppelinServer.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index a37fc224d65..5098540de14 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -38,6 +38,7 @@ import org.apache.zeppelin.notebook.repo.NotebookRepoSync; import org.apache.zeppelin.rest.InterpreterRestApi; import org.apache.zeppelin.rest.NotebookRestApi; +import org.apache.zeppelin.rest.SecurityRestApi; import org.apache.zeppelin.rest.ZeppelinRestApi; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.socket.NotebookServer; @@ -215,7 +216,11 @@ private static ServletContextHandler setupRestApiContextHandler() { cxfContext.addServlet(cxfServletHolder, "/*"); cxfContext.addFilter(new FilterHolder(CorsFilter.class), "/*", - EnumSet.allOf(DispatcherType.class)); + EnumSet.allOf(DispatcherType.class)); + + cxfContext.addFilter(org.apache.shiro.web.servlet.ShiroFilter.class, "/*", + EnumSet.allOf(DispatcherType.class)); + cxfContext.addEventListener(new org.apache.shiro.web.env.EnvironmentLoaderListener()); return cxfContext; } @@ -259,7 +264,7 @@ public Set> getClasses() { } @Override - public java.util.Set getSingletons() { + public Set getSingletons() { Set singletons = new HashSet(); /** Rest-api root endpoint */ @@ -272,6 +277,9 @@ public java.util.Set getSingletons() { InterpreterRestApi interpreterApi = new InterpreterRestApi(replFactory); singletons.add(interpreterApi); + SecurityRestApi securityApi = new SecurityRestApi(); + singletons.add(securityApi); + return singletons; } } From e842e2e37f5cc268297ed7f5d6a76052e7131473 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:18:13 +0200 Subject: [PATCH 43/71] Very simple ticket container No cleanup is done, since the same user accross different devices share the same ticket The Map size is at most the number of different user names having access to a Zeppelin instance --- .../zeppelin/ticket/TicketContainer.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java new file mode 100644 index 00000000000..c1c4b62684a --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java @@ -0,0 +1,51 @@ +package org.apache.zeppelin.ticket; + +import java.util.Calendar; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Created by hayssams on 24/04/15. + * Very simple ticket container + * No cleanup is done, since the same user accross different devices share the same ticket + * The Map size is at most the number of different user names having access to a Zeppelin instance + */ + + +public class TicketContainer { + private static class Entry { + public final String ticket; + // lastAccessTime still unused + public final long lastAccessTime; + Entry(String ticket) { + this.ticket = ticket; + this.lastAccessTime = Calendar.getInstance().getTimeInMillis(); + } + } + private Map sessions = new ConcurrentHashMap<>(); + + public static final TicketContainer instance = new TicketContainer(); + + public boolean isValid(String principal, String ticket) { + Entry entry = sessions.get(principal); + return entry != null && entry.ticket.equals(ticket); + } + + public synchronized String getTicket(String principal) { + Entry entry = sessions.get(principal); + String ticket; + if (entry == null) { + if (principal.equals("anonymous")) + ticket = "anonymous"; // enable testing on anonymous when ticket is required in the url + else + ticket = UUID.randomUUID().toString(); + } + else { + ticket = entry.ticket; + } + entry = new Entry(ticket); + sessions.put(principal, entry); + return ticket; + } +} From 71148894cd46db79eee7a04050eece3cdae1335a Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 27 Apr 2015 10:18:28 +0200 Subject: [PATCH 44/71] User name & ticket are now required to access the notebook api since each note is now attached to a user --- .../apache/zeppelin/rest/NotebookRestApi.java | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 8a933f70397..a3483727b98 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -33,6 +33,7 @@ import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.rest.message.InterpreterSettingListForNoteBind; import org.apache.zeppelin.server.JsonResponse; +import org.apache.zeppelin.ticket.TicketContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,10 +61,15 @@ public NotebookRestApi(Notebook notebook) { * @throws IOException */ @PUT - @Path("interpreter/bind/{noteId}") - public Response bind(@PathParam("noteId") String noteId, String req) throws IOException { + @Path("interpreter/bind/{noteId}/{principal}/{ticket}") + public Response bind(@PathParam("noteId") String noteId, + @PathParam("principal") String principal, + @PathParam("ticket") String ticket, String req) throws Exception { + if (!TicketContainer.instance.isValid(principal, ticket)) + throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); + List settingIdList = gson.fromJson(req, new TypeToken>(){}.getType()); - notebook.bindInterpretersToNote(noteId, settingIdList); + notebook.bindInterpretersToNote(noteId, settingIdList, principal); return new JsonResponse(Status.OK).build(); } @@ -71,12 +77,17 @@ public Response bind(@PathParam("noteId") String noteId, String req) throws IOEx * list binded setting */ @GET - @Path("interpreter/bind/{noteId}") - public Response bind(@PathParam("noteId") String noteId) { - List settingList - = new LinkedList(); + @Path("interpreter/bind/{noteId}/{principal}/{ticket}") + public Response bind(@PathParam("noteId") String noteId, + @PathParam("principal") String principal, + @PathParam("ticket") String ticket) throws Exception { + if (!TicketContainer.instance.isValid(principal, ticket)) + throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); + + List settingList = new LinkedList<>(); - List selectedSettings = notebook.getBindedInterpreterSettings(noteId); + List selectedSettings = + notebook.getBindedInterpreterSettings(noteId, principal); for (InterpreterSetting setting : selectedSettings) { settingList.add(new InterpreterSettingListForNoteBind( setting.id(), From b9d46ed213c327baa1d8bee4eac8e60d7ed3bdaf Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 25 Sep 2015 15:19:28 +0200 Subject: [PATCH 45/71] Revert to 6bc13e2 --- .../apache/zeppelin/rest/NotebookRestApi.java | 27 +++------- .../apache/zeppelin/rest/SecurityRestApi.java | 53 ------------------- .../zeppelin/server/ZeppelinServer.java | 12 +---- .../zeppelin/ticket/TicketContainer.java | 51 ------------------ zeppelin-server/src/main/resources/shiro.ini | 17 ------ 5 files changed, 10 insertions(+), 150 deletions(-) delete mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java delete mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java delete mode 100644 zeppelin-server/src/main/resources/shiro.ini diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index a3483727b98..8a933f70397 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -33,7 +33,6 @@ import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.rest.message.InterpreterSettingListForNoteBind; import org.apache.zeppelin.server.JsonResponse; -import org.apache.zeppelin.ticket.TicketContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,15 +60,10 @@ public NotebookRestApi(Notebook notebook) { * @throws IOException */ @PUT - @Path("interpreter/bind/{noteId}/{principal}/{ticket}") - public Response bind(@PathParam("noteId") String noteId, - @PathParam("principal") String principal, - @PathParam("ticket") String ticket, String req) throws Exception { - if (!TicketContainer.instance.isValid(principal, ticket)) - throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); - + @Path("interpreter/bind/{noteId}") + public Response bind(@PathParam("noteId") String noteId, String req) throws IOException { List settingIdList = gson.fromJson(req, new TypeToken>(){}.getType()); - notebook.bindInterpretersToNote(noteId, settingIdList, principal); + notebook.bindInterpretersToNote(noteId, settingIdList); return new JsonResponse(Status.OK).build(); } @@ -77,17 +71,12 @@ public Response bind(@PathParam("noteId") String noteId, * list binded setting */ @GET - @Path("interpreter/bind/{noteId}/{principal}/{ticket}") - public Response bind(@PathParam("noteId") String noteId, - @PathParam("principal") String principal, - @PathParam("ticket") String ticket) throws Exception { - if (!TicketContainer.instance.isValid(principal, ticket)) - throw new Exception("Invalid principal / ticket:" + principal + "/" + ticket); - - List settingList = new LinkedList<>(); + @Path("interpreter/bind/{noteId}") + public Response bind(@PathParam("noteId") String noteId) { + List settingList + = new LinkedList(); - List selectedSettings = - notebook.getBindedInterpreterSettings(noteId, principal); + List selectedSettings = notebook.getBindedInterpreterSettings(noteId); for (InterpreterSetting setting : selectedSettings) { settingList.add(new InterpreterSettingListForNoteBind( setting.id(), diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java deleted file mode 100644 index c8fad87b97e..00000000000 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.apache.zeppelin.rest; - -import org.apache.zeppelin.server.JsonResponse; -import org.apache.zeppelin.ticket.TicketContainer; -import org.apache.shiro.SecurityUtils; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.Response; -import java.util.HashMap; -import java.util.Map; - -/** - * Zeppelin root rest api endpoint. - * - * @author anthonycorbacho - * @since 0.3.4 - */ -@Path("/security") -@Produces("application/json") -public class SecurityRestApi { - /** - * Required by Swagger. - */ - public SecurityRestApi() { - super(); - } - - /** - * Get ticket - * Returns username & ticket - * for anonymous access, username is always anonymous. - * After getting this ticket, access through websockets become safe - * @return 200 response - */ - @GET - @Path("ticket") - public Response ticket() { - Object oprincipal = SecurityUtils.getSubject().getPrincipal(); - String principal; - if (oprincipal == null) - principal = "anonymous"; - else - principal = oprincipal.toString(); - String ticket = TicketContainer.instance.getTicket(principal); - Map data = new HashMap<>(); - data.put("principal", principal); - data.put("ticket", ticket); - - return new JsonResponse(Response.Status.OK, "", data).build(); - } -} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 5098540de14..a37fc224d65 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -38,7 +38,6 @@ import org.apache.zeppelin.notebook.repo.NotebookRepoSync; import org.apache.zeppelin.rest.InterpreterRestApi; import org.apache.zeppelin.rest.NotebookRestApi; -import org.apache.zeppelin.rest.SecurityRestApi; import org.apache.zeppelin.rest.ZeppelinRestApi; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.socket.NotebookServer; @@ -216,11 +215,7 @@ private static ServletContextHandler setupRestApiContextHandler() { cxfContext.addServlet(cxfServletHolder, "/*"); cxfContext.addFilter(new FilterHolder(CorsFilter.class), "/*", - EnumSet.allOf(DispatcherType.class)); - - cxfContext.addFilter(org.apache.shiro.web.servlet.ShiroFilter.class, "/*", - EnumSet.allOf(DispatcherType.class)); - cxfContext.addEventListener(new org.apache.shiro.web.env.EnvironmentLoaderListener()); + EnumSet.allOf(DispatcherType.class)); return cxfContext; } @@ -264,7 +259,7 @@ public Set> getClasses() { } @Override - public Set getSingletons() { + public java.util.Set getSingletons() { Set singletons = new HashSet(); /** Rest-api root endpoint */ @@ -277,9 +272,6 @@ public Set getSingletons() { InterpreterRestApi interpreterApi = new InterpreterRestApi(replFactory); singletons.add(interpreterApi); - SecurityRestApi securityApi = new SecurityRestApi(); - singletons.add(securityApi); - return singletons; } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java deleted file mode 100644 index c1c4b62684a..00000000000 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.apache.zeppelin.ticket; - -import java.util.Calendar; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Created by hayssams on 24/04/15. - * Very simple ticket container - * No cleanup is done, since the same user accross different devices share the same ticket - * The Map size is at most the number of different user names having access to a Zeppelin instance - */ - - -public class TicketContainer { - private static class Entry { - public final String ticket; - // lastAccessTime still unused - public final long lastAccessTime; - Entry(String ticket) { - this.ticket = ticket; - this.lastAccessTime = Calendar.getInstance().getTimeInMillis(); - } - } - private Map sessions = new ConcurrentHashMap<>(); - - public static final TicketContainer instance = new TicketContainer(); - - public boolean isValid(String principal, String ticket) { - Entry entry = sessions.get(principal); - return entry != null && entry.ticket.equals(ticket); - } - - public synchronized String getTicket(String principal) { - Entry entry = sessions.get(principal); - String ticket; - if (entry == null) { - if (principal.equals("anonymous")) - ticket = "anonymous"; // enable testing on anonymous when ticket is required in the url - else - ticket = UUID.randomUUID().toString(); - } - else { - ticket = entry.ticket; - } - entry = new Entry(ticket); - sessions.put(principal, entry); - return ticket; - } -} diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini deleted file mode 100644 index f619315f6b6..00000000000 --- a/zeppelin-server/src/main/resources/shiro.ini +++ /dev/null @@ -1,17 +0,0 @@ -[users] -admin = password - -[main] - -# Let's use some in-memory caching to reduce the number of runtime lookups against Stormpath. -# A real application might want to use a more robust caching solution (e.g. ehcache or a -# distributed cache). When using such caches, be aware of your cache TTL settings: too high -# a TTL and the cache won't reflect any potential changes in Stormpath fast enough. Too low -# and the cache could evict too often, reducing performance. -cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager -securityManager.cacheManager = $cacheManager - - -[urls] -/** = anon -#/** = authcBasic From df4ad07dc6e31d4ec25339308dbdd7e50aaab848 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 27 Sep 2015 14:19:19 +0200 Subject: [PATCH 46/71] what's root dir on travi-ci ? --- .../java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 96f935248d8..8cd886a1bdd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -221,7 +221,7 @@ private FileObject getRootDir(String owner) throws IOException { FileObject rootDir = fsManager.resolveFile(getPath(owner != null ? "/" + owner : "/")); if (!rootDir.exists()) { - throw new IOException("Root path does not exists"); + throw new IOException("Root path does not exists" + rootDir.getName().getPath()); } if (!isDirectory(rootDir)) { From 290c17b3e8f1dd4cc569d0f56834b4bf62b45080 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 27 Sep 2015 15:55:47 +0200 Subject: [PATCH 47/71] restore default note. --- notebook/2A94M5J1Z/note.json | 341 ++++++++++++++++++ .../notebook/repo/VFSNotebookRepo.java | 2 +- 2 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 notebook/2A94M5J1Z/note.json diff --git a/notebook/2A94M5J1Z/note.json b/notebook/2A94M5J1Z/note.json new file mode 100644 index 00000000000..785ccea3cee --- /dev/null +++ b/notebook/2A94M5J1Z/note.json @@ -0,0 +1,341 @@ +{ + "paragraphs": [ + { + "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836981412_-1007008116", + "id": "20150213-231621_168813393", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:16:21 PM", + "dateStarted": "Apr 1, 2015 9:11:09 PM", + "dateFinished": "Apr 1, 2015 9:11:10 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "title": "Load data into table", + "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "title": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500779206_-1502780787", + "id": "20150210-015259_1403135953", + "result": { + "code": "SUCCESS", + "type": "TEXT", + "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[32] at parallelize at \u003cconsole\u003e:65\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n" + }, + "dateCreated": "Feb 10, 2015 1:52:59 AM", + "dateStarted": "Jul 3, 2015 1:43:40 PM", + "dateFinished": "Jul 3, 2015 1:43:45 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423500782552_-1439281894", + "id": "20150210-015302_1492795503", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n" + }, + "dateCreated": "Feb 10, 2015 1:53:02 AM", + "dateStarted": "Jul 3, 2015 1:43:17 PM", + "dateFinished": "Jul 3, 2015 1:43:23 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "maxAge": "35" + }, + "forms": { + "maxAge": { + "name": "maxAge", + "defaultValue": "30", + "hidden": false + } + } + }, + "jobName": "paragraph_1423720444030_-1424110477", + "id": "20150212-145404_867439529", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n" + }, + "dateCreated": "Feb 12, 2015 2:54:04 PM", + "dateStarted": "Jul 3, 2015 1:43:28 PM", + "dateFinished": "Jul 3, 2015 1:43:29 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age", + "config": { + "colWidth": 4.0, + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false, + "keys": [ + { + "name": "age", + "index": 0.0, + "aggr": "sum" + } + ], + "values": [ + { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + ], + "groups": [], + "scatter": { + "xAxis": { + "name": "age", + "index": 0.0, + "aggr": "sum" + }, + "yAxis": { + "name": "value", + "index": 1.0, + "aggr": "sum" + } + } + } + }, + "settings": { + "params": { + "marital": "single" + }, + "forms": { + "marital": { + "name": "marital", + "defaultValue": "single", + "options": [ + { + "value": "single" + }, + { + "value": "divorced" + }, + { + "value": "married" + } + ], + "hidden": false + } + } + }, + "jobName": "paragraph_1423836262027_-210588283", + "id": "20150213-230422_1600658137", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n" + }, + "dateCreated": "Feb 13, 2015 11:04:22 PM", + "dateStarted": "Jul 3, 2015 1:43:33 PM", + "dateFinished": "Jul 3, 2015 1:43:34 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1423836268492_216498320", + "id": "20150213-230428_1231780373", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n" + }, + "dateCreated": "Feb 13, 2015 11:04:28 PM", + "dateStarted": "Apr 1, 2015 9:12:18 PM", + "dateFinished": "Apr 1, 2015 9:12:18 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + }, + "editorHide": true + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1427420818407_872443482", + "id": "20150326-214658_12335843", + "result": { + "code": "SUCCESS", + "type": "HTML", + "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n" + }, + "dateCreated": "Mar 26, 2015 9:46:58 PM", + "dateStarted": "Jul 3, 2015 1:44:56 PM", + "dateFinished": "Jul 3, 2015 1:44:56 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1435955447812_-158639899", + "id": "20150703-133047_853701097", + "dateCreated": "Jul 3, 2015 1:30:47 PM", + "status": "READY", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Zeppelin Tutorial", + "id": "2A94M5J1Z", + "angularObjects": {}, + "config": { + "looknfeel": "default" + }, + "info": {} +} \ No newline at end of file diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 8cd886a1bdd..96f935248d8 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -221,7 +221,7 @@ private FileObject getRootDir(String owner) throws IOException { FileObject rootDir = fsManager.resolveFile(getPath(owner != null ? "/" + owner : "/")); if (!rootDir.exists()) { - throw new IOException("Root path does not exists" + rootDir.getName().getPath()); + throw new IOException("Root path does not exists"); } if (!isDirectory(rootDir)) { From 638b53384f4902c3c61e6b309623357079420edb Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 27 Sep 2015 16:12:12 +0200 Subject: [PATCH 48/71] Move user notes to notebook/users folders --- .../notebook/repo/VFSNotebookRepo.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 96f935248d8..dd4b4b36782 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -138,7 +138,10 @@ public List list(String owner) throws IOException { @Override public List list() throws IOException { - FileObject rootDir = getRootDir(); + FileObject rootDir = fsManager.resolveFile(getRootDir(), "users"); + if (!rootDir.exists()) + rootDir.createFolder(); + logger.info(rootDir.getName().getPath()); FileObject[] children = rootDir.getChildren(); @@ -155,7 +158,7 @@ public List list() throws IOException { logger.info("OWNER=" + owner); if (!isDirectory(f)) { - // currently one directory per user saved like, [OWNER]/[NOTE_ID]/note.json. + // currently one directory per user saved like, users/[OWNER]/[NOTE_ID]/note.json. // so it must be a directory continue; } @@ -211,14 +214,14 @@ private NoteInfo getNoteInfo(FileObject noteDir) throws IOException { @Override public Note get(String noteId, String owner) throws IOException { - FileObject rootDir = fsManager.resolveFile(getPath("/" + owner)); + FileObject rootDir = fsManager.resolveFile(getPath("/users/" + owner)); FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD); return getNote(noteDir); } private FileObject getRootDir(String owner) throws IOException { - FileObject rootDir = fsManager.resolveFile(getPath(owner != null ? "/" + owner : "/")); + FileObject rootDir = fsManager.resolveFile(getPath(owner != null ? "/users/" + owner : "/")); if (!rootDir.exists()) { throw new IOException("Root path does not exists"); @@ -245,7 +248,11 @@ public void save(Note note) throws IOException { rootDir.createFolder(); - FileObject ownerDir = fsManager.resolveFile(rootDir, note.getOwner()); + FileObject usersDir = fsManager.resolveFile(rootDir, "users"); + if (!usersDir.exists()) + usersDir.createFolder(); + + FileObject ownerDir = fsManager.resolveFile(usersDir, note.getOwner()); if (!ownerDir.exists()) ownerDir.createFolder(); @@ -267,7 +274,7 @@ public void save(Note note) throws IOException { @Override public void remove(String noteId, String owner) throws IOException { - FileObject rootDir = fsManager.resolveFile(getPath("/" + owner)); + FileObject rootDir = fsManager.resolveFile(getPath("/users/" + owner)); FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD); if (!noteDir.exists()) { From 90f874f3035677898e3b1e107ba0ac2913cbe573 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 27 Sep 2015 16:21:14 +0200 Subject: [PATCH 49/71] Move user notes to notebook/users folders --- .../apache/zeppelin/notebook/repo/S3NotebookRepo.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index cff939eccdd..785a613f728 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -97,7 +97,7 @@ public List list() throws IOException { public List list(String owner) throws IOException { List infos = new LinkedList(); NoteInfo info = null; - String ownerPath = owner == null ? "" : "/" + owner; + String ownerPath = owner == null ? "" : "/users/" + owner; try { ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) @@ -163,7 +163,7 @@ private NoteInfo getNoteInfo(String key) throws IOException { @Override public Note get(String noteId, String owner) throws IOException { - return getNote(user + "/" + "notebook" + "/" + owner + "/" + noteId + "/" + "note.json"); + return getNote(user + "/notebook/users/" + owner + "/" + noteId + "/note.json"); } @Override @@ -172,8 +172,8 @@ public void save(Note note) throws IOException { gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.create(); String json = gson.toJson(note); - String key = user + "/" + "notebook" + "/" + - note.getOwner() + "/" + note.id() + "/" + "note.json"; + String key = user + "/notebook/users/" + + note.getOwner() + "/" + note.id() + "/note.json"; File file = File.createTempFile("note", "json"); file.deleteOnExit(); @@ -188,7 +188,7 @@ public void save(Note note) throws IOException { @Override public void remove(String noteId, String owner) throws IOException { - String key = user + "/" + "notebook" + "/" + owner + "/" + noteId; + String key = user + "/notebook/users/" + owner + "/" + noteId; final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName).withPrefix(key); From e65e66cdd9994274c7060cd8d8b1b04593105bc8 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 27 Sep 2015 17:51:38 +0200 Subject: [PATCH 50/71] remove socket registration in onOpen --- .../zeppelin/socket/NotebookServer.java | 95 ++++++++++--------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 0dcd91e0f3a..8f9e603775b 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -15,6 +15,7 @@ * limitations under the License. */ package org.apache.zeppelin.socket; + import java.io.IOException; import java.util.*; @@ -26,6 +27,7 @@ import java.util.Map; import java.util.Set; import javax.servlet.http.HttpServletRequest; + import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObject; @@ -51,23 +53,24 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Strings; import com.google.gson.Gson; + /** * Zeppelin websocket service. * * @author anthonycorbacho */ public class NotebookServer extends WebSocketServlet implements - NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener { + NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener { private static final Logger LOG = LoggerFactory - .getLogger(NotebookServer.class); + .getLogger(NotebookServer.class); Gson gson = new Gson(); Map> userSocketMap = new HashMap<>(); final Map> noteSocketMap = new HashMap<>(); - final List connectedSockets = new LinkedList<>(); private Notebook notebook() { return ZeppelinServer.notebook; } + @Override public boolean checkOrigin(HttpServletRequest request, String origin) { @@ -91,9 +94,6 @@ public WebSocket doWebSocketConnect(HttpServletRequest req, String protocol) { public void onOpen(NotebookSocket conn) { LOG.info("New connection from {} : {}", conn.getRequest().getRemoteAddr(), conn.getRequest().getRemotePort()); - synchronized (connectedSockets) { - connectedSockets.add(conn); - } } @Override @@ -108,7 +108,7 @@ public void onMessage(NotebookSocket conn, String msg) { if (ticket != null && !ticket.equals(messagereceived.ticket)) throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); - /** Lets be elegant here */ + /** Lets be elegant here */ switch (messagereceived.op) { case LIST_NOTES: broadcastNoteList(conn, messagereceived); @@ -170,7 +170,7 @@ public void onMessage(NotebookSocket conn, String msg) { @Override public void onClose(NotebookSocket conn, int code, String reason) { LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() - .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); + .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); removeConnectionFromAllNote(conn); synchronized (userSocketMap) { Collection> allSockets = userSocketMap.values(); @@ -254,7 +254,7 @@ private String getOpenNoteId(NotebookSocket socket) { } private void broadcastToNoteBindedInterpreter(String interpreterGroupId, - Message m) { + Message m) { Notebook notebook = notebook(); List notes = notebook.getAllNotes(m.principal); for (Note note : notes) { @@ -287,22 +287,20 @@ private void broadcast(String noteId, Message m) { private void broadcastAll(NotebookSocket conn, Message m) { synchronized (userSocketMap) { List> notesInfo = (List>) m.get("notes"); - for ( Map info : notesInfo) { - String principal = info.get("principal"); - List conns = userSocketMap.get(principal); - if (conns == null) { - conns = new LinkedList<>(); - userSocketMap.put(principal, conns); - } - if (!conns.contains(conn)) { - conns.add(conn); - } - for (NotebookSocket theconn : conns) { - try { - theconn.send(serializeMessage(m)); - } catch (IOException e) { - e.printStackTrace(); - } + String principal = m.principal; + List conns = userSocketMap.get(principal); + if (conns == null) { + conns = new LinkedList<>(); + userSocketMap.put(principal, conns); + } + if (!conns.contains(conn)) { + conns.add(conn); + } + for (NotebookSocket theconn : conns) { + try { + theconn.send(serializeMessage(m)); + } catch (IOException e) { + e.printStackTrace(); } } } @@ -331,11 +329,13 @@ private void broadcastNoteList(NotebookSocket conn, Message fromMessage) { info.put("principal", fromMessage.principal); notesInfo.add(info); } - broadcastAll(conn, new Message(OP.NOTES_INFO).put("notes", notesInfo)); + Message message = new Message(OP.NOTES_INFO).put("notes", notesInfo); + message.principal = fromMessage.principal; + broadcastAll(conn, message); } private void sendNote(NotebookSocket conn, Notebook notebook, - Message fromMessage) throws IOException { + Message fromMessage) throws IOException { String noteId = (String) fromMessage.get("id"); if (noteId == null) { return; @@ -349,7 +349,7 @@ private void sendNote(NotebookSocket conn, Notebook notebook, } private void sendHomeNote(NotebookSocket conn, Notebook notebook, - Message fromMessage) throws IOException { + Message fromMessage) throws IOException { String noteId = notebook.getConf().getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); Note note = null; @@ -394,7 +394,7 @@ private void updateNote(NotebookSocket conn, Notebook notebook, Message fromMess } private boolean isCronUpdated(Map configA, - Map configB) { + Map configB) { boolean cronUpdated = false; if (configA.get("cron") != null && configB.get("cron") != null && configA.get("cron").equals(configB.get("cron"))) { @@ -414,7 +414,7 @@ private void createNote(NotebookSocket conn, Notebook notebook, Message fromMsg) note.addParagraph(); // it's an empty note. so add one paragraph if (fromMsg != null) { String noteName = (String) fromMsg.get("name"); - if (noteName == null || noteName.isEmpty()){ + if (noteName == null || noteName.isEmpty()) { noteName = "Note " + note.getId(); } note.setName(noteName); @@ -439,7 +439,7 @@ private void removeNote(NotebookSocket conn, Notebook notebook, Message fromMess } private void updateParagraph(NotebookSocket conn, Notebook notebook, - Message fromMessage) throws IOException { + Message fromMessage) throws IOException { String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { return; @@ -455,7 +455,7 @@ private void updateParagraph(NotebookSocket conn, Notebook notebook, note.persist(); broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p)); } - + private void cloneNote(NotebookSocket conn, Notebook notebook, Message fromMessage) throws IOException, CloneNotSupportedException { String noteId = getOpenNoteId(conn); @@ -469,9 +469,9 @@ private void cloneNote(NotebookSocket conn, Notebook notebook, Message fromMessa List boundInterpreterSettingsIds = notebook .getBindedInterpreterSettingsIds(sourceNote.id(), fromMessage.principal); notebook.bindInterpretersToNote( - newNote.id(), - boundInterpreterSettingsIds, - fromMessage.principal); + newNote.id(), + boundInterpreterSettingsIds, + fromMessage.principal); List paragraphs = sourceNote.getParagraphs(); for (Paragraph para : paragraphs) { @@ -484,7 +484,7 @@ private void cloneNote(NotebookSocket conn, Notebook notebook, Message fromMessa } private void removeParagraph(NotebookSocket conn, Notebook notebook, - Message fromMessage) throws IOException { + Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { return; @@ -499,7 +499,7 @@ private void removeParagraph(NotebookSocket conn, Notebook notebook, } private void completion(NotebookSocket conn, Notebook notebook, - Message fromMessage) throws IOException { + Message fromMessage) throws IOException { String paragraphId = (String) fromMessage.get("id"); String buffer = (String) fromMessage.get("buf"); int cursor = (int) Double.parseDouble(fromMessage.get("cursor").toString()); @@ -518,12 +518,12 @@ private void completion(NotebookSocket conn, Notebook notebook, /** * When angular object updated from client * - * @param conn the web socket. - * @param notebook the notebook. + * @param conn the web socket. + * @param notebook the notebook. * @param fromMessage the message. */ private void angularObjectUpdated(WebSocket conn, Notebook notebook, - Message fromMessage) { + Message fromMessage) { String noteId = (String) fromMessage.get("noteId"); String interpreterGroupId = (String) fromMessage.get("interpreterGroupId"); String varName = (String) fromMessage.get("name"); @@ -594,7 +594,7 @@ private void angularObjectUpdated(WebSocket conn, Notebook notebook, } private void moveParagraph(NotebookSocket conn, Notebook notebook, - Message fromMessage) throws IOException { + Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { return; @@ -617,7 +617,7 @@ private void insertParagraph(NotebookSocket conn, Notebook notebook, Message fro } private void cancelParagraph(NotebookSocket conn, Notebook notebook, - Message fromMessage) throws IOException { + Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { return; @@ -629,7 +629,7 @@ private void cancelParagraph(NotebookSocket conn, Notebook notebook, } private void runParagraph(NotebookSocket conn, Notebook notebook, - Message fromMessage) throws IOException { + Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { return; @@ -640,10 +640,10 @@ private void runParagraph(NotebookSocket conn, Notebook notebook, p.setText(text); p.setTitle((String) fromMessage.get("title")); Map params = (Map) fromMessage - .get("params"); + .get("params"); p.settings.setParams(params); Map config = (Map) fromMessage - .get("config"); + .get("config"); p.setConfig(config); // if it's the last paragraph, let's add a new one boolean isTheLastParagraph = note.getLastParagraph().getId() @@ -668,11 +668,11 @@ private void runParagraph(NotebookSocket conn, Notebook notebook, /** * Need description here. - * */ public static class ParagraphJobListener implements JobListener { private NotebookServer notebookServer; private Note note; + public ParagraphJobListener(NotebookServer notebookServer, Note note) { this.notebookServer = notebookServer; this.note = note; @@ -714,6 +714,7 @@ public void afterStatusChange(Job job, Status before, Status after) { public JobListener getParagraphJobListener(Note note) { return new ParagraphJobListener(this, note); } + private void pong() { } @@ -788,7 +789,7 @@ public void onRemove(String interpreterGroupId, AngularObject object) { broadcast( note.id(), new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", object.getName()).put( - "noteId", object.getNoteId())); + "noteId", object.getNoteId())); } } } From 9b4b809002d13bceec45fe243fca3ab667a69189 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sun, 27 Sep 2015 23:13:23 +0200 Subject: [PATCH 51/71] minor code refactoring --- .../apache/zeppelin/rest/NotebookRestApi.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index cec2e796e74..3e4dcff330d 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -63,12 +63,7 @@ public NotebookRestApi(Notebook notebook) { @PUT @Path("interpreter/bind/{noteId}") public Response bind(@PathParam("noteId") String noteId, String req) throws IOException { - Object oprincipal = SecurityUtils.getSubject().getPrincipal(); - String principal; - if (oprincipal == null) - principal = "anonymous"; - else - principal = oprincipal.toString(); + String principal = getPrincipal(); List settingIdList = gson.fromJson(req, new TypeToken>(){}.getType()); notebook.bindInterpretersToNote(noteId, settingIdList, principal); @@ -81,12 +76,7 @@ public Response bind(@PathParam("noteId") String noteId, String req) throws IOEx @GET @Path("interpreter/bind/{noteId}") public Response bind(@PathParam("noteId") String noteId) { - Object oprincipal = SecurityUtils.getSubject().getPrincipal(); - String principal; - if (oprincipal == null) - principal = "anonymous"; - else - principal = oprincipal.toString(); + String principal = getPrincipal(); List settingList = new LinkedList(); @@ -125,4 +115,15 @@ public Response bind(@PathParam("noteId") String noteId) { } return new JsonResponse(Status.OK, "", settingList).build(); } + + private String getPrincipal() { + Object oprincipal = SecurityUtils.getSubject().getPrincipal(); + String principal; + if (oprincipal == null) + principal = "anonymous"; + else + principal = oprincipal.toString(); + return principal; + } + } From 0c8f9a932bc3a2a24657e81fc8eab9c2383bc0de Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Tue, 29 Sep 2015 00:14:59 +0200 Subject: [PATCH 52/71] start adding some Unit tests --- .../zeppelin/rest/SecurityRestApiTest.java | 58 +++++++++++++++++ .../zeppelin/socket/TicketContainerTest.java | 63 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 zeppelin-server/src/test/java/org/apache/zeppelin/rest/SecurityRestApiTest.java create mode 100644 zeppelin-server/src/test/java/org/apache/zeppelin/socket/TicketContainerTest.java diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/SecurityRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/SecurityRestApiTest.java new file mode 100644 index 00000000000..b496f99a117 --- /dev/null +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/SecurityRestApiTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.rest; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.httpclient.methods.GetMethod; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; + +import static org.junit.Assert.*; + +public class SecurityRestApiTest extends AbstractTestRestApi { + Gson gson = new Gson(); + + @BeforeClass + public static void init() throws Exception { + AbstractTestRestApi.startUp(); + } + + @AfterClass + public static void destroy() throws Exception { + AbstractTestRestApi.shutDown(); + } + + @Test + public void testTicket() throws IOException { + GetMethod get = httpGet("/security/ticket"); + get.addRequestHeader("Origin", "http://localhost"); + Map resp = gson.fromJson(get.getResponseBodyAsString(), + new TypeToken>(){}.getType()); + Map body = (Map) resp.get("body"); + assertEquals("anonymous", body.get("principal")); + assertEquals("anonymous", body.get("ticket")); + get.releaseConnection(); + } + +} + diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/TicketContainerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/TicketContainerTest.java new file mode 100644 index 00000000000..b3958055d70 --- /dev/null +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/TicketContainerTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.socket; + +import org.apache.zeppelin.ticket.TicketContainer; +import org.junit.Before; +import org.junit.Test; + +import java.net.UnknownHostException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TicketContainerTest { + private TicketContainer container; + + @Before + public void setUp() throws Exception { + container = TicketContainer.instance; + } + + @Test + public void isValidAnonymous() throws UnknownHostException { + boolean ok = container.isValid("anonymous", "anonymous"); + assertTrue(ok); + } + + @Test + public void isValidExistingPrincipal() throws UnknownHostException { + String ticket = container.getTicket("someuser1"); + boolean ok = container.isValid("someuser1", ticket); + assertTrue(ok); + } + + @Test + public void isValidNonExistingPrincipal() throws UnknownHostException { + boolean ok = container.isValid("unknownuser", "someticket"); + assertFalse(ok); + } + + @Test + public void isValidunkownTicket() throws UnknownHostException { + String ticket = container.getTicket("someuser2"); + boolean ok = container.isValid("someuser2", ticket+"makeitinvalid"); + assertFalse(ok); + } +} + From 42b37d12bec3697d71d258f7780ea551ec477618 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 9 Oct 2015 19:00:08 +0200 Subject: [PATCH 53/71] Include Martin correction to the following bug : if you go from a notebook to the home page by clicking on the top left Zeppelin link. In this case the user is set to anonymous. --- zeppelin-web/src/app/home/home.controller.js | 10 ++++++---- .../src/components/navbar/navbar.controller.js | 12 +++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/zeppelin-web/src/app/home/home.controller.js b/zeppelin-web/src/app/home/home.controller.js index 1a202194412..691ee57d0a9 100644 --- a/zeppelin-web/src/app/home/home.controller.js +++ b/zeppelin-web/src/app/home/home.controller.js @@ -14,10 +14,12 @@ 'use strict'; angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, notebookListDataFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv) { - $rootScope.ticket = { - 'principal':'anonymous', - 'ticket':'anonymous' - }; + if (!$rootScope.ticket) { + $rootScope.ticket = { + 'principal':'anonymous', + 'ticket':'anonymous' + }; + } var vm = this; vm.notes = notebookListDataFactory; diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 73be946c14c..4dc4cacc2ef 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -16,10 +16,12 @@ 'use strict'; angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootScope, $routeParams, notebookListDataFactory, websocketMsgSrv, arrayOrderingSrv, $http) { - $rootScope.ticket = { - 'principal':'anonymous', - 'ticket':'anonymous' - }; + if (!$rootScope.ticket) { + $rootScope.ticket = { + 'principal':'anonymous', + 'ticket':'anonymous' + }; + } /** Current list of notes (ids) */ var vm = this; @@ -27,7 +29,7 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco vm.connected = websocketMsgSrv.isConnected(); vm.websocketMsgSrv = websocketMsgSrv; vm.arrayOrderingSrv = arrayOrderingSrv; - + $('#notebook-list').perfectScrollbar({suppressScrollX: true}); $scope.$on('setNoteMenu', function(event, notes) { From f2775c26a9c9a6d1b2caf15e8b7de862eb748e4a Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 19 Oct 2015 15:30:07 +0200 Subject: [PATCH 54/71] Include support in note creation api --- .../apache/zeppelin/rest/NotebookRestApi.java | 17 +++-- .../zeppelin/socket/NotebookServer.java | 64 +++++++++++++------ .../zeppelin/rest/ZeppelinRestApiTest.java | 16 ++--- .../apache/zeppelin/notebook/Notebook.java | 5 +- 4 files changed, 64 insertions(+), 38 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 6c07e8cce14..3a5a0ae2548 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -145,10 +145,11 @@ public Response getNotebookList() throws IOException { @POST @Path("/") public Response createNote(String message) throws IOException { + String principal = getPrincipal(); logger.info("Create new notebook by JSON {}" , message); NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class); - Note note = notebook.createNote(); + Note note = notebook.createNote(principal); note.addParagraph(); // it's an empty note. so add one paragraph String noteName = request.getName(); if (noteName.isEmpty()) { @@ -157,7 +158,7 @@ public Response createNote(String message) throws IOException { note.setName(noteName); note.persist(); notebookServer.broadcastNote(note); - notebookServer.broadcastNoteList(); + notebookServer.broadcastNoteList(principal); return new JsonResponse(Status.CREATED, "", note.getId() ).build(); } @@ -171,13 +172,14 @@ public Response createNote(String message) throws IOException { @Path("{notebookId}") public Response deleteNote(@PathParam("notebookId") String notebookId) throws IOException { logger.info("Delete notebook {} ", notebookId); + String principal = getPrincipal(); if (!(notebookId.isEmpty())) { - Note note = notebook.getNote(notebookId); + Note note = notebook.getNote(notebookId, principal); if (note != null) { - notebook.removeNote(notebookId); + notebook.removeNote(notebookId, principal); } } - notebookServer.broadcastNoteList(); + notebookServer.broadcastNoteList(principal); return new JsonResponse(Status.OK, "").build(); } /** @@ -191,12 +193,13 @@ public Response deleteNote(@PathParam("notebookId") String notebookId) throws IO public Response cloneNote(@PathParam("notebookId") String notebookId, String message) throws IOException, CloneNotSupportedException, IllegalArgumentException { logger.info("clone notebook by JSON {}" , message); + String principal = getPrincipal(); NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class); String newNoteName = request.getName(); - Note newNote = notebook.cloneNote(notebookId, newNoteName); + Note newNote = notebook.cloneNote(notebookId, newNoteName, principal); notebookServer.broadcastNote(newNote); - notebookServer.broadcastNoteList(); + notebookServer.broadcastNoteList(principal); return new JsonResponse(Status.CREATED, "", newNote.getId()).build(); } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 64fe63d08e4..6dc51a50c2b 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -108,10 +108,12 @@ public void onMessage(NotebookSocket conn, String msg) { if (ticket != null && !ticket.equals(messagereceived.ticket)) throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); + addConnectionToUserSocketMap(conn, messagereceived); + /** Lets be elegant here */ switch (messagereceived.op) { case LIST_NOTES: - broadcastNoteList(conn, messagereceived); + broadcastNoteList(messagereceived.principal); break; case GET_HOME_NOTE: sendHomeNote(conn, notebook, messagereceived); @@ -159,7 +161,7 @@ public void onMessage(NotebookSocket conn, String msg) { angularObjectUpdated(conn, notebook, messagereceived); break; default: - broadcastNoteList(conn, messagereceived); + broadcastNoteList(messagereceived.principal); break; } } catch (Exception e) { @@ -167,6 +169,24 @@ public void onMessage(NotebookSocket conn, String msg) { } } + private void addConnectionToUserSocketMap(NotebookSocket conn, Message messagereceived) { + List conns = userSocketMap.get(messagereceived.principal); + + if (conns == null) { + synchronized (userSocketMap) { + conns = userSocketMap.get(messagereceived.principal); + if (conns == null) { + conns = new LinkedList<>(); + userSocketMap.put(messagereceived.principal, conns); + } + } + } + + if (!conns.contains(conn)) { + conns.add(conn); + } + } + @Override public void onClose(NotebookSocket conn, int code, String reason) { LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() @@ -284,7 +304,7 @@ private void broadcast(String noteId, Message m) { } } - private void broadcastAll(NotebookSocket conn, Message m) { + private void broadcastAll(Message m) { synchronized (userSocketMap) { List> notesInfo = (List>) m.get("notes"); String principal = m.principal; @@ -293,45 +313,47 @@ private void broadcastAll(NotebookSocket conn, Message m) { conns = new LinkedList<>(); userSocketMap.put(principal, conns); } - if (!conns.contains(conn)) { - conns.add(conn); - } + for (NotebookSocket theconn : conns) { try { theconn.send(serializeMessage(m)); } catch (IOException e) { - e.printStackTrace(); + LOG.error("socket error", e); } } } } - public void broadcastNote(Note note) { - broadcast(note.id(), new Message(OP.NOTE).put("note", note)); - } - - public void broadcastNoteList(NotebookSocket conn, Message fromMessage) { + public void broadcastNoteList(String principal) { Notebook notebook = notebook(); + ZeppelinConfiguration conf = notebook.getConf(); String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); boolean hideHomeScreenNotebookFromList = conf .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); - List notes = notebook.getAllNotes(fromMessage.principal); - List> notesInfo = new LinkedList>(); + List notes = notebook.getAllNotes(principal); + List> notesInfo = new LinkedList<>(); for (Note note : notes) { Map info = new HashMap<>(); + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { continue; } + info.put("id", note.id()); info.put("name", note.getName()); - info.put("principal", fromMessage.principal); + info.put("principal", principal); notesInfo.add(info); } + Message message = new Message(OP.NOTES_INFO).put("notes", notesInfo); - message.principal = fromMessage.principal; - broadcastAll(conn, message); + message.principal = principal; + broadcastAll(message); + } + + public void broadcastNote(Note note) { + broadcast(note.id(), new Message(OP.NOTE).put("note", note)); } private void sendNote(NotebookSocket conn, Notebook notebook, @@ -389,7 +411,7 @@ private void updateNote(NotebookSocket conn, Notebook notebook, Message fromMess note.persist(); broadcastNote(note); - broadcastNoteList(conn, fromMessage); + broadcastNoteList(fromMessage.principal); } } @@ -422,7 +444,7 @@ private void createNote(NotebookSocket conn, Notebook notebook, Message fromMsg) note.persist(); broadcastNote(note); - broadcastNoteList(conn, fromMsg); + broadcastNoteList(fromMsg.principal); } private void removeNote(NotebookSocket conn, Notebook notebook, Message fromMessage) @@ -435,7 +457,7 @@ private void removeNote(NotebookSocket conn, Notebook notebook, Message fromMess note.unpersist(); notebook.removeNote(noteId, fromMessage.principal); removeNote(noteId); - broadcastNoteList(conn, fromMessage); + broadcastNoteList(fromMessage.principal); } private void updateParagraph(NotebookSocket conn, Notebook notebook, @@ -462,7 +484,7 @@ private void cloneNote(NotebookSocket conn, Notebook notebook, Message fromMessa String name = (String) fromMessage.get("name"); Note newNote = notebook.cloneNote(noteId, name, fromMessage.principal); broadcastNote(newNote); - broadcastNoteList(conn, fromMessage); + broadcastNoteList(fromMessage.principal); } private void removeParagraph(NotebookSocket conn, Notebook notebook, diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index caa01d6b107..3ff14662904 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -204,7 +204,7 @@ private void testNotebookCreate(String noteName) throws IOException { String newNotebookId = (String) resp.get("body"); LOG.info("newNotebookId:=" + newNotebookId); - Note newNote = ZeppelinServer.notebook.getNote(newNotebookId); + Note newNote = ZeppelinServer.notebook.getNote(newNotebookId, "anonymous"); assertNotNull("Can not find new note by id", newNote); // This is partial test as newNote is in memory but is not persistent String newNoteName = newNote.getName(); @@ -215,7 +215,7 @@ private void testNotebookCreate(String noteName) throws IOException { } assertEquals("compare note name", expectedNoteName, newNoteName); // cleanup - ZeppelinServer.notebook.removeNote(newNotebookId); + ZeppelinServer.notebook.removeNote(newNotebookId, "anonymous"); post.releaseConnection(); } @@ -224,7 +224,7 @@ private void testNotebookCreate(String noteName) throws IOException { public void testDeleteNote() throws IOException { LOG.info("testDeleteNote"); //Create note and get ID - Note note = ZeppelinServer.notebook.createNote(); + Note note = ZeppelinServer.notebook.createNote("anonymous"); String noteId = note.getId(); testDeleteNotebook(noteId); } @@ -244,7 +244,7 @@ private void testDeleteNotebook(String notebookId) throws IOException { delete.releaseConnection(); // make sure note is deleted if (!notebookId.isEmpty()) { - Note deletedNote = ZeppelinServer.notebook.getNote(notebookId); + Note deletedNote = ZeppelinServer.notebook.getNote(notebookId, "anonymous"); assertNull("Deleted note should be null", deletedNote); } } @@ -253,7 +253,7 @@ private void testDeleteNotebook(String notebookId) throws IOException { public void testCloneNotebook() throws IOException, CloneNotSupportedException, IllegalArgumentException { LOG.info("testCloneNotebook"); // Create note to clone - Note note = ZeppelinServer.notebook.createNote(); + Note note = ZeppelinServer.notebook.createNote("anonymous"); assertNotNull("cant create new note", note); note.setName("source note for clone"); Paragraph paragraph = note.addParagraph(); @@ -273,13 +273,13 @@ public void testCloneNotebook() throws IOException, CloneNotSupportedException, String newNotebookId = (String) resp.get("body"); LOG.info("newNotebookId:=" + newNotebookId); - Note newNote = ZeppelinServer.notebook.getNote(newNotebookId); + Note newNote = ZeppelinServer.notebook.getNote(newNotebookId, "anonymous"); assertNotNull("Can not find new note by id", newNote); assertEquals("Compare note names", noteName, newNote.getName()); assertEquals("Compare paragraphs count", note.getParagraphs().size(), newNote.getParagraphs().size()); //cleanup - ZeppelinServer.notebook.removeNote(note.getId()); - ZeppelinServer.notebook.removeNote(newNote.getId()); + ZeppelinServer.notebook.removeNote(note.getId(), "anonymous"); + ZeppelinServer.notebook.removeNote(newNote.getId(), "anonymous"); post.releaseConnection(); } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 484c5ba6cd6..2ce8277e2d4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -138,7 +138,8 @@ public Note cloneNote(String sourceNoteID, String newNoteName, String principal) newNote.setName(newNoteName); } // Copy the interpreter bindings - List boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.id()); + List boundInterpreterSettingsIds = + getBindedInterpreterSettingsIds(sourceNote.id(), principal); bindInterpretersToNote(newNote.id(), boundInterpreterSettingsIds, principal); List paragraphs = sourceNote.getParagraphs(); @@ -307,7 +308,7 @@ private void reloadAllNotes() throws IOException { } List noteInfos = notebookRepo.list(); for (NoteInfo info : noteInfos) { - loadNoteFromRepo(info.getId()); + loadNoteFromRepo(info.getId(), info.getOwner()); } } From 8792a93e20455e4c0241a29e9a0e2f9eeb36e4a2 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 19 Oct 2015 16:58:49 +0200 Subject: [PATCH 55/71] Include users/anonymous directory in note path --- .../test/java/org/apache/zeppelin/notebook/NotebookTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index c975da90a32..cb12da56907 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -109,7 +109,8 @@ public void testSelectingReplImplementation() throws IOException { public void testGetAllNotes() throws IOException { // get all notes after copy the {notebookId}/note.json into notebookDir File srcDir = new File("src/test/resources/2A94M5J1Z"); - File destDir = new File(notebookDir.getAbsolutePath() + "/2A94M5J1Z"); + File destDir = new File(notebookDir.getAbsolutePath() + "/users/anonymous/2A94M5J1Z"); + destDir.getParentFile().mkdirs(); try { FileUtils.copyDirectory(srcDir, destDir); From af166053aa2e43969fc61e586dc6fb4e45eb4b73 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 20 Nov 2015 21:36:49 +0100 Subject: [PATCH 56/71] Added boolean property to prohibit anonymous access zeppelin.anonymous.allowed --- conf/zeppelin-site.xml.template | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 90989478f97..c4218787b68 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -160,5 +160,11 @@ Allowed sources for REST and WebSocket requests (i.e. http://onehost:8080,http://otherhost.com). If you leave * you are vulnerable to https://issues.apache.org/jira/browse/ZEPPELIN-173 + + zeppelin.anonymous.allowed + true + Anonymous user allowed by default + + From 4fc08b5355e7493159a12223ff0fb76ad5017032 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 20 Nov 2015 21:38:48 +0100 Subject: [PATCH 57/71] Handle property zeppelin.anonymous.allowed. When set to set anonymous access is not allowed. --- .../apache/zeppelin/rest/SecurityRestApi.java | 31 +++++++++++++------ .../zeppelin/socket/NotebookServer.java | 7 +++++ .../components/navbar/navbar.controller.js | 14 ++++++--- .../zeppelin/conf/ZeppelinConfiguration.java | 3 +- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java index 65f0b7e92da..9352fa46610 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.rest; +import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.server.JsonResponse; import org.apache.zeppelin.ticket.TicketContainer; import org.apache.shiro.SecurityUtils; @@ -53,6 +54,9 @@ public SecurityRestApi() { @GET @Path("ticket") public Response ticket() { + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + boolean allowAnonymous = conf. + getBoolean(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); Object oprincipal = SecurityUtils.getSubject().getPrincipal(); String principal; if (oprincipal == null) @@ -60,17 +64,24 @@ public Response ticket() { else principal = oprincipal.toString(); - // ticket set to anonymous for anonymous user. Simplify testing. - String ticket; - if ("anonymous".equals(principal)) - ticket = "anonymous"; - else - ticket = TicketContainer.instance.getTicket(principal); + JsonResponse response; + if (!allowAnonymous && principal.equals("anonymous")) { + response = new JsonResponse(Response.Status.FORBIDDEN); + } + else { + // ticket set to anonymous for anonymous user. Simplify testing. + String ticket; + if ("anonymous".equals(principal)) + ticket = "anonymous"; + else + ticket = TicketContainer.instance.getTicket(principal); - Map data = new HashMap<>(); - data.put("principal", principal); - data.put("ticket", ticket); + Map data = new HashMap<>(); + data.put("principal", principal); + data.put("ticket", ticket); - return new JsonResponse(Response.Status.OK, "", data).build(); + response = new JsonResponse(Response.Status.OK, "", data); + } + return response.build(); } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 6dc51a50c2b..a57b9e734e7 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -108,6 +108,13 @@ public void onMessage(NotebookSocket conn, String msg) { if (ticket != null && !ticket.equals(messagereceived.ticket)) throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + boolean allowAnonymous = conf. + getBoolean(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); + if (!allowAnonymous && messagereceived.principal.equals("anonymous")) { + throw new Exception("Anonymous access not allowed "); + } + addConnectionToUserSocketMap(conn, messagereceived); /** Lets be elegant here */ diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 4dc4cacc2ef..588c89abd02 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -49,11 +49,15 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco * */ $http.get('/api/security/ticket'). success(function(ticket, status, headers, config) { - $rootScope.ticket = angular.fromJson(ticket).body; - vm.loadNotes = loadNotes; - vm.isActive = isActive; - - vm.loadNotes(); + if (status === 401 || status === 403) { + // Dislay error message here + } + else { + $rootScope.ticket = angular.fromJson(ticket).body; + vm.loadNotes = loadNotes; + vm.isActive = isActive; + vm.loadNotes(); + } }). error(function(data, status, headers, config) { console.log('Could not get ticket'); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 19ddceb22de..c98fb454642 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -432,7 +432,8 @@ public static enum ConfVars { ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), // Allows a way to specify a ',' separated list of allowed origins for rest and websockets // i.e. http://localhost:8080 - ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"); + ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"), + ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true); private String varName; @SuppressWarnings("rawtypes") From 0c805ed6522717de3e97ff9d2d89538c6264874c Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Fri, 20 Nov 2015 22:37:39 +0100 Subject: [PATCH 58/71] merge with master (add auth) --- .../java/org/apache/zeppelin/rest/NotebookRestApi.java | 2 +- .../org/apache/zeppelin/rest/ZeppelinRestApiTest.java | 4 ++-- .../org/apache/zeppelin/socket/NotebookServerTest.java | 4 ++-- .../java/org/apache/zeppelin/notebook/NotebookTest.java | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 6ae3a754586..7a9e9f14d9c 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -134,7 +134,7 @@ private String getPrincipal() { @GET @Path("/") public Response getNotebookList() throws IOException { - List> notesInfo = notebookServer.generateNotebooksInfo(); + List> notesInfo = notebookServer.generateNotebooksInfo(getPrincipal()); return new JsonResponse(Status.OK, "", notesInfo ).build(); } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 85dd4b457b5..f78ed471f85 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -146,7 +146,7 @@ public void testInterpreterAutoBinding() throws IOException { get.releaseConnection(); //cleanup - ZeppelinServer.notebook.removeNote(note.getId()); + ZeppelinServer.notebook.removeNote(note.getId(), "anonymous"); } @Test @@ -182,7 +182,7 @@ public void testInterpreterRestart() throws IOException, InterruptedException { } assertEquals("

markdown restarted

\n", p.getResult().message()); //cleanup - ZeppelinServer.notebook.removeNote(note.getId()); + ZeppelinServer.notebook.removeNote(note.getId(), "anonymous"); } @Test diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java index 5275d81ac67..63c5464e674 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java @@ -86,7 +86,7 @@ public void checkInvalidOrigin(){ @Test public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException { // create a notebook - Note note1 = notebook.createNote(); + Note note1 = notebook.createNote("anonymous"); // get reference to interpreterGroup InterpreterGroup interpreterGroup = null; @@ -133,7 +133,7 @@ public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() thr verify(sock1, times(0)).send(anyString()); verify(sock2, times(1)).send(anyString()); - notebook.removeNote(note1.getId()); + notebook.removeNote(note1.getId(), "anonymous"); } private NotebookSocket createWebSocket() { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 9f29bd0e6f8..3c0bab0f078 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -210,7 +210,7 @@ public void testSchedule() throws InterruptedException, IOException{ note.setConfig(config); notebook.refreshCron(note.id(), "anonymous"); Thread.sleep(1*1000); - + // remove cron scheduler. config.put("cron", null); note.setConfig(config); @@ -286,8 +286,8 @@ public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedExc // restart interpreter factory.restart(note.getNoteReplLoader().getInterpreterSettings().get(0).id()); registry = note.getNoteReplLoader() - .getInterpreterSettings().get(0).getInterpreterGroup() - .getAngularObjectRegistry(); + .getInterpreterSettings().get(0).getInterpreterGroup() + .getAngularObjectRegistry(); // local and global scope object should be removed assertNull(registry.get("o1", note.id())); @@ -355,7 +355,7 @@ else if(file.isDirectory()){ file.delete(); } } - + @Override public JobListener getParagraphJobListener(Note note) { return new JobListener(){ From b2ed60391112b4ee00da9e88867396324a69d16e Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Sat, 28 Nov 2015 21:03:40 +0100 Subject: [PATCH 59/71] Update test to copy note in to users/[username] subdirectory --- .../org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 961a6265d5d..ce4a14c30fb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -190,7 +190,7 @@ public void testSyncOnList() throws IOException { assertEquals(0, notebookRepoSync.list(1).size()); File srcDir = new File("src/test/resources/2A94M5J1Z"); - File destDir = new File(secNotebookDir + "/2A94M5J1Z"); + File destDir = new File(secNotebookDir + "/users/anonymous/2A94M5J1Z"); /* copy manually new notebook into secondary storage repo and check repos */ try { From a45c578d1faef3872f04e19d192edb4c81891a88 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Wed, 9 Dec 2015 22:59:34 +0100 Subject: [PATCH 60/71] dd auth support --- .../java/org/apache/zeppelin/socket/NotebookServer.java | 4 ++-- .../org/apache/zeppelin/socket/NotebookServerTest.java | 8 ++++---- .../apache/zeppelin/notebook/repo/GitNotebookRepo.java | 4 ++-- .../zeppelin/notebook/repo/NotebookRepoVersioned.java | 3 ++- .../apache/zeppelin/notebook/repo/VFSNotebookRepo.java | 4 ++-- .../zeppelin/notebook/repo/VFSNotebookRepoTest.java | 2 +- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index fde6ea66833..576af6683ad 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -527,7 +527,7 @@ private void cloneNote(NotebookSocket conn, Notebook notebook, Message fromMessa protected Note importNote(NotebookSocket conn, Notebook notebook, Message fromMessage) throws IOException { - Note note = notebook.createNote(); + Note note = notebook.createNote(fromMessage.principal); if (fromMessage != null) { String noteName = (String) ((Map) fromMessage.get("notebook")).get("name"); if (noteName == null || noteName.isEmpty()) { @@ -575,7 +575,7 @@ protected Note importNote(NotebookSocket conn, Notebook notebook, Message fromMe note.persist(); broadcastNote(note); - broadcastNoteList(); + broadcastNoteList(fromMessage.principal); return note; } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java index 8ee2a330800..d7700599b97 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java @@ -151,10 +151,10 @@ public void testImportNotebook() throws IOException { //broadcastNoteList(); failed nothing to worry. } - assertNotEquals(null, notebook.getNote(note.getId())); - assertEquals("Test Zeppelin notebook import", notebook.getNote(note.getId()).getName()); - assertEquals("Test paragraphs import", notebook.getNote(note.getId()).getParagraphs().get(0).getText()); - notebook.removeNote(note.getId()); + assertNotEquals(null, notebook.getNote(note.getId(), "anonymous")); + assertEquals("Test Zeppelin notebook import", notebook.getNote(note.getId(), "anonymous").getName()); + assertEquals("Test paragraphs import", notebook.getNote(note.getId(), "anonymous").getParagraphs().get(0).getText()); + notebook.removeNote(note.getId(), "anonymous"); } private NotebookSocket createWebSocket() { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java index fe4975353eb..ab7f31e21a1 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java @@ -87,9 +87,9 @@ private void maybeAddAndCommit(String pattern) { } @Override - public Note get(String noteId, String rev) throws IOException { + public Note get(String noteId, String rev, String owner) throws IOException { //TODO(bzz): something like 'git checkout rev', that will not change-the-world though - return super.get(noteId); + return super.get(noteId, owner); } @Override diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoVersioned.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoVersioned.java index 4615afd900d..f75047c0ecf 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoVersioned.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoVersioned.java @@ -32,10 +32,11 @@ public interface NotebookRepoVersioned extends NotebookRepo { * * @param noteId Id of the Notebook * @param rev revision of the Notebook + * @param owner revision of the Notebook * @return a Notebook * @throws IOException */ - public Note get(String noteId, String rev) throws IOException; + public Note get(String noteId, String rev, String owner) throws IOException; /** * List of revisions of the given Notebook diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 0894079d06a..368f97f1b94 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -219,7 +219,7 @@ public Note get(String noteId, String owner) throws IOException { return getNote(noteDir); } - private FileObject getRootDir(String owner) throws IOException { + protected FileObject getRootDir(String owner) throws IOException { FileObject rootDir = fsManager.resolveFile(getPath(owner != null ? "/users/" + owner : "/")); if (!rootDir.exists()) { @@ -233,7 +233,7 @@ private FileObject getRootDir(String owner) throws IOException { return rootDir; } - private FileObject getRootDir() throws IOException { + protected FileObject getRootDir() throws IOException { return getRootDir(null); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index e747078ce6a..d5f972e4843 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -67,7 +67,7 @@ public void tearDown() throws Exception { @Test public void testSaveNotebook() throws IOException, InterruptedException { - Note note = notebook.createNote(); + Note note = notebook.createNote("anonymous"); note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); Paragraph p1 = note.addParagraph(); From c70d50f63f7720732a1e6faae0e15b97c1852802 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Wed, 9 Dec 2015 23:40:31 +0100 Subject: [PATCH 61/71] remove extra conf --- .../apache/zeppelin/rest/SecurityRestApi.java | 38 +++++++++---------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java index 9352fa46610..57ce7759ca9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.rest; +import org.apache.shiro.subject.Subject; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.server.JsonResponse; import org.apache.zeppelin.ticket.TicketContainer; @@ -55,33 +56,28 @@ public SecurityRestApi() { @Path("ticket") public Response ticket() { ZeppelinConfiguration conf = ZeppelinConfiguration.create(); - boolean allowAnonymous = conf. - getBoolean(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); - Object oprincipal = SecurityUtils.getSubject().getPrincipal(); + Subject subject = SecurityUtils.getSubject(); String principal; - if (oprincipal == null) + if (subject.isAuthenticated()) { + principal = subject.getPrincipal().toString(); + } + else { principal = "anonymous"; - else - principal = oprincipal.toString(); + } JsonResponse response; - if (!allowAnonymous && principal.equals("anonymous")) { - response = new JsonResponse(Response.Status.FORBIDDEN); - } - else { - // ticket set to anonymous for anonymous user. Simplify testing. - String ticket; - if ("anonymous".equals(principal)) - ticket = "anonymous"; - else - ticket = TicketContainer.instance.getTicket(principal); + // ticket set to anonymous for anonymous user. Simplify testing. + String ticket; + if ("anonymous".equals(principal)) + ticket = "anonymous"; + else + ticket = TicketContainer.instance.getTicket(principal); - Map data = new HashMap<>(); - data.put("principal", principal); - data.put("ticket", ticket); + Map data = new HashMap<>(); + data.put("principal", principal); + data.put("ticket", ticket); - response = new JsonResponse(Response.Status.OK, "", data); - } + response = new JsonResponse(Response.Status.OK, "", data); return response.build(); } } From d5944e081bacf1293854f943607b454f041123fb Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Wed, 9 Dec 2015 23:43:29 +0100 Subject: [PATCH 62/71] remove extra conf --- conf/zeppelin-site.xml.template | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 9ec0bd8d1ba..78d7f1ece4f 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -180,11 +180,5 @@ Allowed sources for REST and WebSocket requests (i.e. http://onehost:8080,http://otherhost.com). If you leave * you are vulnerable to https://issues.apache.org/jira/browse/ZEPPELIN-173 - - zeppelin.anonymous.allowed - true - Anonymous user allowed by default - - From d47d88727e086d063f634c60fea27926c8d22f5d Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 10 Dec 2015 01:20:02 +0100 Subject: [PATCH 63/71] activate basic auth by default --- zeppelin-server/src/main/resources/shiro.ini | 4 ++-- zeppelin-web/src/components/navbar/navbar.controller.js | 4 ++++ zeppelin-web/src/components/navbar/navbar.html | 4 ++-- .../src/components/websocketEvents/websocketMsg.service.js | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini index 408c491e8b7..0de9673f0a8 100644 --- a/zeppelin-server/src/main/resources/shiro.ini +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -30,5 +30,5 @@ securityManager.cacheManager = $cacheManager [urls] -/** = anon -#/** = authcBasic +#/** = anon +/** = authcBasic diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 8caf534d1bd..3883a94f132 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -31,6 +31,7 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco vm.connected = websocketMsgSrv.isConnected(); vm.websocketMsgSrv = websocketMsgSrv; vm.arrayOrderingSrv = arrayOrderingSrv; + vm.authenticated = $rootScope.ticket.principal !== 'anonymous'; angular.element('#notebook-list').perfectScrollbar({suppressScrollX: true}); @@ -42,6 +43,8 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco vm.connected = param; }); + + function loadNotes() { websocketMsgSrv.getNotebookList(); } @@ -59,6 +62,7 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco vm.loadNotes = loadNotes; vm.isActive = isActive; vm.loadNotes(); + vm.authenticated = $rootScope.ticket.principal !== 'anonymous'; } }). error(function(data, status, headers, config) { diff --git a/zeppelin-web/src/components/navbar/navbar.html b/zeppelin-web/src/components/navbar/navbar.html index 83846fd1057..e509d516364 100644 --- a/zeppelin-web/src/components/navbar/navbar.html +++ b/zeppelin-web/src/components/navbar/navbar.html @@ -46,8 +46,8 @@
  • - {{ticket.principal}} Connected - Disconnected + {{ticket.principal}} connected + Disconnected
diff --git a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js index a10bc875242..8762cdfd6d6 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js +++ b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js @@ -122,7 +122,7 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }); }, - isConnected: function(){ + isConnected: function() { return websocketEvents.isConnected(); } From 3efc8092f1dee156a68ed22d256e4b26020d24dc Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 14 Dec 2015 08:47:51 +0100 Subject: [PATCH 64/71] Anonymous access by default --- zeppelin-server/src/main/resources/shiro.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini index 0de9673f0a8..408c491e8b7 100644 --- a/zeppelin-server/src/main/resources/shiro.ini +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -30,5 +30,5 @@ securityManager.cacheManager = $cacheManager [urls] -#/** = anon -/** = authcBasic +/** = anon +#/** = authcBasic From 83deb1018d69e3d4fa95baf499b562ff03e4cfd4 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 28 Dec 2015 08:48:08 +0200 Subject: [PATCH 65/71] fix security params --- .../src/main/java/org/apache/zeppelin/notebook/Notebook.java | 4 ++-- .../java/org/apache/zeppelin/search/LuceneSearchTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index e41df015c36..b92589720d7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -96,7 +96,7 @@ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, if (this.notebookIndex != null) { long start = System.nanoTime(); logger.info("Notebook indexing started..."); - notebookIndex.addIndexDocs(notes.values()); + //notebookIndex.addIndexDocs(notes.values()); logger.info("Notebook indexing finished: {} indexed in {}s", notes.size(), TimeUnit.NANOSECONDS.toSeconds(start - System.nanoTime())); } @@ -137,7 +137,7 @@ private Map getUserNotes(String principal) { */ public Note createNote(List interpreterIds, String principal) throws IOException { NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory); - Note note = new Note(notebookRepo, intpLoader, jobListenerFactory, principal, notebookIndex); + Note note = new Note(notebookRepo, intpLoader, jobListenerFactory, notebookIndex, principal); intpLoader.setNoteId(note.id()); synchronized (notes) { getUserNotes(principal).put(note.id(), note); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java index f74d95eb375..6a0d7832aeb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -251,7 +251,7 @@ private Paragraph addParagraphWithText(Note note, String text) { } private Note newNote(String name) { - Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex); + Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex, "anonymous"); note.setName(name); return note; } From 1402ef1f20ae9dcf4d999d4df19df46f4c5ebfa9 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 28 Dec 2015 08:51:00 +0200 Subject: [PATCH 66/71] fix indentation --- .../main/java/org/apache/zeppelin/rest/NotebookRestApi.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index ebe5fca4783..d15d44b105d 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -137,7 +137,8 @@ public Response bind(@PathParam("noteId") String noteId) { @GET @Path("/") public Response getNotebookList() throws IOException { - List> notesInfo = notebookServer.generateNotebooksInfo(SecurityUtils.getPrincipal()); + List> notesInfo = notebookServer. + generateNotebooksInfo(SecurityUtils.getPrincipal()); return new JsonResponse(Status.OK, "", notesInfo ).build(); } From 47b8f0bee8ed25db8d721867576620bf2e3ae7a1 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Mon, 28 Dec 2015 09:06:42 +0200 Subject: [PATCH 67/71] More SecurityUtils --- .../main/java/org/apache/zeppelin/rest/NotebookRestApi.java | 4 ++-- .../main/java/org/apache/zeppelin/rest/SecurityRestApi.java | 4 +--- .../src/main/java/org/apache/zeppelin/server/CorsFilter.java | 4 +--- .../main/java/org/apache/zeppelin/socket/NotebookServer.java | 2 +- .../java/org/apache/zeppelin/security/SecurityUtilsTest.java | 2 +- .../main/java/org/apache/zeppelin/ticket}/SecurityUtils.java | 2 +- .../main/java/org/apache/zeppelin/ticket/TicketContainer.java | 0 7 files changed, 7 insertions(+), 11 deletions(-) rename {zeppelin-server/src/main/java/org/apache/zeppelin/utils => zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket}/SecurityUtils.java (98%) rename {zeppelin-server => zeppelin-zengine}/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java (100%) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index d15d44b105d..959353fe10e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -46,7 +46,7 @@ import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.server.JsonResponse; import org.apache.zeppelin.socket.NotebookServer; -import org.apache.zeppelin.utils.SecurityUtils; +import org.apache.zeppelin.ticket.SecurityUtils; import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -507,7 +507,7 @@ public Response registerCronJob(@PathParam("notebookId") String notebookId, Stri config.put("cron", request.getCronString()); note.setConfig(config); notebook.refreshCron(note.id(), - org.apache.zeppelin.utils.SecurityUtils.getPrincipal()); + SecurityUtils.getPrincipal()); return new JsonResponse<>(Status.OK).build(); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java index a8c8d9b3ede..a96e1daaa85 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java @@ -17,11 +17,9 @@ package org.apache.zeppelin.rest; -import org.apache.shiro.subject.Subject; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.server.JsonResponse; import org.apache.zeppelin.ticket.TicketContainer; -import org.apache.shiro.SecurityUtils; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -56,7 +54,7 @@ public SecurityRestApi() { @Path("ticket") public Response ticket() { ZeppelinConfiguration conf = ZeppelinConfiguration.create(); - String principal = org.apache.zeppelin.utils.SecurityUtils.getPrincipal(); + String principal = org.apache.zeppelin.ticket.SecurityUtils.getPrincipal(); JsonResponse response; // ticket set to anonymous for anonymous user. Simplify testing. String ticket; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/CorsFilter.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/CorsFilter.java index 0e39242acd8..c520094814c 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/CorsFilter.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/CorsFilter.java @@ -18,13 +18,11 @@ package org.apache.zeppelin.server; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.utils.SecurityUtils; +import org.apache.zeppelin.ticket.SecurityUtils; import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; import java.text.DateFormat; -import java.util.Arrays; import java.util.Date; import java.util.Locale; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index e42d4b72546..54aad80f9f9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -42,7 +42,7 @@ import org.apache.zeppelin.server.ZeppelinServer; import org.apache.zeppelin.socket.Message.OP; import org.apache.zeppelin.ticket.TicketContainer; -import org.apache.zeppelin.utils.SecurityUtils; +import org.apache.zeppelin.ticket.SecurityUtils; import org.eclipse.jetty.websocket.WebSocket; import org.eclipse.jetty.websocket.WebSocketServlet; import org.quartz.SchedulerException; diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/security/SecurityUtilsTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/security/SecurityUtilsTest.java index 0100bb7b08e..c45156e4e74 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/security/SecurityUtilsTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/security/SecurityUtilsTest.java @@ -19,7 +19,7 @@ import static org.junit.Assert.*; import org.apache.commons.configuration.ConfigurationException; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.utils.SecurityUtils; +import org.apache.zeppelin.ticket.SecurityUtils; import org.junit.Test; import java.net.URISyntaxException; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket/SecurityUtils.java similarity index 98% rename from zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket/SecurityUtils.java index b2a2a410941..dbce112b7f3 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket/SecurityUtils.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.zeppelin.utils; +package org.apache.zeppelin.ticket; import org.apache.shiro.subject.Subject; import org.apache.zeppelin.conf.ZeppelinConfiguration; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java similarity index 100% rename from zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java From bb309f39186177f5f04c30fef8c2eb20945648e3 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Tue, 29 Dec 2015 17:12:10 +0200 Subject: [PATCH 68/71] Added support for Lucene. On search only the notebooks of the connected user are returned. --- .../apache/zeppelin/rest/NotebookRestApi.java | 3 +- .../apache/zeppelin/ticket/SecurityUtils.java | 0 .../zeppelin/ticket/TicketContainer.java | 0 zeppelin-server/src/main/resources/shiro.ini | 6 +- .../apache/zeppelin/notebook/Notebook.java | 6 +- .../apache/zeppelin/search/LuceneSearch.java | 76 ++++++++++--------- .../apache/zeppelin/search/SearchService.java | 2 +- .../zeppelin/search/LuceneSearchTest.java | 14 ++-- 8 files changed, 61 insertions(+), 46 deletions(-) rename {zeppelin-zengine => zeppelin-server}/src/main/java/org/apache/zeppelin/ticket/SecurityUtils.java (100%) rename {zeppelin-zengine => zeppelin-server}/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java (100%) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 959353fe10e..fa8d82b9132 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -564,7 +564,8 @@ public Response getCronJob(@PathParam("notebookId") String notebookId) throws @Path("search") public Response search(@QueryParam("q") String queryTerm) { LOG.info("Searching notebooks for: {}", queryTerm); - List> notebooksFound = notebookIndex.query(queryTerm); + List> notebooksFound = notebookIndex.query(queryTerm, + SecurityUtils.getPrincipal()); LOG.info("{} notbooks found", notebooksFound.size()); return new JsonResponse<>(Status.OK, notebooksFound).build(); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket/SecurityUtils.java b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/SecurityUtils.java similarity index 100% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket/SecurityUtils.java rename to zeppelin-server/src/main/java/org/apache/zeppelin/ticket/SecurityUtils.java diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java similarity index 100% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java rename to zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini index 408c491e8b7..30137253ce6 100644 --- a/zeppelin-server/src/main/resources/shiro.ini +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -17,6 +17,8 @@ [users] admin = password +user1 = password +user2 = password [main] @@ -30,5 +32,5 @@ securityManager.cacheManager = $cacheManager [urls] -/** = anon -#/** = authcBasic +#/** = anon +/** = authcBasic diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index b92589720d7..f19776415dc 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -96,7 +96,11 @@ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, if (this.notebookIndex != null) { long start = System.nanoTime(); logger.info("Notebook indexing started..."); - //notebookIndex.addIndexDocs(notes.values()); + Collection notesToIndex = new ArrayList<>(); + for (Map userNotes : notes.values()) { + notesToIndex.addAll(userNotes.values()); + } + notebookIndex.addIndexDocs(notesToIndex); logger.info("Notebook indexing finished: {} indexed in {}s", notes.size(), TimeUnit.NANOSECONDS.toSeconds(start - System.nanoTime())); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java index 7f9cbbdd9db..d1963b65b39 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java @@ -72,6 +72,7 @@ public class LuceneSearch implements SearchService { private static final String SEARCH_FIELD = "contents"; static final String PARAGRAPH = "paragraph"; static final String ID_FIELD = "id"; + static final String OWNER_FIELD = "owner"; Directory ramDirectory; Analyzer analyzer; @@ -93,7 +94,7 @@ public LuceneSearch() { * @see org.apache.zeppelin.search.Search#query(java.lang.String) */ @Override - public List> query(String queryStr) { + public List> query(String queryStr, String owner) { if (null == ramDirectory) { throw new IllegalStateException( "Something went wrong on instance creation time, index dir is null"); @@ -110,7 +111,7 @@ public List> query(String queryStr) { SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter(); Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(query)); - result = doSearch(indexSearcher, query, analyzer, highlighter); + result = doSearch(indexSearcher, query, analyzer, highlighter, owner); indexReader.close(); } catch (IOException e) { LOG.error("Failed to open index dir {}, make sure indexing finished OK", ramDirectory, e); @@ -121,7 +122,7 @@ public List> query(String queryStr) { } private List> doSearch(IndexSearcher searcher, Query query, - Analyzer analyzer, Highlighter highlighter) { + Analyzer analyzer, Highlighter highlighter, String owner) { List> matchingParagraphs = Lists.newArrayList(); ScoreDoc[] hits; try { @@ -132,29 +133,33 @@ private List> doSearch(IndexSearcher searcher, Query query, int id = hits[i].doc; Document doc = searcher.doc(id); String path = doc.get(ID_FIELD); - if (path != null) { - LOG.debug((i + 1) + ". " + path); - String title = doc.get("title"); - if (title != null) { - LOG.debug(" Title: {}", doc.get("title")); - } + String docOwner = doc.get(OWNER_FIELD); + + if (docOwner != null && docOwner.equalsIgnoreCase(owner)) { + if (path != null) { + LOG.debug((i + 1) + ". " + path); + String title = doc.get("title"); + if (title != null) { + LOG.debug(" Title: {}", doc.get("title")); + } - String text = doc.get(SEARCH_FIELD); - TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id, - SEARCH_FIELD, analyzer); - TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3); - LOG.debug(" {} fragments found for query '{}'", frag.length, query); - for (int j = 0; j < frag.length; j++) { - if ((frag[j] != null) && (frag[j].getScore() > 0)) { - LOG.debug(" Fragment: {}", frag[j].toString()); + String text = doc.get(SEARCH_FIELD); + TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id, + SEARCH_FIELD, analyzer); + TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3); + LOG.debug(" {} fragments found for query '{}'", frag.length, query); + for (int j = 0; j < frag.length; j++) { + if ((frag[j] != null) && (frag[j].getScore() > 0)) { + LOG.debug(" Fragment: {}", frag[j].toString()); + } } - } - String fragment = (frag != null && frag.length > 0) ? frag[0].toString() : ""; + String fragment = (frag != null && frag.length > 0) ? frag[0].toString() : ""; - matchingParagraphs.add(ImmutableMap.of("id", path, // /paragraph/ - "name", title, "snippet", fragment, "text", text)); - } else { - LOG.info("{}. No {} for this document", i + 1, ID_FIELD); + matchingParagraphs.add(ImmutableMap.of("id", path, // /paragraph/ + "name", title, "snippet", fragment, "text", text)); + } else { + LOG.info("{}. No {} for this document", i + 1, ID_FIELD); + } } } } catch (IOException | InvalidTokenOffsetsException e) { @@ -182,7 +187,7 @@ private void updateIndexNoteName(Note note) throws IOException { LOG.debug("Skipping empty notebook name"); return; } - updateDoc(noteId, noteName, null); + updateDoc(noteId, noteName, null, note.getOwner()); } private void updateIndexParagraph(Note note, Paragraph p) throws IOException { @@ -190,7 +195,7 @@ private void updateIndexParagraph(Note note, Paragraph p) throws IOException { LOG.debug("Skipping empty paragraph"); return; } - updateDoc(note.getId(), note.getName(), p); + updateDoc(note.getId(), note.getName(), p, note.getOwner()); } /** @@ -202,9 +207,10 @@ private void updateIndexParagraph(Note note, Paragraph p) throws IOException { * @param p * @throws IOException */ - private void updateDoc(String noteId, String noteName, Paragraph p) throws IOException { + private void updateDoc(String noteId, String noteName, Paragraph p, String owner) + throws IOException { String id = formatId(noteId, p); - Document doc = newDocument(id, noteName, p); + Document doc = newDocument(id, noteName, p, owner); try { writer.updateDocument(new Term(ID_FIELD, id), doc); writer.commit(); @@ -244,12 +250,13 @@ static String formatDeleteId(String noteId, Paragraph p) { * @param p paragraph * @return */ - private Document newDocument(String id, String noteName, Paragraph p) { + private Document newDocument(String id, String noteName, Paragraph p, String owner) { Document doc = new Document(); Field pathField = new StringField(ID_FIELD, id, Field.Store.YES); doc.add(pathField); doc.add(new StringField("title", noteName, Field.Store.YES)); + doc.add(new StringField(OWNER_FIELD, owner, Field.Store.YES)); if (null != p) { doc.add(new TextField(SEARCH_FIELD, p.getText(), Field.Store.YES)); @@ -307,13 +314,13 @@ public void addIndexDoc(Note note) { * @throws IOException */ private void addIndexDocAsync(Note note) throws IOException { - indexNoteName(writer, note.getId(), note.getName()); + indexNoteName(writer, note.getId(), note.getName(), note.getOwner()); for (Paragraph doc : note.getParagraphs()) { if (doc.getText() == null) { LOG.debug("Skipping empty paragraph"); continue; } - indexDoc(writer, note.getId(), note.getName(), doc); + indexDoc(writer, note.getId(), note.getName(), doc, note.getOwner()); } } @@ -367,13 +374,14 @@ public void close() { * * @throws IOException */ - private void indexNoteName(IndexWriter w, String noteId, String noteName) throws IOException { + private void indexNoteName(IndexWriter w, String noteId, String noteName, String owner) + throws IOException { LOG.debug("Indexing Notebook {}, '{}'", noteId, noteName); if (null == noteName || noteName.isEmpty()) { LOG.debug("Skipping empty notebook name"); return; } - indexDoc(w, noteId, noteName, null); + indexDoc(w, noteId, noteName, null, owner); } /** @@ -381,10 +389,10 @@ private void indexNoteName(IndexWriter w, String noteId, String noteName) throws * - code of the paragraph (if non-null) * - or just a note name */ - private void indexDoc(IndexWriter w, String noteId, String noteName, Paragraph p) + private void indexDoc(IndexWriter w, String noteId, String noteName, Paragraph p, String owner) throws IOException { String id = formatId(noteId, p); - Document doc = newDocument(id, noteName, p); + Document doc = newDocument(id, noteName, p, owner); w.addDocument(doc); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/SearchService.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/SearchService.java index 64f2b758be2..ff357f5fb85 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/SearchService.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/SearchService.java @@ -39,7 +39,7 @@ public interface SearchService { * @param queryStr a query * @return A list of matching paragraphs (id, text, snippet w/ highlight) */ - public List> query(String queryStr); + public List> query(String queryStr, String owner); /** * Updates all documents in index for the given note: diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java index 6a0d7832aeb..2dd6ff0d46d 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -80,7 +80,7 @@ public void shutDown() { notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); //when - List> results = notebookIndex.query("all"); + List> results = notebookIndex.query("all", "anonymous"); //then assertThat(results).isNotEmpty(); @@ -96,7 +96,7 @@ public void shutDown() { notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); //when - List> results = notebookIndex.query("Notebook1"); + List> results = notebookIndex.query("Notebook1", "anonymous"); //then assertThat(results).isNotEmpty(); @@ -120,7 +120,7 @@ public void shutDown() { public void canNotSearchBeforeIndexing() { //given NO notebookIndex.index() was called //when - List> result = notebookIndex.query("anything"); + List> result = notebookIndex.query("anything", "anonymous"); //then assertThat(result).isEmpty(); //assert logs were printed @@ -139,10 +139,10 @@ public void canNotSearchBeforeIndexing() { notebookIndex.updateIndexDoc(note2); //then - List> results = notebookIndex.query("all"); + List> results = notebookIndex.query("all", "anonymous"); assertThat(results).isEmpty(); - results = notebookIndex.query("indeed"); + results = notebookIndex.query("indeed", "anonymous"); assertThat(results).isNotEmpty(); } @@ -164,7 +164,7 @@ public void canNotSearchBeforeIndexing() { notebookIndex.deleteIndexDocs(note2); //then - assertThat(notebookIndex.query("all")).isEmpty(); + assertThat(notebookIndex.query("all", "anonymous")).isEmpty(); assertThat(resultForQuery("Notebook2")).isEmpty(); List> results = resultForQuery("test"); @@ -215,7 +215,7 @@ public void canNotSearchBeforeIndexing() { } private List> resultForQuery(String q) { - return notebookIndex.query(q); + return notebookIndex.query(q, "anonymous"); } /** From eb007989e3c98add44eb801b027cc5b86e10ada8 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Wed, 30 Dec 2015 15:42:59 +0200 Subject: [PATCH 69/71] Security readme --- Security-README.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 Security-README.md diff --git a/Security-README.md b/Security-README.md new file mode 100644 index 00000000000..53b7c750d6f --- /dev/null +++ b/Security-README.md @@ -0,0 +1,59 @@ +# Vocabulary +username, owner and principal are used interchangeably to designate the currently authenticated user +# What are we securing ? +Zeppelin is basically a web application that spawn remote interpreters to run commands and return HTML fragments to be displayed on the user browser. +The scope of this PR is to secure the access to the notebooks. A user has access to his notebooks and only his notebooks. To achieve this, we use Apache Shiro. +## HTTP Endpoint security +Apache Shiro sits as a servlet filter between the browser and the exposed services and handles the required authentication without any programming required. (See Apache Shiro for more info). +## Websocket security +Securing the HTTP endpoints is not enough, since Zeppelin also communicates with the browser through websockets. To secure this channel, we take the following approach: +1. The browser on startup requests a ticket through HTTP +2. The Apache Shiro Servlet filter handles the user auth +3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser + +All websockets communications require the username and ticket to be submitted by the browser. Upon receiving a websocket message, the server checks that the ticket received is the one assigned to the username through the HTTP request (step 3 above). + +# Strategies to access the principal +Apache Shiro expose the principal through the following call +```org.apache.shiro.SecurityUtils.getSubject().getPrincipal()``` +Apache Shiro stores the ```principal``` and the ```subject``` in a thread local variable. + +That makes it possible to get the principal from wherever we need it in the application as long as we are in the context of a HTTP synchronous request. + +Two strategies are possible : (1) Rely on the Thread local (anti ?)pattern to get the principal whenever we need it or (2) modify the interfaces of the NotebookRepo and SearchService classes to require the owner to be passed as a parameter. + +## Relying on ThreadLocal +The Shiro ThreadLocal subject is available only for HTTP request which is not enough since we need to access notes through websockets (NotebookServer class) where the Shiro filter is not involved. + +We could however explicitly create the Shiro```subject``` on each websocket request using the provided username and ticket in the web socket request. + +The drawback of this approach is that it makes it impossible to write async services and/or multithreaded code involving access to the Shiro principal (The subject is available in the current thread only). + +On the other side, the main benefit of this approach is that it does not require any change to the NotebookRepo and the SearchService interfaces. + +## Updating the service interfaces +Coming from an actor based concurrent & distributed programming background I have decided to go for this approach. This required me to add the owner parameter to a couple of method. + trait NotebookRepo { + ... + public List list(String owner) throws IOException; + public Note get(String noteId, String owner) throws IOException; + ... + } + trait SearchService { + public List<~> query(String queryStr, String owner); + } + +As you can guess the has a significant impact on the existing code since the owner parameter had to be made available explicitly along the code that is executed to handle the request. + +# How Notes are stored +TODO +# Future : How Permissions could be implemented (note sharing, accessible features in iframes …) +TODO + + + + + +``` +` +```` \ No newline at end of file From 3b276057a22c5569ea6f6cf1c5851b4aa5277e2f Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Wed, 30 Dec 2015 15:58:05 +0200 Subject: [PATCH 70/71] markdown fix --- Security-README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Security-README.md b/Security-README.md index 53b7c750d6f..985c8c4ce35 100644 --- a/Security-README.md +++ b/Security-README.md @@ -14,9 +14,11 @@ Securing the HTTP endpoints is not enough, since Zeppelin also communicates with All websockets communications require the username and ticket to be submitted by the browser. Upon receiving a websocket message, the server checks that the ticket received is the one assigned to the username through the HTTP request (step 3 above). # Strategies to access the principal -Apache Shiro expose the principal through the following call -```org.apache.shiro.SecurityUtils.getSubject().getPrincipal()``` -Apache Shiro stores the ```principal``` and the ```subject``` in a thread local variable. +Apache Shiro expose the principal through the following call + + org.apache.shiro.SecurityUtils.getSubject().getPrincipal() + +Apache Shiro stores the `principal` and the `subject` in a thread local variable. That makes it possible to get the principal from wherever we need it in the application as long as we are in the context of a HTTP synchronous request. @@ -25,7 +27,7 @@ Two strategies are possible : (1) Rely on the Thread local (anti ?)pattern to ge ## Relying on ThreadLocal The Shiro ThreadLocal subject is available only for HTTP request which is not enough since we need to access notes through websockets (NotebookServer class) where the Shiro filter is not involved. -We could however explicitly create the Shiro```subject``` on each websocket request using the provided username and ticket in the web socket request. +We could however explicitly create the Shiro `subject` on each websocket request using the provided username and ticket in the web socket request. The drawback of this approach is that it makes it impossible to write async services and/or multithreaded code involving access to the Shiro principal (The subject is available in the current thread only). @@ -33,6 +35,7 @@ On the other side, the main benefit of this approach is that it does not require ## Updating the service interfaces Coming from an actor based concurrent & distributed programming background I have decided to go for this approach. This required me to add the owner parameter to a couple of method. + trait NotebookRepo { ... public List list(String owner) throws IOException; @@ -43,6 +46,7 @@ Coming from an actor based concurrent & distributed programming background I hav public List<~> query(String queryStr, String owner); } + As you can guess the has a significant impact on the existing code since the owner parameter had to be made available explicitly along the code that is executed to handle the request. # How Notes are stored @@ -54,6 +58,6 @@ TODO -``` -` -```` \ No newline at end of file +\`\`\` +\` +\`\`\`\` \ No newline at end of file From 81488ea8a90bf2b3d4793c5526366b0413c217a4 Mon Sep 17 00:00:00 2001 From: Hayssam Saleh Date: Thu, 31 Dec 2015 17:19:33 +0200 Subject: [PATCH 71/71] minor update to import --- .../main/java/org/apache/zeppelin/rest/SecurityRestApi.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java index a96e1daaa85..cd7168e5740 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java @@ -19,6 +19,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.server.JsonResponse; +import org.apache.zeppelin.ticket.SecurityUtils; import org.apache.zeppelin.ticket.TicketContainer; import javax.ws.rs.GET; @@ -54,7 +55,7 @@ public SecurityRestApi() { @Path("ticket") public Response ticket() { ZeppelinConfiguration conf = ZeppelinConfiguration.create(); - String principal = org.apache.zeppelin.ticket.SecurityUtils.getPrincipal(); + String principal = SecurityUtils.getPrincipal(); JsonResponse response; // ticket set to anonymous for anonymous user. Simplify testing. String ticket;