SUBMARINE-586. Add the get/delete api & tests.

helm-charts/submarine/templates/rbac.yaml

@@ -1,52 +1,54 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: "{{ .Values.submarine.server.name }}"
-rules:
-- apiGroups:
-  - kubeflow.org
-  resources:
-  - tfjobs
-  - tfjobs/status
-  - pytorchjobs
-  - pytorchjobs/status
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-  - deletecollection
-  - patch
-  - update
-
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: "{{ .Values.submarine.server.name }}"
-subjects:
-- kind: ServiceAccount
-  namespace: {{ .Release.Namespace }}
-  name: "{{ .Values.submarine.server.name }}"
-roleRef:
-  kind: ClusterRole
-  name: "{{ .Values.submarine.server.name }}"
-  apiGroup: ""
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: "{{ .Values.submarine.server.name }}"
+rules:
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - tfjobs
+  - tfjobs/status
+  - pytorchjobs
+  - pytorchjobs/status
+  - notebooks
+  - notebooks/status
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+  - deletecollection
+  - patch
+  - update
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: "{{ .Values.submarine.server.name }}"
+subjects:
+- kind: ServiceAccount
+  namespace: {{ .Release.Namespace }}
+  name: "{{ .Values.submarine.server.name }}"
+roleRef:
+  kind: ClusterRole
+  name: "{{ .Values.submarine.server.name }}"
+  apiGroup: ""

submarine-cloud/manifests/submarine-cluster/rbac.yaml

@@ -33,6 +33,7 @@ items:
         resources:
           - tfjobs
           - pytorchjobs
+          - notebooks
         verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
   - apiVersion: rbac.authorization.k8s.io/v1beta1
     kind: ClusterRoleBinding

submarine-server/server-api/src/main/java/org/apache/submarine/server/api/notebook/Notebook.java

@@ -117,4 +117,30 @@ public String toString() {
     }
   }
 
+  public void rebuild(Notebook notebook) {
+    if (notebook != null) {
+      if (notebook.getName() != null) {
+        this.setName(notebook.getName());
+      }
+      if (notebook.getUid() != null) {
+        this.setUid(notebook.getUid());
+      }
+      if (notebook.getUrl() != null) {
+        this.setUrl(notebook.getUrl());
+      }
+      if (notebook.getSpec() != null) {
+        this.setSpec(notebook.getSpec());
+      }
+      if (notebook.getStatus() != null) {
+        this.setStatus(notebook.getStatus());
+      }
+      if (notebook.getCreatedTime() != null) {
+        this.setCreatedTime(notebook.getCreatedTime());
+      }
+      if (notebook.getDeletedTime() != null) {
+        this.setDeletedTime(notebook.getDeletedTime());
+      }
+    }
+  }
+
 }

submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/NotebookManager.java

@@ -23,12 +23,16 @@
 import org.apache.submarine.server.SubmarineServer;
 import org.apache.submarine.server.SubmitterManager;
 import org.apache.submarine.server.api.Submitter;
+import org.apache.submarine.server.api.environment.Environment;
 import org.apache.submarine.server.api.notebook.Notebook;
 import org.apache.submarine.server.api.notebook.NotebookId;
 import org.apache.submarine.server.api.spec.NotebookSpec;
+import org.apache.submarine.server.environment.EnvironmentManager;
 
 import javax.ws.rs.core.Response;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -78,19 +82,34 @@ public Notebook createNotebook(NotebookSpec spec) throws SubmarineRuntimeExcepti
     Notebook notebook = submitter.createNotebook(spec);
     notebook.setNotebookId(generateNotebookId());
     notebook.setSpec(spec);
+    NotebookSpec notebookSpec = notebook.getSpec();
+    EnvironmentManager environmentManager = EnvironmentManager.getInstance();
+    Environment environment = environmentManager.getEnvironment(spec.getEnvironment().getName());
+    if (environment.getEnvironmentSpec() != null) {
+      notebookSpec.setEnvironment(environment.getEnvironmentSpec());
+    }
     cachedNotebookMap.putIfAbsent(notebook.getNotebookId().toString(), notebook);
     return notebook;
   }
 
   /**
    * List notebook instances
-   * @param status status, if null will return all notebooks
+   * @param namespace namespace, if null will return all notebooks
    * @return list
    * @throws SubmarineRuntimeException the service error
    */
-  public List<Notebook> listNotebooksByStatus(String status) throws SubmarineRuntimeException {
-    //TODO(ryan): implement the method
-    return null;
+  public List<Notebook> listNotebooksByNamespace(String namespace) throws SubmarineRuntimeException {
+    List<Notebook> notebookList = new ArrayList<>();
+    for (Map.Entry<String, Notebook> entry : cachedNotebookMap.entrySet()) {
+      Notebook notebook = entry.getValue();
+      Notebook patchNotebook = submitter.findNotebook(notebook.getSpec());
+      if (namespace == null || namespace.length() == 0
+              || namespace.toLowerCase().equals(patchNotebook.getSpec().getMeta().getNamespace())) {
+        notebook.rebuild(patchNotebook);
+        notebookList.add(notebook);
+      }
+    }
+    return notebookList;
   }
 
   /**
@@ -100,8 +119,12 @@ public List<Notebook> listNotebooksByStatus(String status) throws SubmarineRunti
    * @throws SubmarineRuntimeException the service error
    */
   public Notebook getNotebook(String id) throws SubmarineRuntimeException {
-    //TODO(ryan): implement the method
-    return null;
+    checkNotebookId(id);
+    Notebook notebook = cachedNotebookMap.get(id);
+    NotebookSpec spec = notebook.getSpec();
+    Notebook patchNotebook = submitter.findNotebook(spec);
+    notebook.rebuild(patchNotebook);
+    return notebook;
   }
 
   /**
@@ -111,8 +134,12 @@ public Notebook getNotebook(String id) throws SubmarineRuntimeException {
    * @throws SubmarineRuntimeException the service error
    */
   public Notebook deleteNotebook(String id) throws SubmarineRuntimeException {
-    //TODO(ryan): implement the method
-    return null;
+    checkNotebookId(id);
+    Notebook notebook = cachedNotebookMap.remove(id);
+    NotebookSpec spec = notebook.getSpec();
+    Notebook patchNotebook = submitter.deleteNotebook(spec);
+    notebook.rebuild(patchNotebook);
+    return notebook;
   }
 
   /**
@@ -136,4 +163,11 @@ private void checkNotebookSpec(NotebookSpec spec) {
     }
   }
 
+  private void checkNotebookId(String id) throws SubmarineRuntimeException {
+    NotebookId notebookId = NotebookId.fromString(id);
+    if (notebookId == null || !cachedNotebookMap.containsKey(id)) {
+      throw new SubmarineRuntimeException(Response.Status.NOT_FOUND.getStatusCode(),
+              "Not found notebook server.");
+    }
+  }
 }

submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/NotebookRestApi.java

@@ -43,7 +43,7 @@
 import java.util.List;
 
 /**
- * Notebook Service REST API v1
+ * Notebook REST API v1. It can accept {@link NotebookSpec} to create a notebook server.
  */
 @Path(RestConstants.V1 + "/" + RestConstants.NOTEBOOK)
 @Produces({MediaType.APPLICATION_JSON + "; " + RestConstants.CHARSET_UTF8})
@@ -94,8 +94,8 @@ public Response createNotebook(NotebookSpec spec) {
   }
 
   /**
-   * List all notebooks created by the user
-   * @param status status
+   * List all notebooks
+   * @param namespace namespace
    * @return notebook list
    */
   @GET
@@ -105,11 +105,11 @@ public Response createNotebook(NotebookSpec spec) {
           responses = {
                   @ApiResponse(description = "successful operation", content = @Content(
                           schema = @Schema(implementation = JsonResponse.class)))})
-  public Response listNotebooks(@QueryParam("status") String status) {
+  public Response listNotebooks(@QueryParam("namespace") String namespace) {
     try {
-      List<Notebook> notebookList = notebookManager.listNotebooksByStatus(status);
+      List<Notebook> notebookList = notebookManager.listNotebooksByNamespace(namespace);
       return new JsonResponse.Builder<List<Notebook>>(Response.Status.OK).success(true)
-              .result(notebookList).build();
+              .message("List all notebook instances").result(notebookList).build();
     } catch (SubmarineRuntimeException e) {
       return parseNotebookServiceException(e);
     }
@@ -134,7 +134,7 @@ public Response getNotebook(@PathParam(RestConstants.NOTEBOOK_ID) String id) {
     try {
       Notebook notebook = notebookManager.getNotebook(id);
       return new JsonResponse.Builder<Notebook>(Response.Status.OK).success(true)
-              .result(notebook).build();
+              .message("Get the notebook instance").result(notebook).build();
     } catch (SubmarineRuntimeException e) {
       return parseNotebookServiceException(e);
     }
@@ -159,7 +159,7 @@ public Response deleteNotebook(@PathParam(RestConstants.NOTEBOOK_ID) String id)
     try {
       Notebook notebook = notebookManager.deleteNotebook(id);
       return new JsonResponse.Builder<Notebook>(Response.Status.OK).success(true)
-              .result(notebook).build();
+              .message("Delete the notebook instance").result(notebook).build();
     } catch (SubmarineRuntimeException e) {
       return parseNotebookServiceException(e);
     }

submarine-server/server-core/src/test/resources/environment/test_env_3.json

@@ -0,0 +1,10 @@
+{
+  "name": "my-submarine-env",
+  "dockerImage" : "apache/submarine:jupyter-notebook-0.5.0-SNAPSHOT",
+  "kernelSpec" : {
+    "name" : "team_default_python_3.7",
+    "channels" : ["defaults"],
+    "dependencies" :
+      ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0"]
+  }
+}

submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java

@@ -21,6 +21,8 @@
 
 import java.io.FileReader;
 import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
 
 import com.google.gson.Gson;
 import com.google.gson.JsonSyntaxException;
@@ -30,6 +32,7 @@
 import io.kubernetes.client.JSON;
 import io.kubernetes.client.apis.CoreV1Api;
 import io.kubernetes.client.apis.CustomObjectsApi;
+import io.kubernetes.client.models.V1DeleteOptionsBuilder;
 import io.kubernetes.client.models.V1Pod;
 import io.kubernetes.client.models.V1PodList;
 import io.kubernetes.client.models.V1Status;
@@ -252,14 +255,34 @@ public Notebook createNotebook(NotebookSpec spec) throws SubmarineRuntimeExcepti
 
   @Override
   public Notebook findNotebook(NotebookSpec spec) throws SubmarineRuntimeException {
-    // TODO(ryan): Implement this method
-    return null;
+    Notebook notebook;
+    try {
+      NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
+      Object object = api.getNamespacedCustomObject(notebookCR.getGroup(), notebookCR.getVersion(),
+              notebookCR.getMetadata().getNamespace(),
+              notebookCR.getPlural(), notebookCR.getMetadata().getName());
+      notebook = parseResponseObject(object);
+    } catch (ApiException e) {
+      throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
+    }
+    return notebook;
   }
 
   @Override
   public Notebook deleteNotebook(NotebookSpec spec) throws SubmarineRuntimeException {
-    // TODO(ryan): Implement this method
-    return null;
+    Notebook notebook;
+    try {
+      NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
+      Object object = api.deleteNamespacedCustomObject(notebookCR.getGroup(), notebookCR.getVersion(),
+              notebookCR.getMetadata().getNamespace(), notebookCR.getPlural(),
+              notebookCR.getMetadata().getName(),
+              new V1DeleteOptionsBuilder().withApiVersion(notebookCR.getApiVersion()).build(),
+              null, null, null);
+      notebook = parseResponseObject(object);
+    } catch (ApiException e) {
+      throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
+    }
+    return notebook;
   }
 
   private Notebook parseResponseObject(Object obj) throws SubmarineRuntimeException {
@@ -280,6 +303,15 @@ private Notebook parseResponseObject(Object obj) throws SubmarineRuntimeExceptio
         notebook.setCreatedTime(createdTime.toString());
         notebook.setStatus(Notebook.Status.STATUS_CREATED.getValue());
       }
+
+      // deleted notebook
+      if (notebookCR.getMetadata().getName() == null) {
+        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
+        Date current = new Date();
+        notebook.setDeletedTime(dateFormat.format(current));
+        notebook.setStatus(Notebook.Status.STATUS_DELETED.toString());
+      }
+
     } catch (JsonSyntaxException e) {
       LOG.error("K8s submitter: parse response object failed by " + e.getMessage(), e);
       throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream response failed.");

submarine-server/server-submitter/submitter-k8s/src/test/resources/notebook_req.json

@@ -4,7 +4,7 @@
     "namespace": "default"
   },
   "environment": {
-    "image": "apache/submarine:tf2.1.0-jupyter"
+    "name": "my-submarine-env"
   },
   "spec": {
     "envVars": {