SUBMARINE-618. Create/Delete IngressRoute with Notebook CR

helm-charts/submarine/templates/rbac.yaml

@@ -38,6 +38,19 @@ rules:
   - deletecollection
   - patch
   - update
+- apiGroups:
+  - traefik.containo.us
+  resources:
+  - ingressroutes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+  - deletecollection
+  - patch
+  - update
 
 ---
 kind: ClusterRoleBinding

submarine-cloud/manifests/submarine-cluster/rbac.yaml

@@ -35,6 +35,10 @@ items:
           - pytorchjobs
           - notebooks
         verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+      - apiGroups: ["traefik.containo.us"]
+        resources:
+          - ingressroutes
+        verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
   - apiVersion: rbac.authorization.k8s.io/v1beta1
     kind: ClusterRoleBinding
     metadata:

submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java

@@ -23,6 +23,10 @@
 import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 import com.google.gson.Gson;
 import com.google.gson.JsonSyntaxException;
@@ -33,6 +37,7 @@
 import io.kubernetes.client.apis.CoreV1Api;
 import io.kubernetes.client.apis.CustomObjectsApi;
 import io.kubernetes.client.models.V1DeleteOptionsBuilder;
+import io.kubernetes.client.models.V1ObjectMeta;
 import io.kubernetes.client.models.V1Pod;
 import io.kubernetes.client.models.V1PodList;
 import io.kubernetes.client.models.V1Status;
@@ -48,7 +53,10 @@
 import org.apache.submarine.server.api.spec.ExperimentMeta;
 import org.apache.submarine.server.api.spec.ExperimentSpec;
 import org.apache.submarine.server.api.spec.NotebookSpec;
+import org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRoute;
+import org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRouteSpec;
 import org.apache.submarine.server.submitter.k8s.model.NotebookCR;
+import org.apache.submarine.server.submitter.k8s.model.ingressroute.SpecRoute;
 import org.apache.submarine.server.submitter.k8s.parser.NotebookSpecParser;
 import org.apache.submarine.server.submitter.k8s.util.MLJobConverter;
 import org.apache.submarine.server.submitter.k8s.model.MLJob;
@@ -239,10 +247,15 @@ public ExperimentLog getExperimentLog(ExperimentSpec spec, String id) {
   public Notebook createNotebook(NotebookSpec spec) throws SubmarineRuntimeException {
     Notebook notebook;
     try {
+      // create notebook custom resource
       NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
       Object object = api.createNamespacedCustomObject(notebookCR.getGroup(), notebookCR.getVersion(),
               notebookCR.getMetadata().getNamespace(), notebookCR.getPlural(), notebookCR, "true");
       notebook = parseResponseObject(object);
+
+      // create Traefik custom resource
+      createIngressRoute(notebookCR.getMetadata().getNamespace(), notebookCR.getMetadata().getName());
+
     } catch (JsonSyntaxException e) {
       LOG.error("K8s submitter: parse response object failed by " + e.getMessage(), e);
       throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream response failed.");
@@ -279,6 +292,7 @@ public Notebook deleteNotebook(NotebookSpec spec) throws SubmarineRuntimeExcepti
               new V1DeleteOptionsBuilder().withApiVersion(notebookCR.getApiVersion()).build(),
               null, null, null);
       notebook = parseResponseObject(object);
+      deleteIngressRoute(notebookCR.getMetadata().getNamespace(), notebookCR.getMetadata().getName());
     } catch (ApiException e) {
       throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
     }
@@ -297,7 +311,7 @@ private Notebook parseResponseObject(Object obj) throws SubmarineRuntimeExceptio
       notebook.setName(notebookCR.getMetadata().getName());
       // notebook url
       notebook.setUrl("/notebook/" + notebookCR.getMetadata().getNamespace() + "/" +
-              notebookCR.getMetadata().getName());
+              notebookCR.getMetadata().getName() + "/");
       DateTime createdTime = notebookCR.getMetadata().getCreationTimestamp();
       if (createdTime != null) {
         notebook.setCreatedTime(createdTime.toString());
@@ -329,6 +343,61 @@ private String getJobLabelSelector(ExperimentSpec experimentSpec) {
     }
   }
 
+  private void createIngressRoute(String namespace, String name) {
+    try {
+      IngressRoute ingressRoute = new IngressRoute();
+      V1ObjectMeta meta = new V1ObjectMeta();
+      meta.setName(name);
+      meta.setNamespace(namespace);
+      ingressRoute.setMetadata(meta);
+      ingressRoute.setSpec(parseIngressRouteSpec(meta.getNamespace(), meta.getName()));
+      api.createNamespacedCustomObject(
+              ingressRoute.getGroup(), ingressRoute.getVersion(),
+              ingressRoute.getMetadata().getNamespace(),
+              ingressRoute.getPlural(), ingressRoute, "true");
+    } catch (ApiException e) {
+      LOG.error("K8s submitter: Create Traefik custom resource object failed by " + e.getMessage(), e);
+      throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
+    } catch (JsonSyntaxException e) {
+      LOG.error("K8s submitter: parse response object failed by " + e.getMessage(), e);
+      throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream response failed.");
+    }
+  }
+
+  private void deleteIngressRoute(String namespace, String name) {
+    try {
+      api.deleteNamespacedCustomObject(
+              IngressRoute.CRD_INGRESSROUTE_GROUP_V1, IngressRoute.CRD_INGRESSROUTE_VERSION_V1,
+              namespace, IngressRoute.CRD_INGRESSROUTE_PLURAL_V1, name,
+              new V1DeleteOptionsBuilder().withApiVersion(IngressRoute.CRD_APIVERSION_V1).build(),
+              null, null, null);
+    } catch (ApiException e) {
+      LOG.error("K8s submitter: Delete Traefik custom resource object failed by " + e.getMessage(), e);
+      throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
+    }
+  }
+
+  private IngressRouteSpec parseIngressRouteSpec(String namespace, String name) {
+    IngressRouteSpec spec = new IngressRouteSpec();
+    Set<String> entryPoints = new HashSet<>();
+    entryPoints.add("web");
+    spec.setEntryPoints(entryPoints);
+
+    SpecRoute route = new SpecRoute();
+    route.setKind("Rule");
+    route.setMatch("PathPrefix(`/notebook/" + namespace + "/" + name + "/`)");
+    Set<Map<String, Object>> serviceMap = new HashSet<>();
+    Map<String, Object> service = new HashMap<>();
+    service.put("name", name);
+    service.put("port", 80);
+    serviceMap.add(service);
+    route.setServices(serviceMap);
+    Set<SpecRoute> routes = new HashSet<>();
+    routes.add(route);
+    spec.setRoutes(routes);
+    return spec;
+  }
+
   private enum ParseOp {
     PARSE_OP_RESULT,
     PARSE_OP_DELETE

submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRoute.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.submitter.k8s.model.ingressroute;
+
+import com.google.gson.annotations.SerializedName;
+import io.kubernetes.client.models.V1ObjectMeta;
+
+public class IngressRoute {
+  public static final String CRD_INGRESSROUTE_GROUP_V1 = "traefik.containo.us";
+  public static final String CRD_INGRESSROUTE_VERSION_V1 = "v1alpha1";
+  public static final String CRD_APIVERSION_V1 = CRD_INGRESSROUTE_GROUP_V1 +
+          "/" + CRD_INGRESSROUTE_VERSION_V1;
+  public static final String CRD_INGRESSROUTE_KIND_V1 = "IngressRoute";
+  public static final String CRD_INGRESSROUTE_PLURAL_V1 = "ingressroutes";
+
+  @SerializedName("apiVersion")
+  private String apiVersion;
+
+  @SerializedName("kind")
+  private String kind;
+
+  @SerializedName("metadata")
+  private V1ObjectMeta metadata;
+
+  private transient String group;
+
+  private transient String version;
+
+  private transient String plural;
+
+  @SerializedName("spec")
+  private IngressRouteSpec spec;
+
+  public IngressRoute() {
+    setApiVersion(CRD_APIVERSION_V1);
+    setKind(CRD_INGRESSROUTE_KIND_V1);
+    setPlural(CRD_INGRESSROUTE_PLURAL_V1);
+    setGroup(CRD_INGRESSROUTE_GROUP_V1);
+    setVersion(CRD_INGRESSROUTE_VERSION_V1);
+  }
+
+  public String getApiVersion() {
+    return apiVersion;
+  }
+
+  public void setApiVersion(String apiVersion) {
+    this.apiVersion = apiVersion;
+  }
+
+  public String getKind() {
+    return kind;
+  }
+
+  public void setKind(String kind) {
+    this.kind = kind;
+  }
+
+  public V1ObjectMeta getMetadata() {
+    return metadata;
+  }
+
+  public void setMetadata(V1ObjectMeta metadata) {
+    this.metadata = metadata;
+  }
+
+  public String getPlural() {
+    return plural;
+  }
+
+  public void setPlural(String plural) {
+    this.plural = plural;
+  }
+
+  public String getGroup() {
+    return group;
+  }
+
+  public void setGroup(String group) {
+    this.group = group;
+  }
+
+  public String getVersion() {
+    return version;
+  }
+
+  public void setVersion(String version) {
+    this.version = version;
+  }
+
+  public IngressRouteSpec getSpec() {
+    return spec;
+  }
+
+  public void setSpec(IngressRouteSpec spec) {
+    this.spec = spec;
+  }
+}

submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRouteSpec.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.submitter.k8s.model.ingressroute;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Set;
+
+public class IngressRouteSpec {
+
+  public IngressRouteSpec() {
+
+  }
+
+  @SerializedName("entryPoints")
+  private Set<String> entryPoints;
+
+  @SerializedName("routes")
+  private Set<SpecRoute> routes;
+
+  public Set<String> getEntryPoints() {
+    return entryPoints;
+  }
+
+  public void setEntryPoints(Set<String> entryPoints) {
+    this.entryPoints = entryPoints;
+  }
+
+  public Set<SpecRoute> getRoutes() {
+    return routes;
+  }
+
+  public void setRoutes(Set<SpecRoute> routes) {
+    this.routes = routes;
+  }
+
+}

submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/SpecRoute.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.submitter.k8s.model.ingressroute;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Map;
+import java.util.Set;
+
+public class SpecRoute {
+
+  public SpecRoute() {
+
+  }
+
+  @SerializedName("match")
+  private String match;
+
+  @SerializedName("kind")
+  private String kind;
+
+  @SerializedName("services")
+  private Set<Map<String, Object>> services;
+
+  public String getMatch() {
+    return match;
+  }
+
+  public void setMatch(String match) {
+    this.match = match;
+  }
+
+  public String getKind() {
+    return kind;
+  }
+
+  public void setKind(String kind) {
+    this.kind = kind;
+  }
+
+  public Set<Map<String, Object>> getServices() {
+    return services;
+  }
+
+  public void setServices(Set<Map<String, Object>> services) {
+    this.services = services;
+  }
+}