diff --git a/helm-charts/submarine/templates/rbac.yaml b/helm-charts/submarine/templates/rbac.yaml index c1081221d..36015a8b7 100644 --- a/helm-charts/submarine/templates/rbac.yaml +++ b/helm-charts/submarine/templates/rbac.yaml @@ -38,6 +38,19 @@ rules: - deletecollection - patch - update +- apiGroups: + - traefik.containo.us + resources: + - ingressroutes + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update --- kind: ClusterRoleBinding diff --git a/submarine-cloud/manifests/submarine-cluster/rbac.yaml b/submarine-cloud/manifests/submarine-cluster/rbac.yaml index 6687a81bf..62931d4dc 100644 --- a/submarine-cloud/manifests/submarine-cluster/rbac.yaml +++ b/submarine-cloud/manifests/submarine-cluster/rbac.yaml @@ -35,6 +35,10 @@ items: - pytorchjobs - notebooks verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["traefik.containo.us"] + resources: + - ingressroutes + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java index cdc33f05e..555e8bb43 100644 --- a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java @@ -23,6 +23,10 @@ import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; @@ -33,6 +37,7 @@ import io.kubernetes.client.apis.CoreV1Api; import io.kubernetes.client.apis.CustomObjectsApi; import io.kubernetes.client.models.V1DeleteOptionsBuilder; +import io.kubernetes.client.models.V1ObjectMeta; import io.kubernetes.client.models.V1Pod; import io.kubernetes.client.models.V1PodList; import io.kubernetes.client.models.V1Status; @@ -48,7 +53,10 @@ import org.apache.submarine.server.api.spec.ExperimentMeta; import org.apache.submarine.server.api.spec.ExperimentSpec; import org.apache.submarine.server.api.spec.NotebookSpec; +import org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRoute; +import org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRouteSpec; import org.apache.submarine.server.submitter.k8s.model.NotebookCR; +import org.apache.submarine.server.submitter.k8s.model.ingressroute.SpecRoute; import org.apache.submarine.server.submitter.k8s.parser.NotebookSpecParser; import org.apache.submarine.server.submitter.k8s.util.MLJobConverter; import org.apache.submarine.server.submitter.k8s.model.MLJob; @@ -239,10 +247,15 @@ public ExperimentLog getExperimentLog(ExperimentSpec spec, String id) { public Notebook createNotebook(NotebookSpec spec) throws SubmarineRuntimeException { Notebook notebook; try { + // create notebook custom resource NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec); Object object = api.createNamespacedCustomObject(notebookCR.getGroup(), notebookCR.getVersion(), notebookCR.getMetadata().getNamespace(), notebookCR.getPlural(), notebookCR, "true"); notebook = parseResponseObject(object); + + // create Traefik custom resource + createIngressRoute(notebookCR.getMetadata().getNamespace(), notebookCR.getMetadata().getName()); + } catch (JsonSyntaxException e) { LOG.error("K8s submitter: parse response object failed by " + e.getMessage(), e); throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream response failed."); @@ -279,6 +292,7 @@ public Notebook deleteNotebook(NotebookSpec spec) throws SubmarineRuntimeExcepti new V1DeleteOptionsBuilder().withApiVersion(notebookCR.getApiVersion()).build(), null, null, null); notebook = parseResponseObject(object); + deleteIngressRoute(notebookCR.getMetadata().getNamespace(), notebookCR.getMetadata().getName()); } catch (ApiException e) { throw new SubmarineRuntimeException(e.getCode(), e.getMessage()); } @@ -297,7 +311,7 @@ private Notebook parseResponseObject(Object obj) throws SubmarineRuntimeExceptio notebook.setName(notebookCR.getMetadata().getName()); // notebook url notebook.setUrl("/notebook/" + notebookCR.getMetadata().getNamespace() + "/" + - notebookCR.getMetadata().getName()); + notebookCR.getMetadata().getName() + "/"); DateTime createdTime = notebookCR.getMetadata().getCreationTimestamp(); if (createdTime != null) { notebook.setCreatedTime(createdTime.toString()); @@ -329,6 +343,61 @@ private String getJobLabelSelector(ExperimentSpec experimentSpec) { } } + private void createIngressRoute(String namespace, String name) { + try { + IngressRoute ingressRoute = new IngressRoute(); + V1ObjectMeta meta = new V1ObjectMeta(); + meta.setName(name); + meta.setNamespace(namespace); + ingressRoute.setMetadata(meta); + ingressRoute.setSpec(parseIngressRouteSpec(meta.getNamespace(), meta.getName())); + api.createNamespacedCustomObject( + ingressRoute.getGroup(), ingressRoute.getVersion(), + ingressRoute.getMetadata().getNamespace(), + ingressRoute.getPlural(), ingressRoute, "true"); + } catch (ApiException e) { + LOG.error("K8s submitter: Create Traefik custom resource object failed by " + e.getMessage(), e); + throw new SubmarineRuntimeException(e.getCode(), e.getMessage()); + } catch (JsonSyntaxException e) { + LOG.error("K8s submitter: parse response object failed by " + e.getMessage(), e); + throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream response failed."); + } + } + + private void deleteIngressRoute(String namespace, String name) { + try { + api.deleteNamespacedCustomObject( + IngressRoute.CRD_INGRESSROUTE_GROUP_V1, IngressRoute.CRD_INGRESSROUTE_VERSION_V1, + namespace, IngressRoute.CRD_INGRESSROUTE_PLURAL_V1, name, + new V1DeleteOptionsBuilder().withApiVersion(IngressRoute.CRD_APIVERSION_V1).build(), + null, null, null); + } catch (ApiException e) { + LOG.error("K8s submitter: Delete Traefik custom resource object failed by " + e.getMessage(), e); + throw new SubmarineRuntimeException(e.getCode(), e.getMessage()); + } + } + + private IngressRouteSpec parseIngressRouteSpec(String namespace, String name) { + IngressRouteSpec spec = new IngressRouteSpec(); + Set entryPoints = new HashSet<>(); + entryPoints.add("web"); + spec.setEntryPoints(entryPoints); + + SpecRoute route = new SpecRoute(); + route.setKind("Rule"); + route.setMatch("PathPrefix(`/notebook/" + namespace + "/" + name + "/`)"); + Set> serviceMap = new HashSet<>(); + Map service = new HashMap<>(); + service.put("name", name); + service.put("port", 80); + serviceMap.add(service); + route.setServices(serviceMap); + Set routes = new HashSet<>(); + routes.add(route); + spec.setRoutes(routes); + return spec; + } + private enum ParseOp { PARSE_OP_RESULT, PARSE_OP_DELETE diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRoute.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRoute.java new file mode 100644 index 000000000..72c5fc6da --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRoute.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s.model.ingressroute; + +import com.google.gson.annotations.SerializedName; +import io.kubernetes.client.models.V1ObjectMeta; + +public class IngressRoute { + public static final String CRD_INGRESSROUTE_GROUP_V1 = "traefik.containo.us"; + public static final String CRD_INGRESSROUTE_VERSION_V1 = "v1alpha1"; + public static final String CRD_APIVERSION_V1 = CRD_INGRESSROUTE_GROUP_V1 + + "/" + CRD_INGRESSROUTE_VERSION_V1; + public static final String CRD_INGRESSROUTE_KIND_V1 = "IngressRoute"; + public static final String CRD_INGRESSROUTE_PLURAL_V1 = "ingressroutes"; + + @SerializedName("apiVersion") + private String apiVersion; + + @SerializedName("kind") + private String kind; + + @SerializedName("metadata") + private V1ObjectMeta metadata; + + private transient String group; + + private transient String version; + + private transient String plural; + + @SerializedName("spec") + private IngressRouteSpec spec; + + public IngressRoute() { + setApiVersion(CRD_APIVERSION_V1); + setKind(CRD_INGRESSROUTE_KIND_V1); + setPlural(CRD_INGRESSROUTE_PLURAL_V1); + setGroup(CRD_INGRESSROUTE_GROUP_V1); + setVersion(CRD_INGRESSROUTE_VERSION_V1); + } + + public String getApiVersion() { + return apiVersion; + } + + public void setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + } + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public V1ObjectMeta getMetadata() { + return metadata; + } + + public void setMetadata(V1ObjectMeta metadata) { + this.metadata = metadata; + } + + public String getPlural() { + return plural; + } + + public void setPlural(String plural) { + this.plural = plural; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public IngressRouteSpec getSpec() { + return spec; + } + + public void setSpec(IngressRouteSpec spec) { + this.spec = spec; + } +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRouteSpec.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRouteSpec.java new file mode 100644 index 000000000..b29a91874 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/IngressRouteSpec.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s.model.ingressroute; + +import com.google.gson.annotations.SerializedName; + +import java.util.Set; + +public class IngressRouteSpec { + + public IngressRouteSpec() { + + } + + @SerializedName("entryPoints") + private Set entryPoints; + + @SerializedName("routes") + private Set routes; + + public Set getEntryPoints() { + return entryPoints; + } + + public void setEntryPoints(Set entryPoints) { + this.entryPoints = entryPoints; + } + + public Set getRoutes() { + return routes; + } + + public void setRoutes(Set routes) { + this.routes = routes; + } + +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/SpecRoute.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/SpecRoute.java new file mode 100644 index 000000000..d6208e78e --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ingressroute/SpecRoute.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s.model.ingressroute; + +import com.google.gson.annotations.SerializedName; + +import java.util.Map; +import java.util.Set; + +public class SpecRoute { + + public SpecRoute() { + + } + + @SerializedName("match") + private String match; + + @SerializedName("kind") + private String kind; + + @SerializedName("services") + private Set> services; + + public String getMatch() { + return match; + } + + public void setMatch(String match) { + this.match = match; + } + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public Set> getServices() { + return services; + } + + public void setServices(Set> services) { + this.services = services; + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/notebook/notebook.component.html b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/notebook/notebook.component.html index e1c4f7eba..795e286fe 100644 --- a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/notebook/notebook.component.html +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/notebook/notebook.component.html @@ -78,7 +78,7 @@

Notebook

- {{ data.name }} + {{ data.name }} {{ data.spec.environment.name }} {{ data.spec.environment.dockerImage }}