Skip to content
This repository has been archived by the owner on Jul 10, 2024. It is now read-only.

Commit

Permalink
SUBMARINE-618. Create/Delete IngressRoute with Notebook CR
Browse files Browse the repository at this point in the history
### What is this PR for?
Traefik uses CRD (IngressRoute) to retrieve its routing configuration.
In order to route the traffic to jupyter notebooks, we should add routing rules by creating a IngressRoute witch pair with notebook's service.

### What type of PR is it?
[Improvement]

### Todos
* [ ] - Task

### What is the Jira issue?
[SUBMARINE-618](https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-618)

### How should this be tested?
[Travis CI](https://travis-ci.org/github/lowc1012/submarine/builds/727777761)

### Screenshots (if appropriate)
![img1](https://user-images.githubusercontent.com/52355146/93370999-382bfd80-f884-11ea-88c1-216012475cad.png)
<img width="613" alt="img2" src="https://user-images.githubusercontent.com/52355146/93371062-50038180-f884-11ea-99ff-e90f426e8941.png">
<img width="612" alt="img3" src="https://user-images.githubusercontent.com/52355146/93371068-53970880-f884-11ea-927d-7aeb86e4591b.png">

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Ryan Lo <lowc1012@gmail.com>

Closes #404 from lowc1012/SUBMARINE-618 and squashes the following commits:

8081319 [Ryan Lo] SUBMARINE-618. Create/Delete IngressRoute with Notebook CR
  • Loading branch information
lowc1012 authored and jiwq committed Sep 18, 2020
1 parent 329fec8 commit 3d19524
Show file tree
Hide file tree
Showing 7 changed files with 321 additions and 2 deletions.
13 changes: 13 additions & 0 deletions helm-charts/submarine/templates/rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,19 @@ rules:
- deletecollection
- patch
- update
- apiGroups:
- traefik.containo.us
resources:
- ingressroutes
verbs:
- get
- list
- watch
- create
- delete
- deletecollection
- patch
- update

---
kind: ClusterRoleBinding
Expand Down
4 changes: 4 additions & 0 deletions submarine-cloud/manifests/submarine-cluster/rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ items:
- pytorchjobs
- notebooks
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: ["traefik.containo.us"]
resources:
- ingressroutes
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
Expand All @@ -33,6 +37,7 @@
import io.kubernetes.client.apis.CoreV1Api;
import io.kubernetes.client.apis.CustomObjectsApi;
import io.kubernetes.client.models.V1DeleteOptionsBuilder;
import io.kubernetes.client.models.V1ObjectMeta;
import io.kubernetes.client.models.V1Pod;
import io.kubernetes.client.models.V1PodList;
import io.kubernetes.client.models.V1Status;
Expand All @@ -48,7 +53,10 @@
import org.apache.submarine.server.api.spec.ExperimentMeta;
import org.apache.submarine.server.api.spec.ExperimentSpec;
import org.apache.submarine.server.api.spec.NotebookSpec;
import org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRoute;
import org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRouteSpec;
import org.apache.submarine.server.submitter.k8s.model.NotebookCR;
import org.apache.submarine.server.submitter.k8s.model.ingressroute.SpecRoute;
import org.apache.submarine.server.submitter.k8s.parser.NotebookSpecParser;
import org.apache.submarine.server.submitter.k8s.util.MLJobConverter;
import org.apache.submarine.server.submitter.k8s.model.MLJob;
Expand Down Expand Up @@ -239,10 +247,15 @@ public ExperimentLog getExperimentLog(ExperimentSpec spec, String id) {
public Notebook createNotebook(NotebookSpec spec) throws SubmarineRuntimeException {
Notebook notebook;
try {
// create notebook custom resource
NotebookCR notebookCR = NotebookSpecParser.parseNotebook(spec);
Object object = api.createNamespacedCustomObject(notebookCR.getGroup(), notebookCR.getVersion(),
notebookCR.getMetadata().getNamespace(), notebookCR.getPlural(), notebookCR, "true");
notebook = parseResponseObject(object);

// create Traefik custom resource
createIngressRoute(notebookCR.getMetadata().getNamespace(), notebookCR.getMetadata().getName());

} catch (JsonSyntaxException e) {
LOG.error("K8s submitter: parse response object failed by " + e.getMessage(), e);
throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream response failed.");
Expand Down Expand Up @@ -279,6 +292,7 @@ public Notebook deleteNotebook(NotebookSpec spec) throws SubmarineRuntimeExcepti
new V1DeleteOptionsBuilder().withApiVersion(notebookCR.getApiVersion()).build(),
null, null, null);
notebook = parseResponseObject(object);
deleteIngressRoute(notebookCR.getMetadata().getNamespace(), notebookCR.getMetadata().getName());
} catch (ApiException e) {
throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
}
Expand All @@ -297,7 +311,7 @@ private Notebook parseResponseObject(Object obj) throws SubmarineRuntimeExceptio
notebook.setName(notebookCR.getMetadata().getName());
// notebook url
notebook.setUrl("/notebook/" + notebookCR.getMetadata().getNamespace() + "/" +
notebookCR.getMetadata().getName());
notebookCR.getMetadata().getName() + "/");
DateTime createdTime = notebookCR.getMetadata().getCreationTimestamp();
if (createdTime != null) {
notebook.setCreatedTime(createdTime.toString());
Expand Down Expand Up @@ -329,6 +343,61 @@ private String getJobLabelSelector(ExperimentSpec experimentSpec) {
}
}

private void createIngressRoute(String namespace, String name) {
try {
IngressRoute ingressRoute = new IngressRoute();
V1ObjectMeta meta = new V1ObjectMeta();
meta.setName(name);
meta.setNamespace(namespace);
ingressRoute.setMetadata(meta);
ingressRoute.setSpec(parseIngressRouteSpec(meta.getNamespace(), meta.getName()));
api.createNamespacedCustomObject(
ingressRoute.getGroup(), ingressRoute.getVersion(),
ingressRoute.getMetadata().getNamespace(),
ingressRoute.getPlural(), ingressRoute, "true");
} catch (ApiException e) {
LOG.error("K8s submitter: Create Traefik custom resource object failed by " + e.getMessage(), e);
throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
} catch (JsonSyntaxException e) {
LOG.error("K8s submitter: parse response object failed by " + e.getMessage(), e);
throw new SubmarineRuntimeException(500, "K8s Submitter parse upstream response failed.");
}
}

private void deleteIngressRoute(String namespace, String name) {
try {
api.deleteNamespacedCustomObject(
IngressRoute.CRD_INGRESSROUTE_GROUP_V1, IngressRoute.CRD_INGRESSROUTE_VERSION_V1,
namespace, IngressRoute.CRD_INGRESSROUTE_PLURAL_V1, name,
new V1DeleteOptionsBuilder().withApiVersion(IngressRoute.CRD_APIVERSION_V1).build(),
null, null, null);
} catch (ApiException e) {
LOG.error("K8s submitter: Delete Traefik custom resource object failed by " + e.getMessage(), e);
throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
}
}

private IngressRouteSpec parseIngressRouteSpec(String namespace, String name) {
IngressRouteSpec spec = new IngressRouteSpec();
Set<String> entryPoints = new HashSet<>();
entryPoints.add("web");
spec.setEntryPoints(entryPoints);

SpecRoute route = new SpecRoute();
route.setKind("Rule");
route.setMatch("PathPrefix(`/notebook/" + namespace + "/" + name + "/`)");
Set<Map<String, Object>> serviceMap = new HashSet<>();
Map<String, Object> service = new HashMap<>();
service.put("name", name);
service.put("port", 80);
serviceMap.add(service);
route.setServices(serviceMap);
Set<SpecRoute> routes = new HashSet<>();
routes.add(route);
spec.setRoutes(routes);
return spec;
}

private enum ParseOp {
PARSE_OP_RESULT,
PARSE_OP_DELETE
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.submarine.server.submitter.k8s.model.ingressroute;

import com.google.gson.annotations.SerializedName;
import io.kubernetes.client.models.V1ObjectMeta;

public class IngressRoute {
public static final String CRD_INGRESSROUTE_GROUP_V1 = "traefik.containo.us";
public static final String CRD_INGRESSROUTE_VERSION_V1 = "v1alpha1";
public static final String CRD_APIVERSION_V1 = CRD_INGRESSROUTE_GROUP_V1 +
"/" + CRD_INGRESSROUTE_VERSION_V1;
public static final String CRD_INGRESSROUTE_KIND_V1 = "IngressRoute";
public static final String CRD_INGRESSROUTE_PLURAL_V1 = "ingressroutes";

@SerializedName("apiVersion")
private String apiVersion;

@SerializedName("kind")
private String kind;

@SerializedName("metadata")
private V1ObjectMeta metadata;

private transient String group;

private transient String version;

private transient String plural;

@SerializedName("spec")
private IngressRouteSpec spec;

public IngressRoute() {
setApiVersion(CRD_APIVERSION_V1);
setKind(CRD_INGRESSROUTE_KIND_V1);
setPlural(CRD_INGRESSROUTE_PLURAL_V1);
setGroup(CRD_INGRESSROUTE_GROUP_V1);
setVersion(CRD_INGRESSROUTE_VERSION_V1);
}

public String getApiVersion() {
return apiVersion;
}

public void setApiVersion(String apiVersion) {
this.apiVersion = apiVersion;
}

public String getKind() {
return kind;
}

public void setKind(String kind) {
this.kind = kind;
}

public V1ObjectMeta getMetadata() {
return metadata;
}

public void setMetadata(V1ObjectMeta metadata) {
this.metadata = metadata;
}

public String getPlural() {
return plural;
}

public void setPlural(String plural) {
this.plural = plural;
}

public String getGroup() {
return group;
}

public void setGroup(String group) {
this.group = group;
}

public String getVersion() {
return version;
}

public void setVersion(String version) {
this.version = version;
}

public IngressRouteSpec getSpec() {
return spec;
}

public void setSpec(IngressRouteSpec spec) {
this.spec = spec;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.submarine.server.submitter.k8s.model.ingressroute;

import com.google.gson.annotations.SerializedName;

import java.util.Set;

public class IngressRouteSpec {

public IngressRouteSpec() {

}

@SerializedName("entryPoints")
private Set<String> entryPoints;

@SerializedName("routes")
private Set<SpecRoute> routes;

public Set<String> getEntryPoints() {
return entryPoints;
}

public void setEntryPoints(Set<String> entryPoints) {
this.entryPoints = entryPoints;
}

public Set<SpecRoute> getRoutes() {
return routes;
}

public void setRoutes(Set<SpecRoute> routes) {
this.routes = routes;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.submarine.server.submitter.k8s.model.ingressroute;

import com.google.gson.annotations.SerializedName;

import java.util.Map;
import java.util.Set;

public class SpecRoute {

public SpecRoute() {

}

@SerializedName("match")
private String match;

@SerializedName("kind")
private String kind;

@SerializedName("services")
private Set<Map<String, Object>> services;

public String getMatch() {
return match;
}

public void setMatch(String match) {
this.match = match;
}

public String getKind() {
return kind;
}

public void setKind(String kind) {
this.kind = kind;
}

public Set<Map<String, Object>> getServices() {
return services;
}

public void setServices(Set<Map<String, Object>> services) {
this.services = services;
}
}
Loading

0 comments on commit 3d19524

Please sign in to comment.