From 50c2bd5a5906bb4e4dc978be1734454ae05dc66b Mon Sep 17 00:00:00 2001 From: Guy Daich Date: Mon, 15 Jan 2018 19:13:52 +0200 Subject: [PATCH 01/11] CHE-5908 Begin k8s infra poc Signed-off-by: Guy Daich --- .../kubernetes/files/che-kubernetes.yaml | 317 ++++++++++++++++++ .../openshift/OpenShiftInternalRuntime.java | 75 ++++- .../openshift/ServerResolver.java | 3 +- .../openshift/project/KubernetesIngress.java | 119 +++++++ .../openshift/project/OpenShiftProject.java | 25 +- .../OpenShiftInternalRuntimeTest.java | 9 +- .../project/OpenShiftProjectTest.java | 4 +- .../workspace/server/hc/ServersChecker.java | 19 +- 8 files changed, 549 insertions(+), 22 deletions(-) create mode 100644 dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml create mode 100644 infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java diff --git a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml new file mode 100644 index 00000000000..2429424a21b --- /dev/null +++ b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml @@ -0,0 +1,317 @@ +--- +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app: che + name: che +- apiVersion: v1 + kind: Service + metadata: + labels: + app: che + name: che-host + spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: che +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + labels: + app: che + name: che-data-volume + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + labels: + app: che + name: claim-che-workspace + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +- apiVersion: v1 + kind: ConfigMap + metadata: + labels: + app: che + name: che + data: + CHE_MULTIUSER: "false" + CHE_INFRASTRUCTURE_ACTIVE: openshift + CHE_INFRA_OPENSHIFT_MASTER__URL: "" + CHE_INFRA_OPENSHIFT_USERNAME: "" + CHE_INFRA_OPENSHIFT_PASSWORD: "" + CHE_INFRA_OPENSHIFT_OAUTH__TOKEN: "" + CHE_INFRA_OPENSHIFT_TRUST__CERTS: "false" + CHE_INFRA_OPENSHIFT_TLS__ENABLED: "false" + CHE_INFRA_OPENSHIFT_PVC_STRATEGY: "common" + CHE_INFRA_OPENSHIFT_PROJECT: "" + CHE_INFRA_OPENSHIFT_MACHINE__START__TIMEOUT__MIN: "5" + CHE_INFRA_OPENSHIFT_BOOTSTRAPPER_BINARY__URL: http://192.168.99.100/agent-binaries/linux_amd64/bootstrapper/bootstrapper + CHE_API: http://192.168.99.100/api + CHE_HOST: 192.168.99.100 + CHE_WEBSOCKET_ENDPOINT: ws://192.168.99.100/api/websocket + CHE_PORT: "8080" + CHE_LOCAL_CONF_DIR: /etc/conf + CHE_LOGS_DIR: /data/logs + CHE_PREDEFINED_STACKS_RELOAD__ON__START: "true" + CHE_LOG_LEVEL: INFO + CHE_PORT: "8080" + CHE_DEBUG_SERVER: "false" + CHE_OAUTH_GITHUB_FORCEACTIVATION: "true" + CHE_WORKSPACE_AUTO_START: "false" + CHE_KEYCLOAK_OSO_ENDPOINT: "" + CHE_KEYCLOAK_GITHUB_ENDPOINT: "" + CHE_SERVER_JAVA_OPTS: -XX:+UseG1GC -XX:+UseStringDeduplication -XX:MinHeapFreeRatio=20 -XX:MaxHeapFreeRatio=40 -XX:MaxRAM=700m -Xms256m + CHE_WORKSPACE_JAVA_OPTIONS: -XX:+UseG1GC -XX:+UseStringDeduplication -XX:MinHeapFreeRatio=20 -XX:MaxHeapFreeRatio=40 -XX:MaxRAM=1300m -Xms256m + CHE_KEYCLOAK_AUTH__SERVER__URL: not-in-use + CHE_KEYCLOAK_REALM: "" + CHE_KEYCLOAK_CLIENT__ID: "" + CHE_OAUTH_GITHUB_CLIENTID: "" + CHE_OAUTH_GITHUB_CLIENTSECRET: "" +- apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + labels: + app: che + name: che + spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: che + template: + metadata: + labels: + app: che + spec: + containers: + - env: + - name: CHE_OAUTH_GITHUB_CLIENTID + valueFrom: + configMapKeyRef: + key: CHE_OAUTH_GITHUB_CLIENTID + name: che + - name: CHE_OAUTH_GITHUB_CLIENTSECRET + valueFrom: + configMapKeyRef: + key: CHE_OAUTH_GITHUB_CLIENTSECRET + name: che + - name: CHE_MULTIUSER + valueFrom: + configMapKeyRef: + key: CHE_MULTIUSER + name: che + - name: CHE_KEYCLOAK_AUTH__SERVER__URL + valueFrom: + configMapKeyRef: + key: CHE_KEYCLOAK_AUTH__SERVER__URL + name: che + - name: CHE_KEYCLOAK_REALM + valueFrom: + configMapKeyRef: + key: CHE_KEYCLOAK_REALM + name: che + - name: CHE_KEYCLOAK_CLIENT__ID + valueFrom: + configMapKeyRef: + key: CHE_KEYCLOAK_CLIENT__ID + name: che + - name: CHE_INFRA_OPENSHIFT_MASTER__URL + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_MASTER__URL + name: che + - name: CHE_INFRA_OPENSHIFT_USERNAME + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_USERNAME + name: che + - name: CHE_INFRASTRUCTURE_ACTIVE + valueFrom: + configMapKeyRef: + key: CHE_INFRASTRUCTURE_ACTIVE + name: che + - name: CHE_INFRA_OPENSHIFT_PASSWORD + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_PASSWORD + name: che + - name: CHE_INFRA_OPENSHIFT_OAUTH__TOKEN + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_OAUTH__TOKEN + name: che + - name: CHE_INFRA_OPENSHIFT_TRUST__CERTS + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_TRUST__CERTS + name: che + - name: CHE_INFRA_OPENSHIFT_PVC_STRATEGY + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_PVC_STRATEGY + name: che + - name: CHE_INFRA_OPENSHIFT_TLS__ENABLED + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_TLS__ENABLED + name: che + - name: CHE_INFRA_OPENSHIFT_PROJECT + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_PROJECT + name: che + - name: CHE_INFRA_OPENSHIFT_MACHINE__START__TIMEOUT__MIN + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_MACHINE__START__TIMEOUT__MIN + name: che + - name: CHE_INFRA_OPENSHIFT_BOOTSTRAPPER_BINARY__URL + valueFrom: + configMapKeyRef: + key: CHE_INFRA_OPENSHIFT_BOOTSTRAPPER_BINARY__URL + name: che + - name: CHE_WEBSOCKET_ENDPOINT + valueFrom: + configMapKeyRef: + key: CHE_WEBSOCKET_ENDPOINT + name: che + - name: CHE_LOGS_DIR + valueFrom: + configMapKeyRef: + key: CHE_LOGS_DIR + name: che + - name: CHE_LOCAL_CONF_DIR + valueFrom: + configMapKeyRef: + key: CHE_LOCAL_CONF_DIR + name: che + - name: CHE_PREDEFINED_STACKS_RELOAD__ON__START + valueFrom: + configMapKeyRef: + key: CHE_PREDEFINED_STACKS_RELOAD__ON__START + name: che + - name: CHE_LOG_LEVEL + valueFrom: + configMapKeyRef: + key: CHE_LOG_LEVEL + name: che + - name: CHE_PORT + valueFrom: + configMapKeyRef: + key: CHE_PORT + name: che + - name: CHE_DEBUG_SERVER + valueFrom: + configMapKeyRef: + key: CHE_DEBUG_SERVER + name: che + - name: CHE_HOST + valueFrom: + configMapKeyRef: + key: CHE_HOST + name: che + - name: CHE_API + valueFrom: + configMapKeyRef: + key: CHE_API + name: che + - name: CHE_OAUTH_GITHUB_FORCEACTIVATION + valueFrom: + configMapKeyRef: + key: CHE_OAUTH_GITHUB_FORCEACTIVATION + name: che + - name: CHE_WORKSPACE_AUTO__START + valueFrom: + configMapKeyRef: + key: CHE_WORKSPACE_AUTO_START + name: che + - name: JAVA_OPTS + valueFrom: + configMapKeyRef: + key: CHE_SERVER_JAVA_OPTS + name: che + - name: CHE_WORKSPACE_JAVA_OPTIONS + valueFrom: + configMapKeyRef: + key: CHE_WORKSPACE_JAVA_OPTIONS + name: che + - name: CHE_KEYCLOAK_OSO_ENDPOINT + valueFrom: + configMapKeyRef: + key: CHE_KEYCLOAK_OSO_ENDPOINT + name: che + - name: CHE_KEYCLOAK_GITHUB_ENDPOINT + valueFrom: + configMapKeyRef: + key: CHE_KEYCLOAK_GITHUB_ENDPOINT + name: che + - name: CHE_KEYCLOAK_AUTH__SERVER__URL + valueFrom: + configMapKeyRef: + key: CHE_KEYCLOAK_AUTH__SERVER__URL + name: che + - name: CHE_KEYCLOAK_REALM + valueFrom: + configMapKeyRef: + key: CHE_KEYCLOAK_REALM + name: che + - name: CHE_KEYCLOAK_CLIENT__ID + valueFrom: + configMapKeyRef: + key: CHE_KEYCLOAK_CLIENT__ID + name: che + image: secure-registry.ingress.multi.devx-che.k8s.sapcloud.io/i31362/che6-k8s:latest + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /api/system/state + port: 8080 + scheme: HTTP + initialDelaySeconds: 120 + timeoutSeconds: 10 + name: che + ports: + - containerPort: 8080 + name: http + - containerPort: 8000 + name: http-debug + readinessProbe: + httpGet: + path: /api/system/state + port: 8080 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 60 + resources: + limits: + memory: 600Mi + requests: + memory: 256Mi + volumeMounts: + - mountPath: /data + name: che-data-volume + serviceAccountName: che + volumes: + - name: che-data-volume + persistentVolumeClaim: + claimName: che-data-volume \ No newline at end of file diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java index dc0b3424e58..715410e97a7 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java @@ -18,19 +18,26 @@ import com.google.inject.assistedinject.Assisted; import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.client.Watcher.Action; import io.fabric8.openshift.api.model.Route; +import io.fabric8.openshift.api.model.RouteBuilder; +import io.fabric8.openshift.api.model.RouteSpecBuilder; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Inject; import javax.inject.Named; import org.eclipse.che.api.core.model.workspace.Warning; +import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.core.model.workspace.runtime.Machine; import org.eclipse.che.api.core.model.workspace.runtime.MachineStatus; import org.eclipse.che.api.core.model.workspace.runtime.Server; @@ -53,6 +60,7 @@ import org.eclipse.che.api.workspace.shared.dto.event.RuntimeStatusEvent; import org.eclipse.che.api.workspace.shared.dto.event.ServerStatusEvent; import org.eclipse.che.dto.server.DtoFactory; +import org.eclipse.che.workspace.infrastructure.openshift.Annotations.Deserializer; import org.eclipse.che.workspace.infrastructure.openshift.bootstrapper.OpenShiftBootstrapperFactory; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; @@ -84,6 +92,7 @@ public class OpenShiftInternalRuntime extends InternalRuntime warnings) { + @Assisted List warnings, + @Named("che.host") String host) { super(context, urlRewriter, warnings, false); this.eventService = eventService; this.bootstrapperFactory = bootstrapperFactory; @@ -108,6 +118,7 @@ public OpenShiftInternalRuntime( this.probesFactory = probesFactory; this.project = project; this.machines = new ConcurrentHashMap<>(); + this.host = host; } @Override @@ -123,14 +134,26 @@ protected void internalStart(Map startOptions) throws Infrastruc createdServices.add(project.services().create(service)); } - List createdRoutes = new ArrayList<>(); - for (Route route : osEnv.getRoutes().values()) { - createdRoutes.add(project.routes().create(route)); - } + // List createdRoutes = new ArrayList<>(); + // for (Route route : osEnv.getRoutes().values()) { + // createdRoutes.add(project.routes().create(route)); + // } // TODO https://github.com/eclipse/che/issues/7653 // project.pods().watch(new AbnormalStopHandler()); // project.pods().watchContainers(new MachineLogsPublisher()); + // needed for resolution later on, even though nroutes are actually created by ingress + // /workspace{wsid}/server-{port} => service({wsid}):server-port => pod({wsid}):{port} + List createdRoutes = + osEnv + .getRoutes() + .values() + .stream() + .flatMap(r -> getRouteSteamByRouteServers(r)) + .collect(Collectors.toList()); + LOG.info("Creating Routes: " + createdRoutes); + project.kubernetesIngress().create(createdRoutes); + createPods(createdServices, createdRoutes); // TODO Rework it to parallel waiting https://github.com/eclipse/che/issues/7067 @@ -171,6 +194,48 @@ protected void internalStart(Map startOptions) throws Infrastruc } } + // breaks down original route with multiple protocols and endpoints to a single protocol + // routes, annotated with only the releveant server + private Stream getRouteSteamByRouteServers(Route r) { + + Deserializer deserializer = Annotations.newDeserializer(r.getMetadata().getAnnotations()); + final String machineName = deserializer.machineName(); + return deserializer + .servers() + .entrySet() + .stream() + .map( + entry -> { + ServerConfig server = entry.getValue(); + + String serverPath = "/" + getContext().getIdentity().getWorkspaceId(); + String targetServer = r.getSpec().getPort().getTargetPort().getStrVal(); + + if (targetServer != null) { + serverPath = serverPath + "/" + targetServer; + } + + if (server.getPath() != null) { + serverPath = serverPath + server.getPath(); + } + + Map severMap = new HashMap(); + severMap.put(entry.getKey(), server); + return new RouteBuilder(r) + .withSpec( + new RouteSpecBuilder(r.getSpec()).withHost(host).withPath(serverPath).build()) + .withMetadata( + new ObjectMetaBuilder() + .withAnnotations( + Annotations.newSerializer() + .servers(severMap) + .machineName(machineName) + .annotations()) + .build()) + .build(); + }); + } + @Override public Map getInternalMachines() { return ImmutableMap.copyOf(machines); diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java index 1f734827b6f..873e34399b6 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java @@ -96,7 +96,8 @@ private void fillRouteServers(Route route, Map servers) { config.getProtocol(), route.getSpec().getHost(), null, - config.getPath(), + // config.getPath(), + route.getSpec().getPath(), config.getAttributes()))); } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java new file mode 100644 index 00000000000..d58f6e397cb --- /dev/null +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.openshift.project; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressPath; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressPathBuilder; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressRuleValue; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressRuleValueBuilder; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.api.model.extensions.IngressBackend; +import io.fabric8.kubernetes.api.model.extensions.IngressBackendBuilder; +import io.fabric8.kubernetes.api.model.extensions.IngressBuilder; +import io.fabric8.kubernetes.api.model.extensions.IngressRule; +import io.fabric8.kubernetes.api.model.extensions.IngressRuleBuilder; +import io.fabric8.kubernetes.api.model.extensions.IngressSpec; +import io.fabric8.kubernetes.api.model.extensions.IngressSpecBuilder; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.openshift.api.model.Route; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; + +/** Created by I313632 on 05/01/2018. */ +public class KubernetesIngress { + private final String namespace; + private final String workspaceId; + private final OpenShiftClientFactory clientFactory; + + KubernetesIngress(String namespace, String workspaceId, OpenShiftClientFactory clientFactory) { + this.namespace = namespace; + this.workspaceId = workspaceId; + this.clientFactory = clientFactory; + } + + /** + * Creates specified route. + * + * @param routes - List of OS routes to create as an ingress + * @return created route + * @throws InfrastructureException when any exception occurs + */ + public Ingress create(List routes) throws InfrastructureException { + try { + List httpIngressPaths = new ArrayList<>(); + HashSet servers = new HashSet<>(); + for (Route route : routes) { + String server = route.getSpec().getPort().getTargetPort().getStrVal(); + if (!servers.contains(server)) { + IngressBackend ingressBackend = + new IngressBackendBuilder() + .withServiceName(route.getSpec().getTo().getName()) + .withNewServicePort(server) + .build(); + + String serverPath = "/" + workspaceId + "/" + server; + + HTTPIngressPath httpIngressPath = + new HTTPIngressPathBuilder() + // .withPath(route.getSpec().getPath()) + .withPath(serverPath) + .withBackend(ingressBackend) + .build(); + servers.add(server); + httpIngressPaths.add(httpIngressPath); + } + } + + HTTPIngressRuleValue httpIngressRuleValue = + new HTTPIngressRuleValueBuilder().withPaths(httpIngressPaths).build(); + IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build(); + IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build(); + Map ingressAnontations = new HashMap<>(); + ingressAnontations.put("ingress.kubernetes.io/rewrite-target", "/"); + ingressAnontations.put("ingress.kubernetes.io/ssl-redirect", "false"); + ingressAnontations.put("kubernetes.io/ingress.class", "nginx"); + Ingress ingress = + new IngressBuilder() + .withSpec(ingressSpec) + .withMetadata( + new ObjectMetaBuilder() + .withName(workspaceId + "-ingress") + .withAnnotations(ingressAnontations) + .build()) + .build(); + return clientFactory + .create() + .extensions() + .ingresses() + .inNamespace(namespace) + .withName(workspaceId + "-ingress") + .create(ingress); + } catch (KubernetesClientException e) { + throw new InfrastructureException(e.getMessage(), e); + } + } + + public void delete() throws InfrastructureException { + clientFactory + .create() + .extensions() + .ingresses() + .inNamespace(namespace) + .withName(workspaceId + "-ingress") + .delete(); + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java index 4adb1e8926c..097ba91b544 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java @@ -11,6 +11,7 @@ package org.eclipse.che.workspace.infrastructure.openshift.project; import com.google.common.annotations.VisibleForTesting; +import io.fabric8.kubernetes.api.model.Namespace; import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; @@ -39,6 +40,7 @@ public class OpenShiftProject { private final OpenShiftServices services; private final OpenShiftRoutes routes; private final OpenShiftPersistentVolumeClaims pvcs; + private final KubernetesIngress ingress; @VisibleForTesting OpenShiftProject( @@ -46,12 +48,14 @@ public class OpenShiftProject { OpenShiftPods pods, OpenShiftServices services, OpenShiftRoutes routes, - OpenShiftPersistentVolumeClaims pvcs) { + OpenShiftPersistentVolumeClaims pvcs, + KubernetesIngress kubernetesIngress) { this.workspaceId = workspaceId; this.pods = pods; this.services = services; this.routes = routes; this.pvcs = pvcs; + this.ingress = kubernetesIngress; } public OpenShiftProject(OpenShiftClientFactory clientFactory, String name, String workspaceId) @@ -61,6 +65,7 @@ public OpenShiftProject(OpenShiftClientFactory clientFactory, String name, Strin this.services = new OpenShiftServices(name, workspaceId, clientFactory); this.routes = new OpenShiftRoutes(name, workspaceId, clientFactory); this.pvcs = new OpenShiftPersistentVolumeClaims(name, clientFactory); + this.ingress = new KubernetesIngress(name, workspaceId, clientFactory); final OpenShiftClient client = clientFactory.create(); if (get(name, client) == null) { create(name, client); @@ -87,6 +92,12 @@ public OpenShiftPersistentVolumeClaims persistentVolumeClaims() { return pvcs; } + /** Returns object for managing {@link PersistentVolumeClaim} instances inside project. */ + public KubernetesIngress kubernetesIngress() { + return ingress; + } + + /** Removes all object except persistent volume claim inside project. */ /** Removes all object except persistent volume claims inside project. */ public void cleanUp() throws InfrastructureException { doRemove(pods::delete, services::delete, routes::delete); @@ -120,21 +131,15 @@ private void doRemove(RemoveOperation... operations) throws InfrastructureExcept private void create(String projectName, OpenShiftClient client) throws InfrastructureException { try { - client - .projectrequests() - .createNew() - .withNewMetadata() - .withName(projectName) - .endMetadata() - .done(); + client.namespaces().createNew().withNewMetadata().withName(projectName).endMetadata().done(); } catch (KubernetesClientException e) { throw new InfrastructureException(e.getMessage(), e); } } - private Project get(String projectName, OpenShiftClient client) throws InfrastructureException { + private Namespace get(String projectName, OpenShiftClient client) throws InfrastructureException { try { - return client.projects().withName(projectName).get(); + return client.namespaces().withName(projectName).get(); } catch (KubernetesClientException e) { if (e.getCode() == 403) { // project is foreign or doesn't exist diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java index 8db49f68b03..cd351bedc53 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java @@ -156,7 +156,8 @@ public void setup() throws Exception { workspaceProbesFactory, context, project, - emptyList()); + emptyList(), + ""); when(context.getEnvironment()).thenReturn(osEnv); when(serverCheckerFactory.create(any(), anyString(), any())).thenReturn(serversChecker); when(context.getIdentity()).thenReturn(IDENTITY); @@ -197,7 +198,7 @@ public void startsOpenShiftEnvironment() throws Exception { internalRuntime.internalStart(emptyMap()); verify(pods).create(any()); - verify(routes).create(any()); + // verify(routes).create(any()); verify(services).create(any()); verify(bootstrapper, times(2)).bootstrap(); verify(eventService, times(4)).publish(any()); @@ -240,7 +241,7 @@ public void stopsWaitingAllMachineStartWhenOneMachineStartFailed() throws Except internalRuntime.internalStart(emptyMap()); } catch (Exception rethrow) { verify(pods).create(any()); - verify(routes).create(any()); + // verify(routes).create(any()); verify(services).create(any()); verify(bootstrapper).bootstrap(); verify(eventService, times(4)).publish(any()); @@ -280,7 +281,7 @@ public void throwsInfrastructureExceptionWhenBootstrapInterrupted() throws Excep } catch (Exception rethrow) { verify(project).cleanUp(); verify(pods).create(any()); - verify(routes).create(any()); + // verify(routes).create(any()); verify(services).create(any()); verify(bootstrapper).bootstrap(); verifyEventsOrder(newEvent(M1_NAME, STARTING), newEvent(M1_NAME, RUNNING)); diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectTest.java index 886aac8e36d..444ab5542f9 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectTest.java @@ -52,6 +52,7 @@ public class OpenShiftProjectTest { @Mock private OpenShiftPersistentVolumeClaims pvcs; @Mock private OpenShiftClientFactory clientFactory; @Mock private OpenShiftClient openShiftClient; + @Mock private KubernetesIngress kubernetesIngress; private OpenShiftProject openShiftProject; @@ -59,7 +60,8 @@ public class OpenShiftProjectTest { public void setUp() throws Exception { when(clientFactory.create()).thenReturn(openShiftClient); - openShiftProject = new OpenShiftProject(WORKSPACE_ID, pods, services, routes, pvcs); + openShiftProject = + new OpenShiftProject(WORKSPACE_ID, pods, services, routes, pvcs, kubernetesIngress); } @Test diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/ServersChecker.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/ServersChecker.java index 70cba3ff255..1ef90e83d81 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/ServersChecker.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/ServersChecker.java @@ -179,12 +179,29 @@ private ServerChecker getChecker(String serverRef, Server server) throws Infrast try { // TODO: ws -> http is workaround used for terminal websocket server, // should be removed after server checks added to model + // url = + // UriBuilder.fromUri(server.getUrl().replaceFirst("^ws", "http")) + // .replacePath(livenessCheckPath) + // .queryParam("token", + // machineTokenProvider.getToken(runtimeIdentity.getWorkspaceId())) + // .build() + // .toURL(); + String relativeLivenessCheckPath = UriBuilder.fromUri(server.getUrl()).build().getPath(); + if (relativeLivenessCheckPath.lastIndexOf("/") > 0) { + relativeLivenessCheckPath = + relativeLivenessCheckPath.substring(0, relativeLivenessCheckPath.lastIndexOf("/")); + relativeLivenessCheckPath = relativeLivenessCheckPath + livenessCheckPath; + } else { + relativeLivenessCheckPath = livenessCheckPath; + } + url = UriBuilder.fromUri(server.getUrl().replaceFirst("^ws", "http")) - .replacePath(livenessCheckPath) + .replacePath(relativeLivenessCheckPath) .queryParam("token", machineTokenProvider.getToken(runtimeIdentity.getWorkspaceId())) .build() .toURL(); + System.out.println("checking liveness for: " + url); } catch (MalformedURLException e) { throw new InternalInfrastructureException( "Server " + serverRef + " URL is invalid. Error: " + e.getMessage(), e); From 7e571b6eff5403e16fbc4d5521ade3836e91f3ba Mon Sep 17 00:00:00 2001 From: Guy Daich Date: Sun, 21 Jan 2018 18:22:56 +0200 Subject: [PATCH 02/11] CHE-5908 Change Server Exposure, Resolving, Project & Infra to support ingress-based routes Signed-off-by: Guy Daich --- .../kubernetes/files/che-kubernetes.yaml | 2 +- .../openshift/OpenShiftInternalRuntime.java | 95 ++-- .../openshift/ServerExposer.java | 102 +++-- .../openshift/ServerResolver.java | 54 ++- .../environment/OpenShiftEnvironment.java | 24 +- .../openshift/project/KubernetesIngress.java | 194 +++++--- .../openshift/project/OpenShiftPods.java | 1 + .../provision/server/ServersConverter.java | 11 +- .../OpenShiftInternalRuntimeTest.java | 40 +- .../openshift/ServerExposerTest.java | 2 +- .../openshift/ServerResolverTest.java | 424 +++++++++--------- 11 files changed, 530 insertions(+), 419 deletions(-) diff --git a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml index 2429424a21b..c8bd976df3f 100644 --- a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml +++ b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml @@ -280,7 +280,7 @@ items: configMapKeyRef: key: CHE_KEYCLOAK_CLIENT__ID name: che - image: secure-registry.ingress.multi.devx-che.k8s.sapcloud.io/i31362/che6-k8s:latest + image: imagePullPolicy: Always livenessProbe: httpGet: diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java index 715410e97a7..9a4e22366f1 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java @@ -18,26 +18,19 @@ import com.google.inject.assistedinject.Assisted; import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.extensions.Ingress; import io.fabric8.kubernetes.client.Watcher.Action; -import io.fabric8.openshift.api.model.Route; -import io.fabric8.openshift.api.model.RouteBuilder; -import io.fabric8.openshift.api.model.RouteSpecBuilder; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.inject.Inject; import javax.inject.Named; import org.eclipse.che.api.core.model.workspace.Warning; -import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.core.model.workspace.runtime.Machine; import org.eclipse.che.api.core.model.workspace.runtime.MachineStatus; import org.eclipse.che.api.core.model.workspace.runtime.Server; @@ -60,7 +53,6 @@ import org.eclipse.che.api.workspace.shared.dto.event.RuntimeStatusEvent; import org.eclipse.che.api.workspace.shared.dto.event.ServerStatusEvent; import org.eclipse.che.dto.server.DtoFactory; -import org.eclipse.che.workspace.infrastructure.openshift.Annotations.Deserializer; import org.eclipse.che.workspace.infrastructure.openshift.bootstrapper.OpenShiftBootstrapperFactory; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; @@ -92,7 +84,6 @@ public class OpenShiftInternalRuntime extends InternalRuntime warnings, - @Named("che.host") String host) { + @Assisted List warnings) { super(context, urlRewriter, warnings, false); this.eventService = eventService; this.bootstrapperFactory = bootstrapperFactory; @@ -118,7 +108,6 @@ public OpenShiftInternalRuntime( this.probesFactory = probesFactory; this.project = project; this.machines = new ConcurrentHashMap<>(); - this.host = host; } @Override @@ -144,17 +133,26 @@ protected void internalStart(Map startOptions) throws Infrastruc // needed for resolution later on, even though nroutes are actually created by ingress // /workspace{wsid}/server-{port} => service({wsid}):server-port => pod({wsid}):{port} - List createdRoutes = - osEnv - .getRoutes() - .values() - .stream() - .flatMap(r -> getRouteSteamByRouteServers(r)) - .collect(Collectors.toList()); - LOG.info("Creating Routes: " + createdRoutes); - project.kubernetesIngress().create(createdRoutes); - - createPods(createdServices, createdRoutes); + List createdIngresses = new ArrayList<>(); + for (Ingress ingress : osEnv.getIngresses().values()) { + // create + createdIngresses.add(project.kubernetesIngress().create(ingress)); + } + + // wait for LB ip + List actualIngresses = new ArrayList<>(); + for (Ingress ingress : osEnv.getIngresses().values()) { + Ingress actualIngress = + project + .kubernetesIngress() + .wait( + ingress.getMetadata().getName(), + machineStartTimeoutMin, + p -> (p.getStatus().getLoadBalancer().getIngress().size() > 0)); + actualIngresses.add(actualIngress); + } + + createPods(createdServices, actualIngresses); // TODO Rework it to parallel waiting https://github.com/eclipse/che/issues/7067 for (OpenShiftMachine machine : machines.values()) { @@ -194,48 +192,6 @@ protected void internalStart(Map startOptions) throws Infrastruc } } - // breaks down original route with multiple protocols and endpoints to a single protocol - // routes, annotated with only the releveant server - private Stream getRouteSteamByRouteServers(Route r) { - - Deserializer deserializer = Annotations.newDeserializer(r.getMetadata().getAnnotations()); - final String machineName = deserializer.machineName(); - return deserializer - .servers() - .entrySet() - .stream() - .map( - entry -> { - ServerConfig server = entry.getValue(); - - String serverPath = "/" + getContext().getIdentity().getWorkspaceId(); - String targetServer = r.getSpec().getPort().getTargetPort().getStrVal(); - - if (targetServer != null) { - serverPath = serverPath + "/" + targetServer; - } - - if (server.getPath() != null) { - serverPath = serverPath + server.getPath(); - } - - Map severMap = new HashMap(); - severMap.put(entry.getKey(), server); - return new RouteBuilder(r) - .withSpec( - new RouteSpecBuilder(r.getSpec()).withHost(host).withPath(serverPath).build()) - .withMetadata( - new ObjectMetaBuilder() - .withAnnotations( - Annotations.newSerializer() - .servers(severMap) - .machineName(machineName) - .annotations()) - .build()) - .build(); - }); - } - @Override public Map getInternalMachines() { return ImmutableMap.copyOf(machines); @@ -295,12 +251,13 @@ private void checkMachineServers(OpenShiftMachine machine) * Creates OpenShift pods and resolves machine servers based on routes and services. * * @param services created OpenShift services - * @param routes created OpenShift routes + * @param ingresses created OpenShift ingresses * @throws InfrastructureException when any error occurs while creating OpenShift pods */ @VisibleForTesting - void createPods(List services, List routes) throws InfrastructureException { - final ServerResolver serverResolver = ServerResolver.of(services, routes); + // void createPods(List services, List routes) throws InfrastructureException { + void createPods(List services, List ingresses) throws InfrastructureException { + final ServerResolver serverResolver = ServerResolver.of(services, ingresses); final OpenShiftEnvironment environment = getContext().getEnvironment(); final Map machineConfigs = environment.getMachines(); for (Pod toCreate : environment.getPods().values()) { diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposer.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposer.java index 1d8054a6e09..5f9a39b787f 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposer.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposer.java @@ -20,10 +20,23 @@ import io.fabric8.kubernetes.api.model.ContainerPort; import io.fabric8.kubernetes.api.model.ContainerPortBuilder; import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.ServicePort; import io.fabric8.kubernetes.api.model.ServicePortBuilder; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressPath; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressPathBuilder; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressRuleValue; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressRuleValueBuilder; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.api.model.extensions.IngressBackend; +import io.fabric8.kubernetes.api.model.extensions.IngressBackendBuilder; +import io.fabric8.kubernetes.api.model.extensions.IngressBuilder; +import io.fabric8.kubernetes.api.model.extensions.IngressRule; +import io.fabric8.kubernetes.api.model.extensions.IngressRuleBuilder; +import io.fabric8.kubernetes.api.model.extensions.IngressSpec; +import io.fabric8.kubernetes.api.model.extensions.IngressSpecBuilder; import io.fabric8.openshift.api.model.Route; import java.util.ArrayList; import java.util.Collection; @@ -111,13 +124,19 @@ public class ServerExposer { private final Container container; private final Pod pod; private final OpenShiftEnvironment openShiftEnvironment; + private final String host; public ServerExposer( - String machineName, Pod pod, Container container, OpenShiftEnvironment openShiftEnvironment) { + String machineName, + Pod pod, + Container container, + OpenShiftEnvironment openShiftEnvironment, + String host) { this.machineName = machineName; this.pod = pod; this.container = container; this.openShiftEnvironment = openShiftEnvironment; + this.host = host; } /** @@ -158,6 +177,7 @@ public void expose(Map servers) { for (ServicePort servicePort : portToServicePort.values()) { int port = servicePort.getTargetPort().getIntVal(); + Map routesServers = externalServers .entrySet() @@ -165,15 +185,17 @@ public void expose(Map servers) { .filter(e -> parseInt(e.getValue().getPort().split("/")[0]) == port) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); - Route route = - new RouteBuilder() + Ingress ingress = + new IngressBuilder() .withName(serviceName + '-' + servicePort.getName()) .withMachineName(machineName) .withTargetPort(servicePort.getName()) .withServers(routesServers) .withTo(serviceName) + .withHost(host) .build(); - openShiftEnvironment.getRoutes().put(route.getMetadata().getName(), route); + + openShiftEnvironment.getIngresses().put(ingress.getMetadata().getName(), ingress); } } @@ -266,59 +288,79 @@ public ServiceBuilder withMachineName(String machineName) { } } - private static class RouteBuilder { + private static class IngressBuilder { private String name; private String serviceName; private IntOrString targetPort; private Map serversConfigs; private String machineName; + private String host; - private RouteBuilder withName(String name) { + private IngressBuilder withName(String name) { this.name = name; return this; } - private RouteBuilder withTo(String serviceName) { + private IngressBuilder withTo(String serviceName) { this.serviceName = serviceName; return this; } - private RouteBuilder withTargetPort(String targetPortName) { + private IngressBuilder withTargetPort(String targetPortName) { this.targetPort = new IntOrString(targetPortName); return this; } - private RouteBuilder withServers(Map serversConfigs) { + private IngressBuilder withServers(Map serversConfigs) { this.serversConfigs = serversConfigs; return this; } - public RouteBuilder withMachineName(String machineName) { + public IngressBuilder withMachineName(String machineName) { this.machineName = machineName; return this; } - private Route build() { - io.fabric8.openshift.api.model.RouteBuilder builder = - new io.fabric8.openshift.api.model.RouteBuilder(); + public IngressBuilder withHost(String host) { + this.host = host; + return this; + } - return builder - .withNewMetadata() - .withName(name.replace("/", "-")) - .withAnnotations( - Annotations.newSerializer() - .servers(serversConfigs) - .machineName(machineName) - .annotations()) - .endMetadata() - .withNewSpec() - .withNewTo() - .withName(serviceName) - .endTo() - .withNewPort() - .withTargetPort(targetPort) - .endPort() - .endSpec() + private Ingress build() { + + IngressBackend ingressBackend = + new IngressBackendBuilder() + .withServiceName(serviceName) + .withNewServicePort(targetPort.getStrVal()) + .build(); + + String serverPath = "/" + serviceName + "/" + targetPort.getStrVal(); + HTTPIngressPath httpIngressPath = + new HTTPIngressPathBuilder().withPath(serverPath).withBackend(ingressBackend).build(); + + HTTPIngressRuleValue httpIngressRuleValue = + new HTTPIngressRuleValueBuilder().withPaths(httpIngressPath).build(); + IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build(); + // new IngressRuleBuilder().withHttp(httpIngressRuleValue).withHost(host).build(); + IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build(); + + Map ingressAnnotations = new HashMap<>(); + ingressAnnotations.put("ingress.kubernetes.io/rewrite-target", "/"); + ingressAnnotations.put("ingress.kubernetes.io/ssl-redirect", "false"); + ingressAnnotations.put("kubernetes.io/ingress.class", "nginx"); + ingressAnnotations.putAll( + Annotations.newSerializer() + .servers(serversConfigs) + .machineName(machineName) + .annotations()); + + return new io.fabric8.kubernetes.api.model.extensions.IngressBuilder() + .withSpec(ingressSpec) + .withMetadata( + new ObjectMetaBuilder() + .withName(serviceName + targetPort.getStrVal() + "-ingress") + .withAnnotations(ingressAnnotations) + .build()) .build(); } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java index 873e34399b6..9e14f10515e 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java @@ -13,6 +13,8 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.api.model.extensions.IngressRule; import io.fabric8.openshift.api.model.Route; import java.util.HashMap; import java.util.List; @@ -35,9 +37,11 @@ */ public class ServerResolver { private final Multimap services; - private final Multimap routes; + // private final Multimap routes; + private final Multimap ingresses; - private ServerResolver(List services, List routes) { + // private ServerResolver(List services, List routes) { + private ServerResolver(List services, List ingresses) { this.services = ArrayListMultimap.create(); for (Service service : services) { String machineName = @@ -45,16 +49,24 @@ private ServerResolver(List services, List routes) { this.services.put(machineName, service); } - this.routes = ArrayListMultimap.create(); - for (Route route : routes) { + // this.routes = ArrayListMultimap.create(); + // for (Route route : routes) { + // String machineName = + // Annotations.newDeserializer(route.getMetadata().getAnnotations()).machineName(); + // this.routes.put(machineName, route); + // } + + this.ingresses = ArrayListMultimap.create(); + for (Ingress ingress : ingresses) { String machineName = - Annotations.newDeserializer(route.getMetadata().getAnnotations()).machineName(); - this.routes.put(machineName, route); + Annotations.newDeserializer(ingress.getMetadata().getAnnotations()).machineName(); + this.ingresses.put(machineName, ingress); } } - public static ServerResolver of(List services, List routes) { - return new ServerResolver(services, routes); + // public static ServerResolver of(List services, List routes) { + public static ServerResolver of(List services, List ingresses) { + return new ServerResolver(services, ingresses); } /** @@ -66,7 +78,8 @@ public static ServerResolver of(List services, List routes) { public Map resolve(String machineName) { Map servers = new HashMap<>(); services.get(machineName).forEach(service -> fillServiceServers(service, servers)); - routes.get(machineName).forEach(route -> fillRouteServers(route, servers)); + // routes.get(machineName).forEach(route -> fillRouteServers(route, servers)); + ingresses.get(machineName).forEach(ingress -> fillIngressServers(ingress, servers)); return servers; } @@ -101,6 +114,29 @@ private void fillRouteServers(Route route, Map servers) { config.getAttributes()))); } + private void fillIngressServers(Ingress ingress, Map servers) { + IngressRule ingressRule = ingress.getSpec().getRules().get(0); + + // host either set by rule, or determined by LB ip + final String host = + ingressRule.getHost() != null + ? ingressRule.getHost() + : ingress.getStatus().getLoadBalancer().getIngress().get(0).getIp(); + + Annotations.newDeserializer(ingress.getMetadata().getAnnotations()) + .servers() + .forEach( + (name, config) -> + servers.put( + name, + newServer( + config.getProtocol(), + host, + null, + ingressRule.getHttp().getPaths().get(0).getPath() + "/" + config.getPath(), + config.getAttributes()))); + } + /** Constructs {@link ServerImpl} instance from provided parameters. */ private ServerImpl newServer( String protocol, String host, String port, String path, Map attributes) { diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironment.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironment.java index c7c47e6dedd..20f3c13a7aa 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironment.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironment.java @@ -13,6 +13,7 @@ import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.extensions.Ingress; import io.fabric8.openshift.api.model.Route; import java.util.ArrayList; import java.util.HashMap; @@ -35,6 +36,7 @@ public class OpenShiftEnvironment extends InternalEnvironment { private final Map pods; private final Map services; private final Map routes; + private final Map ingresses; private final Map persistentVolumeClaims; public static Builder builder() { @@ -48,11 +50,13 @@ private OpenShiftEnvironment( Map pods, Map services, Map routes, + Map ingresses, Map persistentVolumeClaims) { super(internalRecipe, machines, warnings); this.pods = pods; this.services = services; this.routes = routes; + this.ingresses = ingresses; this.persistentVolumeClaims = persistentVolumeClaims; } @@ -71,6 +75,11 @@ public Map getRoutes() { return routes; } + /** Returns services that should be created when environment starts. */ + public Map getIngresses() { + return ingresses; + } + /** Returns PVCs that should be created when environment starts. */ public Map getPersistentVolumeClaims() { return persistentVolumeClaims; @@ -83,6 +92,7 @@ public static class Builder { private final Map pods = new HashMap<>(); private final Map services = new HashMap<>(); private final Map routes = new HashMap<>(); + private final Map ingresses = new HashMap<>(); private final Map persistentVolumeClaims = new HashMap<>(); private Builder() {} @@ -117,6 +127,11 @@ public Builder setRoutes(Map route) { return this; } + public Builder setIngresses(Map ingresses) { + this.ingresses.putAll(ingresses); + return this; + } + public Builder setPersistentVolumeClaims(Map pvcs) { this.persistentVolumeClaims.putAll(pvcs); return this; @@ -124,7 +139,14 @@ public Builder setPersistentVolumeClaims(Map pvcs public OpenShiftEnvironment build() { return new OpenShiftEnvironment( - internalRecipe, machines, warnings, pods, services, routes, persistentVolumeClaims); + internalRecipe, + machines, + warnings, + pods, + services, + routes, + ingresses, + persistentVolumeClaims); } } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java index d58f6e397cb..fb95d13e333 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java @@ -10,26 +10,17 @@ */ package org.eclipse.che.workspace.infrastructure.openshift.project; -import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; -import io.fabric8.kubernetes.api.model.extensions.HTTPIngressPath; -import io.fabric8.kubernetes.api.model.extensions.HTTPIngressPathBuilder; -import io.fabric8.kubernetes.api.model.extensions.HTTPIngressRuleValue; -import io.fabric8.kubernetes.api.model.extensions.HTTPIngressRuleValueBuilder; +import io.fabric8.kubernetes.api.model.extensions.DoneableIngress; import io.fabric8.kubernetes.api.model.extensions.Ingress; -import io.fabric8.kubernetes.api.model.extensions.IngressBackend; -import io.fabric8.kubernetes.api.model.extensions.IngressBackendBuilder; -import io.fabric8.kubernetes.api.model.extensions.IngressBuilder; -import io.fabric8.kubernetes.api.model.extensions.IngressRule; -import io.fabric8.kubernetes.api.model.extensions.IngressRuleBuilder; -import io.fabric8.kubernetes.api.model.extensions.IngressSpec; -import io.fabric8.kubernetes.api.model.extensions.IngressSpecBuilder; import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.openshift.api.model.Route; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; +import io.fabric8.kubernetes.client.Watch; +import io.fabric8.kubernetes.client.Watcher; +import io.fabric8.kubernetes.client.dsl.Resource; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; @@ -45,68 +36,135 @@ public class KubernetesIngress { this.clientFactory = clientFactory; } - /** - * Creates specified route. - * - * @param routes - List of OS routes to create as an ingress - * @return created route - * @throws InfrastructureException when any exception occurs - */ - public Ingress create(List routes) throws InfrastructureException { + public Ingress create(Ingress ingress) throws InfrastructureException { try { - List httpIngressPaths = new ArrayList<>(); - HashSet servers = new HashSet<>(); - for (Route route : routes) { - String server = route.getSpec().getPort().getTargetPort().getStrVal(); - if (!servers.contains(server)) { - IngressBackend ingressBackend = - new IngressBackendBuilder() - .withServiceName(route.getSpec().getTo().getName()) - .withNewServicePort(server) - .build(); - - String serverPath = "/" + workspaceId + "/" + server; - - HTTPIngressPath httpIngressPath = - new HTTPIngressPathBuilder() - // .withPath(route.getSpec().getPath()) - .withPath(serverPath) - .withBackend(ingressBackend) - .build(); - servers.add(server); - httpIngressPaths.add(httpIngressPath); - } - } - - HTTPIngressRuleValue httpIngressRuleValue = - new HTTPIngressRuleValueBuilder().withPaths(httpIngressPaths).build(); - IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build(); - IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build(); - Map ingressAnontations = new HashMap<>(); - ingressAnontations.put("ingress.kubernetes.io/rewrite-target", "/"); - ingressAnontations.put("ingress.kubernetes.io/ssl-redirect", "false"); - ingressAnontations.put("kubernetes.io/ingress.class", "nginx"); - Ingress ingress = - new IngressBuilder() - .withSpec(ingressSpec) - .withMetadata( - new ObjectMetaBuilder() - .withName(workspaceId + "-ingress") - .withAnnotations(ingressAnontations) - .build()) - .build(); return clientFactory .create() .extensions() .ingresses() .inNamespace(namespace) - .withName(workspaceId + "-ingress") + .withName(ingress.getMetadata().getName()) .create(ingress); } catch (KubernetesClientException e) { throw new InfrastructureException(e.getMessage(), e); } } + public Ingress wait(String name, int timeoutMin, Predicate predicate) + throws InfrastructureException { + CompletableFuture future = new CompletableFuture<>(); + Watch watch = null; + try { + + Resource ingressResource = + clientFactory.create().extensions().ingresses().inNamespace(namespace).withName(name); + + watch = + ingressResource.watch( + new Watcher() { + @Override + public void eventReceived(Action action, Ingress ingress) { + if (predicate.test(ingress)) { + future.complete(ingress); + } + } + + @Override + public void onClose(KubernetesClientException cause) { + future.completeExceptionally( + new InfrastructureException( + "Waiting for ingress '" + name + "' was interrupted")); + } + }); + + Ingress actualIngress = ingressResource.get(); + if (actualIngress == null) { + throw new InfrastructureException("Specified ingress " + name + " doesn't exist"); + } + if (predicate.test(actualIngress)) { + return actualIngress; + } + try { + return future.get(timeoutMin, TimeUnit.MINUTES); + } catch (ExecutionException e) { + throw new InfrastructureException(e.getCause().getMessage(), e); + } catch (TimeoutException e) { + throw new InfrastructureException("Waiting for ingress '" + name + "' reached timeout"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new InfrastructureException("Waiting for ingress '" + name + "' was interrupted"); + } + } catch (KubernetesClientException e) { + throw new InfrastructureException(e.getMessage(), e); + } finally { + if (watch != null) { + watch.close(); + } + } + } + + /** + * Creates specified route. + * + * @param routes - List of OS routes to create as an ingress + * @return created route + * @throws InfrastructureException when any exception occurs + */ + // public Ingress create(List routes) throws InfrastructureException { + // try { + // List httpIngressPaths = new ArrayList<>(); + // HashSet servers = new HashSet<>(); + // for (Route route : routes) { + // String server = route.getSpec().getPort().getTargetPort().getStrVal(); + // if (!servers.contains(server)) { + // IngressBackend ingressBackend = + // new IngressBackendBuilder() + // .withServiceName(route.getSpec().getTo().getName()) + // .withNewServicePort(server) + // .build(); + // + // String serverPath = "/" + workspaceId + "/" + server; + // + // HTTPIngressPath httpIngressPath = + // new HTTPIngressPathBuilder() + // // .withPath(route.getSpec().getPath()) + // .withPath(serverPath) + // .withBackend(ingressBackend) + // .build(); + // servers.add(server); + // httpIngressPaths.add(httpIngressPath); + // } + // } + // + // HTTPIngressRuleValue httpIngressRuleValue = + // new HTTPIngressRuleValueBuilder().withPaths(httpIngressPaths).build(); + // IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build(); + // IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build(); + // Map ingressAnontations = new HashMap<>(); + // ingressAnontations.put("ingress.kubernetes.io/rewrite-target", "/"); + // ingressAnontations.put("ingress.kubernetes.io/ssl-redirect", "false"); + // ingressAnontations.put("kubernetes.io/ingress.class", "nginx"); + // Ingress ingress = + // new IngressBuilder() + // .withSpec(ingressSpec) + // .withMetadata( + // new ObjectMetaBuilder() + // .withName(workspaceId + "-ingress") + // .withAnnotations(ingressAnontations) + // .build()) + // .build(); + // return clientFactory + // .create() + // .extensions() + // .ingresses() + // .inNamespace(namespace) + // .withName(workspaceId + "-ingress") + // .create(ingress); + // } catch (KubernetesClientException e) { + // throw new InfrastructureException(e.getMessage(), e); + // } + // } + public void delete() throws InfrastructureException { clientFactory .create() diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPods.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPods.java index 37020c60fa7..03c9a1b35d2 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPods.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPods.java @@ -150,6 +150,7 @@ public Pod wait(String name, int timeoutMin, Predicate predicate) CompletableFuture future = new CompletableFuture<>(); Watch watch = null; try { + PodResource podResource = clientFactory.create().pods().inNamespace(namespace).withName(name); diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/server/ServersConverter.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/server/ServersConverter.java index efbe9048337..879f332a7b1 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/server/ServersConverter.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/server/ServersConverter.java @@ -14,6 +14,8 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodSpec; import java.util.Map; +import javax.inject.Inject; +import javax.inject.Named; import javax.inject.Singleton; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; @@ -36,6 +38,13 @@ @Singleton public class ServersConverter implements ConfigurationProvisioner { + private final String host; + + @Inject + public ServersConverter(@Named("che.host") String host) { + this.host = host; + } + @Override public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) throws InfrastructureException { @@ -47,7 +56,7 @@ public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) InternalMachineConfig machineConfig = osEnv.getMachines().get(machineName); if (!machineConfig.getServers().isEmpty()) { ServerExposer serverExposer = - new ServerExposer(machineName, podConfig, containerConfig, osEnv); + new ServerExposer(machineName, podConfig, containerConfig, osEnv, host); serverExposer.expose(machineConfig.getServers()); } } diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java index cd351bedc53..84e814a9dc5 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java @@ -10,7 +10,6 @@ */ package org.eclipse.che.workspace.infrastructure.openshift; -import static com.google.common.collect.Lists.newArrayList; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -33,7 +32,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.ImmutableList; @@ -156,8 +154,7 @@ public void setup() throws Exception { workspaceProbesFactory, context, project, - emptyList(), - ""); + emptyList()); when(context.getEnvironment()).thenReturn(osEnv); when(serverCheckerFactory.create(any(), anyString(), any())).thenReturn(serversChecker); when(context.getIdentity()).thenReturn(IDENTITY); @@ -305,23 +302,24 @@ public void throwsInfrastructureExceptionWhenOpenShiftProjectCleanupFailed() thr internalRuntime.internalStop(emptyMap()); } - @Test - public void testRepublishContainerOutputAsMachineLogEvents() throws Exception { - final MachineLogsPublisher logsPublisher = internalRuntime.new MachineLogsPublisher(); - final ContainerEvent out1 = mockContainerEvent("pulling image", "07/07/2007 19:01:22"); - final ContainerEvent out2 = mockContainerEvent("image pulled", "07/07/2007 19:08:53"); - final ArgumentCaptor captor = ArgumentCaptor.forClass(MachineLogEvent.class); - - internalRuntime.createPods( - newArrayList(allServices.values()), newArrayList(allRoutes.values())); - logsPublisher.handle(out1); - logsPublisher.handle(out2); - - verify(eventService, atLeastOnce()).publish(captor.capture()); - final ImmutableList machineLogs = - ImmutableList.of(asMachineLogEvent(out1), asMachineLogEvent(out2)); - assertTrue(captor.getAllValues().containsAll(machineLogs)); - } + // @Test + // public void testRepublishContainerOutputAsMachineLogEvents() throws Exception { + // final MachineLogsPublisher logsPublisher = internalRuntime.new MachineLogsPublisher(); + // final ContainerEvent out1 = mockContainerEvent("pulling image", "07/07/2007 19:01:22"); + // final ContainerEvent out2 = mockContainerEvent("image pulled", "07/07/2007 19:08:53"); + // final ArgumentCaptor captor = + // ArgumentCaptor.forClass(MachineLogEvent.class); + // + // internalRuntime.createPods( + // newArrayList(allServices.values()), newArrayList(allRoutes.values())); + // logsPublisher.handle(out1); + // logsPublisher.handle(out2); + // + // verify(eventService, atLeastOnce()).publish(captor.capture()); + // final ImmutableList machineLogs = + // ImmutableList.of(asMachineLogEvent(out1), asMachineLogEvent(out2)); + // assertTrue(captor.getAllValues().containsAll(machineLogs)); + // } @Test public void testDoNotPublishForeignMachineOutput() throws Exception { diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposerTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposerTest.java index d52fa8c1d30..3ca9a9febe1 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposerTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposerTest.java @@ -73,7 +73,7 @@ public void setUp() throws Exception { openShiftEnvironment = OpenShiftEnvironment.builder().setPods(ImmutableMap.of("pod", pod)).build(); - this.serverExposer = new ServerExposer(MACHINE_NAME, pod, container, openShiftEnvironment); + this.serverExposer = new ServerExposer(MACHINE_NAME, pod, container, openShiftEnvironment, ""); } @Test diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolverTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolverTest.java index a74e3ac71a2..a3193f6a8b4 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolverTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolverTest.java @@ -10,24 +10,9 @@ */ package org.eclipse.che.workspace.infrastructure.openshift; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static org.eclipse.che.api.core.model.workspace.runtime.ServerStatus.UNKNOWN; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import com.google.common.collect.ImmutableMap; -import io.fabric8.kubernetes.api.model.Service; -import io.fabric8.kubernetes.api.model.ServiceBuilder; -import io.fabric8.kubernetes.api.model.ServicePortBuilder; -import io.fabric8.openshift.api.model.Route; -import io.fabric8.openshift.api.model.RouteBuilder; import java.util.Map; -import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; -import org.eclipse.che.api.workspace.server.model.impl.ServerImpl; -import org.eclipse.che.workspace.infrastructure.openshift.Annotations.Serializer; -import org.testng.annotations.Test; /** * Test for {@link ServerResolver}. @@ -40,207 +25,210 @@ public class ServerResolverTest { private static final int CONTAINER_PORT = 3054; private static final String ROUTE_HOST = "localhost"; - @Test - public void - testResolvingServersWhenThereIsNoTheCorrespondingServiceAndRouteForTheSpecifiedMachine() { - // given - Service nonMatchedByPodService = - createService("nonMatched", "foreignMachine", CONTAINER_PORT, null); - Route route = - createRoute( - "nonMatched", - "foreignMachine", - ImmutableMap.of( - "http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); - - ServerResolver serverResolver = - ServerResolver.of(singletonList(nonMatchedByPodService), singletonList(route)); - - // when - Map resolved = serverResolver.resolve("machine"); - - // then - assertTrue(resolved.isEmpty()); - } - - @Test - public void testResolvingServersWhenThereIsMatchedRouteForTheSpecifiedMachine() { - Route route = - createRoute( - "matched", - "machine", - ImmutableMap.of( - "http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); - - ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); - - Map resolved = serverResolver.resolve("machine"); - - assertEquals(resolved.size(), 1); - assertEquals( - resolved.get("http-server"), - new ServerImpl() - .withUrl("http://localhost/api") - .withStatus(UNKNOWN) - .withAttributes(ATTRIBUTES_MAP)); - } - - @Test - public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsNull() { - Route route = - createRoute( - "matched", - "machine", - singletonMap( - "http-server", new ServerConfigImpl("3054", "http", null, ATTRIBUTES_MAP))); - - ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); - - Map resolved = serverResolver.resolve("machine"); - - assertEquals(resolved.size(), 1); - assertEquals( - resolved.get("http-server"), - new ServerImpl() - .withUrl("http://localhost") - .withStatus(UNKNOWN) - .withAttributes(ATTRIBUTES_MAP)); - } - - @Test - public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsEmpty() { - Route route = - createRoute( - "matched", - "machine", - singletonMap("http-server", new ServerConfigImpl("3054", "http", "", ATTRIBUTES_MAP))); - - ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); - - Map resolved = serverResolver.resolve("machine"); - - assertEquals(resolved.size(), 1); - assertEquals( - resolved.get("http-server"), - new ServerImpl() - .withUrl("http://localhost") - .withStatus(UNKNOWN) - .withAttributes(ATTRIBUTES_MAP)); - } - - @Test - public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsRelative() { - Route route = - createRoute( - "matched", - "machine", - singletonMap( - "http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); - - ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); - - Map resolved = serverResolver.resolve("machine"); - - assertEquals(resolved.size(), 1); - assertEquals( - resolved.get("http-server"), - new ServerImpl() - .withUrl("http://localhost/api") - .withStatus(UNKNOWN) - .withAttributes(ATTRIBUTES_MAP)); - } - - @Test - public void testResolvingInternalServers() { - Service service = - createService( - "service11", - "machine", - CONTAINER_PORT, - singletonMap( - "http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); - Route route = createRoute("matched", "machine", null); - - ServerResolver serverResolver = ServerResolver.of(singletonList(service), singletonList(route)); - - Map resolved = serverResolver.resolve("machine"); - - assertEquals(resolved.size(), 1); - assertEquals( - resolved.get("http-server"), - new ServerImpl() - .withUrl("http://service11:3054/api") - .withStatus(UNKNOWN) - .withAttributes(ATTRIBUTES_MAP)); - } - - @Test - public void testResolvingInternalServersWithPortWithTransportProtocol() { - Service service = - createService( - "service11", - "machine", - CONTAINER_PORT, - singletonMap( - "http-server", new ServerConfigImpl("3054/udp", "xxx", "api", ATTRIBUTES_MAP))); - Route route = createRoute("matched", "machine", null); - - ServerResolver serverResolver = ServerResolver.of(singletonList(service), singletonList(route)); - - Map resolved = serverResolver.resolve("machine"); - - assertEquals(resolved.size(), 1); - assertEquals( - resolved.get("http-server"), - new ServerImpl() - .withUrl("xxx://service11:3054/api") - .withStatus(UNKNOWN) - .withAttributes(ATTRIBUTES_MAP)); - } - - private Service createService( - String name, String machineName, Integer port, Map servers) { - Serializer serializer = Annotations.newSerializer(); - serializer.machineName(machineName); - if (servers != null) { - serializer.servers(servers); - } - - return new ServiceBuilder() - .withNewMetadata() - .withName(name) - .withAnnotations(serializer.annotations()) - .endMetadata() - .withNewSpec() - .withPorts( - new ServicePortBuilder() - .withPort(port) - .withNewTargetPort() - .withIntVal(port) - .endTargetPort() - .build()) - .endSpec() - .build(); - } - - private Route createRoute( - String name, String machineName, Map servers) { - Serializer serializer = Annotations.newSerializer(); - serializer.machineName(machineName); - if (servers != null) { - serializer.servers(servers); - } - return new RouteBuilder() - .withNewMetadata() - .withName(name) - .withAnnotations(serializer.annotations()) - .endMetadata() - .withNewSpec() - .withHost(ROUTE_HOST) - .withNewTo() - .withName(name) - .endTo() - .endSpec() - .build(); - } + // @Test + // public void + // testResolvingServersWhenThereIsNoTheCorrespondingServiceAndRouteForTheSpecifiedMachine() { + // // given + // Service nonMatchedByPodService = + // createService("nonMatched", "foreignMachine", CONTAINER_PORT, null); + // Route route = + // createRoute( + // "nonMatched", + // "foreignMachine", + // ImmutableMap.of( + // "http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); + // + // ServerResolver serverResolver = + // ServerResolver.of(singletonList(nonMatchedByPodService), singletonList(route)); + // + // // when + // Map resolved = serverResolver.resolve("machine"); + // + // // then + // assertTrue(resolved.isEmpty()); + // } + // + // @Test + // public void testResolvingServersWhenThereIsMatchedRouteForTheSpecifiedMachine() { + // Route route = + // createRoute( + // "matched", + // "machine", + // ImmutableMap.of( + // "http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); + // + // ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); + // + // Map resolved = serverResolver.resolve("machine"); + // + // assertEquals(resolved.size(), 1); + // assertEquals( + // resolved.get("http-server"), + // new ServerImpl() + // .withUrl("http://localhost/api") + // .withStatus(UNKNOWN) + // .withAttributes(ATTRIBUTES_MAP)); + // } + // + // @Test + // public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsNull() { + // Route route = + // createRoute( + // "matched", + // "machine", + // singletonMap( + // "http-server", new ServerConfigImpl("3054", "http", null, ATTRIBUTES_MAP))); + // + // ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); + // + // Map resolved = serverResolver.resolve("machine"); + // + // assertEquals(resolved.size(), 1); + // assertEquals( + // resolved.get("http-server"), + // new ServerImpl() + // .withUrl("http://localhost") + // .withStatus(UNKNOWN) + // .withAttributes(ATTRIBUTES_MAP)); + // } + // + // @Test + // public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsEmpty() { + // Route route = + // createRoute( + // "matched", + // "machine", + // singletonMap("http-server", new ServerConfigImpl("3054", "http", "", + // ATTRIBUTES_MAP))); + // + // ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); + // + // Map resolved = serverResolver.resolve("machine"); + // + // assertEquals(resolved.size(), 1); + // assertEquals( + // resolved.get("http-server"), + // new ServerImpl() + // .withUrl("http://localhost") + // .withStatus(UNKNOWN) + // .withAttributes(ATTRIBUTES_MAP)); + // } + // + // @Test + // public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsRelative() { + // Route route = + // createRoute( + // "matched", + // "machine", + // singletonMap( + // "http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); + // + // ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); + // + // Map resolved = serverResolver.resolve("machine"); + // + // assertEquals(resolved.size(), 1); + // assertEquals( + // resolved.get("http-server"), + // new ServerImpl() + // .withUrl("http://localhost/api") + // .withStatus(UNKNOWN) + // .withAttributes(ATTRIBUTES_MAP)); + // } + // + // @Test + // public void testResolvingInternalServers() { + // Service service = + // createService( + // "service11", + // "machine", + // CONTAINER_PORT, + // singletonMap( + // "http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); + // Route route = createRoute("matched", "machine", null); + // + // ServerResolver serverResolver = ServerResolver.of(singletonList(service), + // singletonList(route)); + // + // Map resolved = serverResolver.resolve("machine"); + // + // assertEquals(resolved.size(), 1); + // assertEquals( + // resolved.get("http-server"), + // new ServerImpl() + // .withUrl("http://service11:3054/api") + // .withStatus(UNKNOWN) + // .withAttributes(ATTRIBUTES_MAP)); + // } + // + // @Test + // public void testResolvingInternalServersWithPortWithTransportProtocol() { + // Service service = + // createService( + // "service11", + // "machine", + // CONTAINER_PORT, + // singletonMap( + // "http-server", new ServerConfigImpl("3054/udp", "xxx", "api", ATTRIBUTES_MAP))); + // Route route = createRoute("matched", "machine", null); + // + // ServerResolver serverResolver = ServerResolver.of(singletonList(service), + // singletonList(route)); + // + // Map resolved = serverResolver.resolve("machine"); + // + // assertEquals(resolved.size(), 1); + // assertEquals( + // resolved.get("http-server"), + // new ServerImpl() + // .withUrl("xxx://service11:3054/api") + // .withStatus(UNKNOWN) + // .withAttributes(ATTRIBUTES_MAP)); + // } + // + // private Service createService( + // String name, String machineName, Integer port, Map servers) { + // Serializer serializer = Annotations.newSerializer(); + // serializer.machineName(machineName); + // if (servers != null) { + // serializer.servers(servers); + // } + // + // return new ServiceBuilder() + // .withNewMetadata() + // .withName(name) + // .withAnnotations(serializer.annotations()) + // .endMetadata() + // .withNewSpec() + // .withPorts( + // new ServicePortBuilder() + // .withPort(port) + // .withNewTargetPort() + // .withIntVal(port) + // .endTargetPort() + // .build()) + // .endSpec() + // .build(); + // } + // + // private Route createRoute( + // String name, String machineName, Map servers) { + // Serializer serializer = Annotations.newSerializer(); + // serializer.machineName(machineName); + // if (servers != null) { + // serializer.servers(servers); + // } + // return new RouteBuilder() + // .withNewMetadata() + // .withName(name) + // .withAnnotations(serializer.annotations()) + // .endMetadata() + // .withNewSpec() + // .withHost(ROUTE_HOST) + // .withNewTo() + // .withName(name) + // .endTo() + // .endSpec() + // .build(); + // } } From 242f56a8fd88a9ce1e57a40654d608330fafb0b4 Mon Sep 17 00:00:00 2001 From: Sergii Leshchenko Date: Thu, 1 Feb 2018 16:45:39 +0200 Subject: [PATCH 03/11] CHE-5908 Add Kubernetes infrastructure --- .../che/api/deploy/WsMasterModule.java | 7 +- .../webapp/WEB-INF/classes/che/che.properties | 73 ++- dockerfiles/init/manifests/che.env | 58 +- .../openshift/files/scripts/che-config | 18 +- .../openshift/files/scripts/deploy_che.sh | 22 +- .../modules/openshift/files/scripts/ocp.sh | 2 +- .../DockerfileEnvironmentConverterTest.java | 2 +- infrastructures/kubernetes/pom.xml | 135 +++++ .../kubernetes}/Annotations.java | 10 +- .../infrastructure/kubernetes}/Constants.java | 8 +- .../kubernetes/KubernetesClientFactory.java | 103 ++++ .../KubernetesEnvironmentProvisioner.java | 90 ++++ .../kubernetes/KubernetesInfraModule.java | 69 +++ .../kubernetes/KubernetesInfrastructure.java | 82 +++ .../kubernetes/KubernetesInternalRuntime.java | 443 ++++++++++++++++ .../kubernetes/KubernetesMachine.java} | 24 +- .../kubernetes/KubernetesRuntimeContext.java | 65 +++ .../KubernetesRuntimeContextFactory.java | 23 + .../kubernetes/KubernetesRuntimeFactory.java | 24 + .../infrastructure/kubernetes}/Names.java | 9 +- .../bootstrapper/KubernetesBootstrapper.java} | 42 +- .../KubernetesBootstrapperFactory.java} | 10 +- .../environment/KubernetesEnvironment.java | 141 +++++ .../KubernetesEnvironmentFactory.java | 190 +++++++ .../KubernetesEnvironmentValidator.java} | 16 +- .../DockerImageEnvironmentConverter.java | 12 +- .../namespace/KubernetesIngresses.java} | 104 +--- .../namespace/KubernetesNamespace.java | 157 ++++++ .../namespace/KubernetesNamespaceFactory.java | 46 ++ .../namespace/KubernetesObjectUtil.java} | 26 +- .../KubernetesPersistentVolumeClaims.java} | 14 +- .../kubernetes/namespace/KubernetesPods.java} | 32 +- .../namespace/KubernetesServices.java} | 18 +- .../RemoveNamespaceOnWorkspaceRemove.java | 71 +++ .../namespace}/event/ContainerEvent.java | 2 +- .../event/ContainerEventHandler.java | 4 +- .../namespace}/event/PodActionHandler.java | 4 +- .../namespace}/pvc/CommonPVCStrategy.java | 50 +- .../namespace}/pvc/PVCSubPathHelper.java | 22 +- .../pvc/UniqueWorkspacePVCStrategy.java | 71 +-- .../namespace}/pvc/WorkspacePVCCleaner.java | 18 +- .../pvc/WorkspaceVolumeStrategyProvider.java | 4 +- .../pvc/WorkspaceVolumesStrategy.java | 14 +- .../provision/ConfigurationProvisioner.java | 34 ++ .../InstallerServersPortProvisioner.java | 16 +- .../KubernetesCheApiEnvVarProvider.java} | 8 +- .../LogsVolumeMachineProvisioner.java | 6 +- .../provision/UniqueNamesProvisioner.java | 63 +++ .../provision/env/EnvVarsConverter.java | 16 +- .../env/LogsRootEnvVariableProvider.java | 2 +- .../limits/ram/RamLimitProvisioner.java | 18 +- .../restartpolicy/RestartPolicyRewriter.java | 14 +- .../provision/server/ServersConverter.java | 56 ++ .../server/KubernetesServerExposer.java} | 115 ++-- .../server/KubernetesServerResolver.java} | 95 ++-- .../kubernetes}/util/Containers.java | 2 +- .../kubernetes}/util/KubernetesSize.java | 2 +- .../kubernetes}/AnnotationsTest.java | 2 +- .../KubernetesEnvironmentProvisionerTest.java | 97 ++++ .../KubernetesInternalRuntimeTest.java | 496 ++++++++++++++++++ .../KubernetesEnvironmentFactoryTest.java | 221 ++++++++ .../KubernetesEnvironmentValidatorTest.java} | 24 +- .../DockerImageEnvironmentConverterTest.java | 14 +- .../namespace/KubernetesNamespaceTest.java | 145 +++++ .../RemoveNamespaceOnWorkspaceRemoveTest.java | 67 +++ .../namespace}/pvc/CommonPVCStrategyTest.java | 70 +-- .../namespace}/pvc/PVCSubPathHelperTest.java | 47 +- .../pvc/UniqueWorkspacePVCStrategyTest.java | 74 +-- .../pvc/WorkspacePVCCleanerTest.java | 10 +- .../InstallerServersPortProvisionerTest.java | 14 +- .../KubernetesCheApiEnvVarProviderTest.java} | 15 +- .../provision/UniqueNamesProvisionerTest.java | 96 ++++ .../env/LogsRootEnvVariableProviderTest.java | 4 +- .../limits/ram/RamLimitProvisionerTest.java | 14 +- .../RestartPolicyRewriterTest.java | 16 +- .../server/KubernetesServerExposerTest.java | 407 ++++++++++++++ .../server/KubernetesServerResolverTest.java | 265 ++++++++++ .../kubernetes}/util/ContainersTest.java | 2 +- .../kubernetes}/util/KubernetesSizeTest.java | 2 +- .../src/test/resources/logback-test.xml | 35 ++ infrastructures/openshift/pom.xml | 21 +- .../openshift/OpenShiftClientFactory.java | 14 +- .../OpenShiftEnvironmentProvisioner.java | 41 +- .../openshift/OpenShiftInfraModule.java | 29 +- .../openshift/OpenShiftInfrastructure.java | 16 +- .../openshift/OpenShiftInternalRuntime.java | 401 ++------------ .../openshift/OpenShiftRuntimeContext.java | 32 +- .../openshift/OpenShiftRuntimeFactory.java | 3 +- .../environment/OpenShiftEnvironment.java | 69 +-- .../OpenShiftEnvironmentFactory.java | 13 +- .../openshift/project/OpenShiftProject.java | 116 +--- .../project/OpenShiftProjectFactory.java | 6 +- .../openshift/project/OpenShiftRoutes.java | 4 +- .../RemoveProjectOnWorkspaceRemove.java | 2 +- .../provision/ConfigurationProvisioner.java | 35 -- ...er.java => OpenShiftServersConverter.java} | 29 +- .../OpenShiftUniqueNamesProvisioner.java | 44 ++ ...visioner.java => RouteTlsProvisioner.java} | 14 +- .../provision/UniqueNamesProvisioner.java | 64 --- .../server/OpenShiftServerExposer.java | 184 +++++++ .../server/OpenShiftServerResolver.java | 70 +++ .../OpenShiftEnvironmentProvisionerTest.java | 24 +- .../OpenShiftInternalRuntimeTest.java | 225 +------- ...t.java => OpenShiftServerExposerTest.java} | 15 +- .../OpenShiftServerResolverTest.java | 254 +++++++++ .../openshift/ServerResolverTest.java | 234 --------- .../OpenShiftEnvironmentFactoryTest.java | 7 +- .../project/OpenShiftProjectTest.java | 26 +- .../LogsVolumeMachineProvisionerTest.java | 3 +- ... OpenShiftUniqueNamesProvisionerTest.java} | 10 +- .../provision/RouteTlsProvisionerTest.java | 96 ++++ .../route/TlsRouteProvisionerTest.java | 54 -- infrastructures/pom.xml | 1 + pom.xml | 5 + .../workspace/server/hc/ServersChecker.java | 43 +- .../ExecServerLivenessProbeConfigFactory.java | 2 +- ...minalServerLivenessProbeConfigFactory.java | 4 +- .../server/hc/ServersCheckerTest.java | 6 +- .../hc/probe/WorkspaceProbesFactoryTest.java | 8 +- 119 files changed, 5200 insertions(+), 1943 deletions(-) create mode 100644 infrastructures/kubernetes/pom.xml rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/Annotations.java (92%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/Constants.java (80%) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesClientFactory.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfrastructure.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftMachine.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesMachine.java} (81%) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeContext.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeContextFactory.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeFactory.java rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/Names.java (86%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/bootstrapper/OpenShiftBootstrapper.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java} (77%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/bootstrapper/OpenShiftBootstrapperFactory.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapperFactory.java} (71%) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironment.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentFactory.java rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentValidator.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentValidator.java} (77%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/environment/convert/DockerImageEnvironmentConverter.java (83%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesIngresses.java} (52%) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespace.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespaceFactory.java rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftObjectUtil.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesObjectUtil.java} (82%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPersistentVolumeClaims.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPersistentVolumeClaims.java} (86%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPods.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPods.java} (92%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftServices.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesServices.java} (77%) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/event/ContainerEvent.java (96%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/event/ContainerEventHandler.java (78%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/event/PodActionHandler.java (80%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/CommonPVCStrategy.java (75%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/PVCSubPathHelper.java (90%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/UniqueWorkspacePVCStrategy.java (72%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/WorkspacePVCCleaner.java (77%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/WorkspaceVolumeStrategyProvider.java (90%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/WorkspaceVolumesStrategy.java (74%) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/ConfigurationProvisioner.java rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/InstallerServersPortProvisioner.java (91%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftCheApiEnvVarProvider.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/KubernetesCheApiEnvVarProvider.java} (76%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/LogsVolumeMachineProvisioner.java (84%) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/UniqueNamesProvisioner.java rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/env/EnvVarsConverter.java (69%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/env/LogsRootEnvVariableProvider.java (94%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/limits/ram/RamLimitProvisioner.java (65%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/restartpolicy/RestartPolicyRewriter.java (76%) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposer.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java} (76%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerResolver.java} (67%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/util/Containers.java (96%) rename infrastructures/{openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes}/util/KubernetesSize.java (97%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes}/AnnotationsTest.java (98%) create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentFactoryTest.java rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentValidatorTest.java => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentValidatorTest.java} (77%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes}/environment/convert/DockerImageEnvironmentConverterTest.java (82%) create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespaceTest.java create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemoveTest.java rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/CommonPVCStrategyTest.java (79%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/PVCSubPathHelperTest.java (73%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/UniqueWorkspacePVCStrategyTest.java (80%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace}/pvc/WorkspacePVCCleanerTest.java (90%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/InstallerServersPortProvisionerTest.java (96%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/installer/OpenShiftCheApiEnvVarProviderTest.java => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/KubernetesCheApiEnvVarProviderTest.java} (62%) create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/UniqueNamesProvisionerTest.java rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/env/LogsRootEnvVariableProviderTest.java (87%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/limits/ram/RamLimitProvisionerTest.java (88%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes}/provision/restartpolicy/RestartPolicyRewriterTest.java (84%) create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerResolverTest.java rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes}/util/ContainersTest.java (98%) rename infrastructures/{openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift => kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes}/util/KubernetesSizeTest.java (94%) create mode 100644 infrastructures/kubernetes/src/test/resources/logback-test.xml delete mode 100644 infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/ConfigurationProvisioner.java rename infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/{server/ServersConverter.java => OpenShiftServersConverter.java} (68%) create mode 100644 infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftUniqueNamesProvisioner.java rename infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/{route/TlsRouteProvisioner.java => RouteTlsProvisioner.java} (85%) delete mode 100644 infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/UniqueNamesProvisioner.java create mode 100644 infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java create mode 100644 infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerResolver.java rename infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/{ServerExposerTest.java => OpenShiftServerExposerTest.java} (96%) create mode 100644 infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftServerResolverTest.java delete mode 100644 infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolverTest.java rename infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/{UniqueNamesProvisionerTest.java => OpenShiftUniqueNamesProvisionerTest.java} (90%) create mode 100644 infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/RouteTlsProvisionerTest.java delete mode 100644 infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/route/TlsRouteProvisionerTest.java diff --git a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java index b9896cd7d5f..6676d75558a 100644 --- a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java +++ b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java @@ -77,7 +77,10 @@ import org.eclipse.che.security.PasswordEncryptor; import org.eclipse.che.workspace.infrastructure.docker.DockerInfraModule; import org.eclipse.che.workspace.infrastructure.docker.local.LocalDockerModule; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInfraModule; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInfrastructure; import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftInfraModule; +import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftInfrastructure; import org.eclipse.persistence.config.PersistenceUnitProperties; import org.flywaydb.core.internal.util.PlaceholderReplacer; @@ -229,8 +232,10 @@ protected void configure() { .properties(persistenceProperties)); String infrastructure = System.getenv("CHE_INFRASTRUCTURE_ACTIVE"); - if ("openshift".equals(infrastructure)) { + if (OpenShiftInfrastructure.NAME.equals(infrastructure)) { install(new OpenShiftInfraModule()); + } else if (KubernetesInfrastructure.NAME.equals(infrastructure)) { + install(new KubernetesInfraModule()); } else { install(new LocalDockerModule()); install(new DockerInfraModule()); diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties index 027c6119b50..9c3efcbbb97 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties @@ -317,46 +317,46 @@ db.schema.flyway.scripts.locations=classpath:che-schema # Note that this property is needed for backward compatibility and will be removed soon. che.predefined.stacks.reload_on_start=false -### OpenShift Infra parameters - -che.infra.openshift.master_url= -che.infra.openshift.username= -che.infra.openshift.password= -che.infra.openshift.oauth_token= -che.infra.openshift.trust_certs= +### Kubernetes Infra parameters -# Create routes with Transport Layer Security (TLS) enabled -che.infra.openshift.tls_enabled=false +# Configuration of Kubernetes client that Infra will use +che.infra.kubernetes.master_url= +che.infra.kubernetes.username= +che.infra.kubernetes.password= +che.infra.kubernetes.oauth_token= +che.infra.kubernetes.trust_certs= -# Defines OpenShift namespace in which all workspaces will be created. -# If not set, every workspace will be created in a new project, where project name = workspace id -che.infra.openshift.project= +# Defines Kubernetes namespace in which all workspaces will be created. +# If not set, every workspace will be created in a new namespace, where namespace = workspace id +# +# Ignored for OpenShift infra. Use `che.infra.openshift.project` instead +che.infra.kubernetes.namespace= -che.infra.openshift.machine_start_timeout_min=5 +che.infra.kubernetes.machine_start_timeout_min=5 -che.infra.openshift.bootstrapper.binary_url=http://${CHE_HOST}:${CHE_PORT}/agent-binaries/linux_amd64/bootstrapper/bootstrapper -che.infra.openshift.bootstrapper.timeout_min=10 -che.infra.openshift.bootstrapper.installer_timeout_sec=180 -che.infra.openshift.bootstrapper.server_check_period_sec=3 +che.infra.kubernetes.bootstrapper.binary_url=http://${CHE_HOST}:${CHE_PORT}/agent-binaries/linux_amd64/bootstrapper/bootstrapper +che.infra.kubernetes.bootstrapper.timeout_min=10 +che.infra.kubernetes.bootstrapper.installer_timeout_sec=180 +che.infra.kubernetes.bootstrapper.server_check_period_sec=3 # Defines whether use the Persistent Volume Claim for che workspace needs # e.g backup projects, logs etc or disable it. -che.infra.openshift.pvc.enabled=true +che.infra.kubernetes.pvc.enabled=true # Defined which strategy will be used while choosing PVC for workspaces. # # Supported strategies: # - 'common' -# PVC with name that is configured by 'che.infra.openshift.pvc.name' +# PVC with name that is configured by 'che.infra.kubernetes.pvc.name' # property will be used for storing workspaces data of each workspace # in an OpenShift project. # Existing PVC will be used or new one will be created if it doesn't exist. # # - 'unique' -# PVC with name that is evaluated as '{che.infra.openshift.pvc.name} + '-' + {generated_8_chars}' +# PVC with name that is evaluated as '{che.infra.kubernetes.pvc.name} + '-' + {generated_8_chars}' # will be used for storing of workspaces data. # Existing PVC will be used or a new one will be created if it doesn't exist. -che.infra.openshift.pvc.strategy=common +che.infra.kubernetes.pvc.strategy=common # Defines whether to run a job that creates workspace's subpath directories in persistent volume for the 'common' strategy before launching a workspace. # Necessary in some versions of OpenShift/Kubernetes as workspace subpath volume mounts are created with root permissions, @@ -364,21 +364,21 @@ che.infra.openshift.pvc.strategy=common # The default is "true", but should be set to false if the version of Openshift/Kubernetes creates subdirectories with user permissions. # Relevant issue: https://github.com/kubernetes/kubernetes/issues/41638 # Note that this property has effect only if the 'common' PVC strategy used. -che.infra.openshift.pvc.precreate_subpaths=true +che.infra.kubernetes.pvc.precreate_subpaths=true # Defines the name of Persistent Volume Claim for che workspaces. -che.infra.openshift.pvc.name=claim-che-workspace +che.infra.kubernetes.pvc.name=claim-che-workspace # Defines the size of Persistent Volume Claim of che workspace. # Format described here: # https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources -che.infra.openshift.pvc.quantity=10Gi +che.infra.kubernetes.pvc.quantity=10Gi # Pod that is launched when performing persistent volume claim maintenance jobs on OpenShift -che.infra.openshift.pvc.jobs.image=centos:centos7 +che.infra.kubernetes.pvc.jobs.image=centos:centos7 # Defines pod memory limit for persistent volume claim maintenance jobs -che.infra.openshift.pvc.jobs.memorylimit=250Mi +che.infra.kubernetes.pvc.jobs.memorylimit=250Mi # Defines Persistent Volume Claim access mode. # Note that for common PVC strategy changing of access mode affects the number of simultaneously running workspaces. @@ -386,14 +386,29 @@ che.infra.openshift.pvc.jobs.memorylimit=250Mi # bounded only by che limits configuration like(RAM, CPU etc). # Detailed information about access mode is described here: # https://docs.openshift.com/container-platform/latest/architecture/additional_concepts/storage.html#pv-access-modes -che.infra.openshift.pvc.access_mode=ReadWriteOnce +che.infra.kubernetes.pvc.access_mode=ReadWriteOnce # Defined range of ports for installers servers # # By default, installer will use own port, but if it conflicts with another installer servers # then OpenShift infrastructure will reconfigure installer to use first available from this range -che.infra.openshift.installer_server_min_port=10000 -che.infra.openshift.installer_server_max_port=20000 +che.infra.kubernetes.installer_server_min_port=10000 +che.infra.kubernetes.installer_server_max_port=20000 + + +### OpenShift Infra parameters +# +# Since OpenShift infrastructure reuse Kubernetes infrastructure components +# OpenShift infrastructure reuse most of the Kubernetes configuration attributes. +# + +# Defines OpenShift namespace in which all workspaces will be created. +# If not set, every workspace will be created in a new project, where project name = workspace id +che.infra.openshift.project= + +# Create routes with Transport Layer Security (TLS) enabled +che.infra.openshift.tls_enabled=false + # Single port mode wildcard domain host & port. nip.io is used by default che.singleport.wildcard_domain.host=NULL diff --git a/dockerfiles/init/manifests/che.env b/dockerfiles/init/manifests/che.env index 25a71b27db9..6968edbdac7 100644 --- a/dockerfiles/init/manifests/che.env +++ b/dockerfiles/init/manifests/che.env @@ -436,59 +436,81 @@ CHE_SINGLE_PORT=false ######################################################################################## ##### ##### -##### Openshift Infrastructure ##### +##### Kubernetes Infrastructure ##### ##### ##### # -#CHE_INFRA_OPENSHIFT_MASTER__URL= -#CHE_INFRA_OPENSHIFT_USERNAME= -#CHE_INFRA_OPENSHIFT_PASSWORD= -#CHE_INFRA_OPENSHIFT_OAUTH__TOKEN= -#CHE_INFRA_OPENSHIFT_TRUST__CERTS= +#Configuration of Kubernetes client that Infra will use +#CHE_INFRA_KUBERNETES_MASTER__URL= +#CHE_INFRA_KUBERNETES_USERNAME= +#CHE_INFRA_KUBERNETES_PASSWORD= +#CHE_INFRA_KUBERNETES_OAUTH__TOKEN= +#CHE_INFRA_KUBERNETES_TRUST__CERTS= + +# Defines Kubernetes namespace in which all workspaces will be created. +# If not set, every workspace will be created in a new namespace, where namespace = workspace id +# +# Ignored for OpenShift infra. Use `CHE_INFRA_OPENSHIFT_PROJECT` instead +#CHE_INFRA_KUBERNETES_NAMESPACE= -#CHE_INFRA_OPENSHIFT_MACHINE__START__TIMEOUT__MIN=5 +#CHE_INFRA_KUBERNETES_MACHINE__START__TIMEOUT__MIN=5 # Defines whether use the Persistent Volume Claim for che workspace needs # e.g backup projects, logs etc or disable it. -#CHE_INFRA_OPENSHIFT_PVC_ENABLED=true +#CHE_INFRA_KUBERNETES_PVC_ENABLED=true # Defined which strategy will be used while choosing PVC for workspaces. # # Supported strategies: # - 'common' -# PVC with name that is configured by 'che.infra.openshift.pvc.name' +# PVC with name that is configured by 'che.infra.kubernetes.pvc.name' # the property will be used for storing workspaces data of each workspace # in an OpenShift project. # Existing PVC will be used or new one will be created if it doesn't exist. # # - 'unique' -# PVC with name that is evaluated as '{che.infra.openshift.pvc.name} + '-' + {generated_8_chars}' +# PVC with name that is evaluated as '{che.infra.kubernetes.pvc.name} + '-' + {generated_8_chars}' # will be used for storing of workspaces data. # Existing PVC will be used or a new one will be created if it doesn't exist. -#CHE_INFRA_OPENSHIFT_PVC_STRATEGY=common +#CHE_INFRA_KUBERNETES_PVC_STRATEGY=common # Defines whether to run a job that creates workspace's subpath directories in persistent volume for the 'common' strategy before launching a workspace. # Necessary in some versions of OpenShift/Kubernetes as workspace subpath volume mounts are created with root permissions, # and thus cannot be modified by workspaces running as a user (presents an error importing projects into a workspace in Che). -# The default is "true", but should be set to false if the version of Openshift/Kubernetes creates subdirectories with user permissions. +# The default is "true", but should be set to false if the version of Kubernetes/Kubernetes creates subdirectories with user permissions. # Relevant issue: https://github.com/kubernetes/kubernetes/issues/41638 # Note that this property has effect only if the 'common' PVC strategy used. -#CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS=true +#CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS=true # Defines the name of Persistent Volume Claim for che workspace. -#CHE_INFRA_OPENSHIFT_PVC_NAME=claim-che-workspace +#CHE_INFRA_KUBERNETES_PVC_NAME=claim-che-workspace # Defines the size of Persistent Volume Claim of che workspace. # Format described here: -# https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources -#CHE_INFRA_OPENSHIFT_PVC_QUANTITY=10Gi +# https://docs.kubernetes.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources +#CHE_INFRA_KUBERNETES_PVC_QUANTITY=10Gi # Defines Persistent Volume Claim access mode. # Note that for common PVC strategy changing of access mode affects the number of simultaneously running workspaces. # If OpenShift flavor where che running is using PVs with RWX access mode then a limit of running workspaces at the same time # bounded only by che limits configuration like(RAM, CPU etc). # Detailed information about access mode is described here: -# https://docs.openshift.com/container-platform/latest/architecture/additional_concepts/storage.html#pv-access-modes -#CHE_INFRA_OPENSHIFT_PVC_ACCESS__MODE=ReadWriteOnce +# https://docs.kubernetes.com/container-platform/latest/architecture/additional_concepts/storage.html#pv-access-modes +#CHE_INFRA_KUBERNETES_PVC_ACCESS__MODE=ReadWriteOnce + +######################################################################################## +##### ##### +##### Openshift Infrastructure ##### +##### ##### +# +# Since OpenShift infrastructure reuse Kubernetes infrastructure components +# OpenShift infrastructure reuse most of the Kubernetes configuration attributes. + +# Defines OpenShift namespace in which all workspaces will be created. +# If not set, every workspace will be created in a new project, where project name = workspace id +#CHE_INFRA_OPENSHIFT_PROJECT= + +# Create routes with Transport Layer Security (TLS) enabled +CHE_INFRA_OPENSHIFT_TLS_ENABLED=false ######################################################################################## diff --git a/dockerfiles/init/modules/openshift/files/scripts/che-config b/dockerfiles/init/modules/openshift/files/scripts/che-config index 713d8963bbc..d5b23f80a2a 100644 --- a/dockerfiles/init/modules/openshift/files/scripts/che-config +++ b/dockerfiles/init/modules/openshift/files/scripts/che-config @@ -6,17 +6,17 @@ CHE_API: ${HTTP_PROTOCOL}://che-${OPENSHIFT_NAMESPACE_URL}/api CHE_WEBSOCKET_ENDPOINT: ${WS_PROTOCOL}://che-${OPENSHIFT_NAMESPACE_URL}/api/websocket CHE_DEBUG_SERVER: ${CHE_DEBUG_SERVER} CHE_INFRASTRUCTURE_ACTIVE: openshift -CHE_INFRA_OPENSHIFT_BOOTSTRAPPER_BINARY__URL: ${HTTP_PROTOCOL}://che-${OPENSHIFT_NAMESPACE_URL}/agent-binaries/linux_amd64/bootstrapper/bootstrapper -CHE_INFRA_OPENSHIFT_MACHINE__START__TIMEOUT__MIN: "5" -CHE_INFRA_OPENSHIFT_MASTER__URL: ${CHE_INFRA_OPENSHIFT_MASTER__URL} -CHE_INFRA_OPENSHIFT_OAUTH__TOKEN: ${CHE_INFRA_OPENSHIFT_OAUTH__TOKEN} -CHE_INFRA_OPENSHIFT_PASSWORD: ${CHE_INFRA_OPENSHIFT_PASSWORD} +CHE_INFRA_KUBERNETES_BOOTSTRAPPER_BINARY__URL: ${HTTP_PROTOCOL}://che-${OPENSHIFT_NAMESPACE_URL}/agent-binaries/linux_amd64/bootstrapper/bootstrapper +CHE_INFRA_KUBERNETES_MACHINE__START__TIMEOUT__MIN: "5" +CHE_INFRA_KUBERNETES_MASTER__URL: ${CHE_INFRA_KUBERNETES_MASTER__URL} +CHE_INFRA_KUBERNETES_OAUTH__TOKEN: ${CHE_INFRA_KUBERNETES_OAUTH__TOKEN} +CHE_INFRA_KUBERNETES_PASSWORD: ${CHE_INFRA_KUBERNETES_PASSWORD} CHE_INFRA_OPENSHIFT_PROJECT: ${CHE_INFRA_OPENSHIFT_PROJECT} -CHE_INFRA_OPENSHIFT_PVC_STRATEGY: ${CHE_INFRA_OPENSHIFT_PVC_STRATEGY} -CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS: ${CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS} +CHE_INFRA_KUBERNETES_PVC_STRATEGY: ${CHE_INFRA_KUBERNETES_PVC_STRATEGY} +CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS: ${CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS} CHE_INFRA_OPENSHIFT_TLS__ENABLED: ${ENABLE_SSL} -CHE_INFRA_OPENSHIFT_TRUST__CERTS: "false" -CHE_INFRA_OPENSHIFT_USERNAME: ${CHE_INFRA_OPENSHIFT_USERNAME} +CHE_INFRA_KUBERNETES_TRUST__CERTS: "false" +CHE_INFRA_KUBERNETES_USERNAME: ${CHE_INFRA_KUBERNETES_USERNAME} CHE_KEYCLOAK_AUTH__SERVER__URL: ${CHE_KEYCLOAK_AUTH__SERVER__URL} CHE_KEYCLOAK_CLIENT__ID: ${CHE_KEYCLOAK_CLIENT__ID} CHE_KEYCLOAK_GITHUB_ENDPOINT: ${KEYCLOAK_GITHUB_ENDPOINT} diff --git a/dockerfiles/init/modules/openshift/files/scripts/deploy_che.sh b/dockerfiles/init/modules/openshift/files/scripts/deploy_che.sh index 5e053f2ea57..2e9dc801c55 100755 --- a/dockerfiles/init/modules/openshift/files/scripts/deploy_che.sh +++ b/dockerfiles/init/modules/openshift/files/scripts/deploy_che.sh @@ -168,7 +168,7 @@ if [ "${OPENSHIFT_FLAVOR}" == "minishift" ]; then # Set minishift configuration # ---------------------- DEFAULT_CHE_INFRA_OPENSHIFT_PROJECT="" - DEFAULT_CHE_INFRA_OPENSHIFT_PVC_STRATEGY="unique" + DEFAULT_CHE_INFRA_KUBERNETES_PVC_STRATEGY="unique" DEFAULT_OPENSHIFT_ENDPOINT="https://${MINISHIFT_IP}:8443/" DEFAULT_OPENSHIFT_USERNAME="developer" DEFAULT_OPENSHIFT_PASSWORD="developer" @@ -181,14 +181,14 @@ if [ "${OPENSHIFT_FLAVOR}" == "minishift" ]; then DEFAULT_ENABLE_SSL="false" DEFAULT_CHE_LOG_LEVEL="INFO" DEFAULT_CHE_PREDEFINED_STACKS_RELOAD="false" - DEFAULT_CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS="true" + DEFAULT_CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS="true" elif [ "${OPENSHIFT_FLAVOR}" == "osio" ]; then # ---------------------- # Set osio configuration # ---------------------- if [ -z "${OPENSHIFT_TOKEN+x}" ]; then echo "[CHE] **ERROR** Env var OPENSHIFT_TOKEN is unset. You need to set it with your OSO token to continue. To retrieve your token: https://console.starter-us-east-2.openshift.com/console/command-line. Aborting"; exit 1; fi DEFAULT_CHE_INFRA_OPENSHIFT_PROJECT=${CHE_OPENSHIFT_PROJECT:-${DEFAULT_CHE_OPENSHIFT_PROJECT}} - DEFAULT_CHE_INFRA_OPENSHIFT_PVC_STRATEGY="common" + DEFAULT_CHE_INFRA_KUBERNETES_PVC_STRATEGY="common" DEFAULT_OPENSHIFT_ENDPOINT="https://api.starter-us-east-2.openshift.com" DEFAULT_CHE_OPENSHIFT_PROJECT="$(oc get projects -o=custom-columns=NAME:.metadata.name --no-headers | grep "\\-che$")" DEFAULT_OPENSHIFT_ROUTING_SUFFIX="8a09.starter-us-east-2.openshiftapps.com" @@ -197,13 +197,13 @@ elif [ "${OPENSHIFT_FLAVOR}" == "osio" ]; then DEFAULT_ENABLE_SSL="true" DEFAULT_CHE_LOG_LEVEL="INFO" DEFAULT_CHE_PREDEFINED_STACKS_RELOAD="true" - DEFAULT_CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS="false" + DEFAULT_CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS="false" elif [ "${OPENSHIFT_FLAVOR}" == "ocp" ]; then # ---------------------- # Set ocp configuration # ---------------------- DEFAULT_CHE_INFRA_OPENSHIFT_PROJECT="" - DEFAULT_CHE_INFRA_OPENSHIFT_PVC_STRATEGY="unique" + DEFAULT_CHE_INFRA_KUBERNETES_PVC_STRATEGY="unique" DEFAULT_OPENSHIFT_USERNAME="developer" DEFAULT_OPENSHIFT_PASSWORD="developer" DEFAULT_CHE_OPENSHIFT_PROJECT="eclipse-che" @@ -212,7 +212,7 @@ elif [ "${OPENSHIFT_FLAVOR}" == "ocp" ]; then DEFAULT_ENABLE_SSL="true" DEFAULT_CHE_LOG_LEVEL="INFO" DEFAULT_CHE_PREDEFINED_STACKS_RELOAD="true" - DEFAULT_CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS="true" + DEFAULT_CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS="true" fi # -------------------------------------------------------- @@ -244,10 +244,10 @@ fi CHE_OAUTH_GITHUB_CLIENTID=${CHE_OAUTH_GITHUB_CLIENTID:-} CHE_OAUTH_GITHUB_CLIENTSECRET=${CHE_OAUTH_GITHUB_CLIENTSECRET:-} CHE_INFRA_OPENSHIFT_PROJECT=${CHE_INFRA_OPENSHIFT_PROJECT:-${DEFAULT_CHE_INFRA_OPENSHIFT_PROJECT}} -if [ -z ${CHE_INFRA_OPENSHIFT_USERNAME+x} ]; then CHE_INFRA_OPENSHIFT_USERNAME=$OPENSHIFT_USERNAME; fi -if [ -z ${CHE_INFRA_OPENSHIFT_PASSWORD+x} ]; then CHE_INFRA_OPENSHIFT_PASSWORD=$OPENSHIFT_PASSWORD; fi -if [ -z ${CHE_INFRA_OPENSHIFT_OAUTH__TOKEN+x} ]; then CHE_INFRA_OPENSHIFT_OAUTH__TOKEN=$OPENSHIFT_TOKEN; fi -CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS=${CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS:-${DEFAULT_CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS}} +if [ -z ${CHE_INFRA_KUBERNETES_USERNAME+x} ]; then CHE_INFRA_KUBERNETES_USERNAME=$OPENSHIFT_USERNAME; fi +if [ -z ${CHE_INFRA_KUBERNETES_PASSWORD+x} ]; then CHE_INFRA_KUBERNETES_PASSWORD=$OPENSHIFT_PASSWORD; fi +if [ -z ${CHE_INFRA_KUBERNETES_OAUTH__TOKEN+x} ]; then CHE_INFRA_KUBERNETES_OAUTH__TOKEN=$OPENSHIFT_TOKEN; fi +CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS=${CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS:-${DEFAULT_CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS}} OPENSHIFT_ROUTING_SUFFIX=${OPENSHIFT_ROUTING_SUFFIX:-${DEFAULT_OPENSHIFT_ROUTING_SUFFIX}} DEFAULT_OPENSHIFT_NAMESPACE_URL="${CHE_OPENSHIFT_PROJECT}.${OPENSHIFT_ROUTING_SUFFIX}" @@ -261,7 +261,7 @@ CHE_DEBUG_SERVER=${CHE_DEBUG_SERVER:-${DEFAULT_CHE_DEBUG_SERVER}} OC_SKIP_TLS=${OC_SKIP_TLS:-${DEFAULT_OC_SKIP_TLS}} CHE_APPLY_RESOURCE_QUOTAS=${CHE_APPLY_RESOURCE_QUOTAS:-${DEFAULT_CHE_APPLY_RESOURCE_QUOTAS}} IMAGE_PULL_POLICY=${IMAGE_PULL_POLICY:-${DEFAULT_IMAGE_PULL_POLICY}} -CHE_INFRA_OPENSHIFT_PVC_STRATEGY=${CHE_INFRA_OPENSHIFT_PVC_STRATEGY:-${DEFAULT_CHE_INFRA_OPENSHIFT_PVC_STRATEGY}} +CHE_INFRA_KUBERNETES_PVC_STRATEGY=${CHE_INFRA_KUBERNETES_PVC_STRATEGY:-${DEFAULT_CHE_INFRA_KUBERNETES_PVC_STRATEGY}} CHE_HOST="${OPENSHIFT_NAMESPACE_URL}" if [ "${ENABLE_SSL}" == "true" ]; then HTTP_PROTOCOL="https" diff --git a/dockerfiles/init/modules/openshift/files/scripts/ocp.sh b/dockerfiles/init/modules/openshift/files/scripts/ocp.sh index eb21eb4c782..fd821d42446 100755 --- a/dockerfiles/init/modules/openshift/files/scripts/ocp.sh +++ b/dockerfiles/init/modules/openshift/files/scripts/ocp.sh @@ -61,7 +61,7 @@ export OPENSHIFT_FLAVOR="ocp" DEFAULT_OPENSHIFT_ENDPOINT="https://${OC_PUBLIC_HOSTNAME}:8443" export OPENSHIFT_ENDPOINT=${OPENSHIFT_ENDPOINT:-${DEFAULT_OPENSHIFT_ENDPOINT}} -export CHE_INFRA_OPENSHIFT_MASTER__URL=${CHE_INFRA_OPENSHIFT_MASTER__URL:-${OPENSHIFT_ENDPOINT}} +export CHE_INFRA_KUBERNETES_MASTER__URL=${CHE_INFRA_KUBERNETES_MASTER__URL:-${OPENSHIFT_ENDPOINT}} DEFAULT_ENABLE_SSL="false" export ENABLE_SSL=${ENABLE_SSL:-${DEFAULT_ENABLE_SSL}} diff --git a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/environment/convert/DockerfileEnvironmentConverterTest.java b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/environment/convert/DockerfileEnvironmentConverterTest.java index 0ab7d620e93..17c5ae87d3a 100644 --- a/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/environment/convert/DockerfileEnvironmentConverterTest.java +++ b/infrastructures/docker/infrastructure/src/test/java/org/eclipse/che/workspace/infrastructure/docker/environment/convert/DockerfileEnvironmentConverterTest.java @@ -10,7 +10,7 @@ */ package org.eclipse.che.workspace.infrastructure.docker.environment.convert; -import static com.google.common.collect.Maps.*; +import static com.google.common.collect.Maps.newLinkedHashMap; import static java.util.Collections.singletonMap; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/infrastructures/kubernetes/pom.xml b/infrastructures/kubernetes/pom.xml new file mode 100644 index 00000000000..160a02e2808 --- /dev/null +++ b/infrastructures/kubernetes/pom.xml @@ -0,0 +1,135 @@ + + + + 4.0.0 + + che-infrastructures-parent + org.eclipse.che.infrastructure + 6.1.0-SNAPSHOT + ../pom.xml + + infrastructure-kubernetes + Infrastructure :: Kubernetes + + + com.google.code.gson + gson + + + com.google.guava + guava + + + com.google.inject + guice + + + com.google.inject.extensions + guice-assistedinject + + + com.google.inject.extensions + guice-multibindings + + + com.squareup.okhttp3 + okhttp + + + io.fabric8 + kubernetes-client + + + io.fabric8 + kubernetes-model + + + javax.annotation + javax.annotation-api + + + javax.inject + javax.inject + + + org.eclipse.che.core + che-core-api-core + + + org.eclipse.che.core + che-core-api-dto + + + org.eclipse.che.core + che-core-api-installer + + + org.eclipse.che.core + che-core-api-installer-shared + + + org.eclipse.che.core + che-core-api-model + + + org.eclipse.che.core + che-core-api-workspace + + + org.eclipse.che.core + che-core-api-workspace-shared + + + org.eclipse.che.core + che-core-commons-annotations + + + org.eclipse.che.core + che-core-commons-lang + + + org.eclipse.che.infrastructure.docker + docker-environment + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + test + + + org.hamcrest + hamcrest-core + test + + + org.mockito + mockito-core + test + + + org.mockitong + mockitong + test + + + org.testng + testng + test + + + diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/Annotations.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/Annotations.java similarity index 92% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/Annotations.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/Annotations.java index 22c6a8e2867..6acab45f29f 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/Annotations.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/Annotations.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift; +package org.eclipse.che.workspace.infrastructure.kubernetes; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; @@ -23,7 +23,7 @@ /** * Helps to convert servers related entities (like {@link ServerConfig} and machine name) to - * OpenShift annotations and vise-versa. + * Kubernetes annotations and vise-versa. * * @author Sergii Leshchenko */ @@ -58,12 +58,12 @@ public static Deserializer newDeserializer(Map annotations) { return new Deserializer(annotations); } - /** Helps to serialize ServerConfig entities to OpenShift annotations. */ + /** Helps to serialize ServerConfig entities to Kubernetes annotations. */ public static class Serializer { private final Map annotations = new HashMap<>(); /** - * Serializes server configuration as OpenShift annotations. Appends serialization result to + * Serializes server configuration as Kubernetes annotations. Appends serialization result to * this aggregate. * * @param ref server reference e.g. "exec-agent" @@ -98,7 +98,7 @@ public Map annotations() { } } - /** Helps to deserialize OpenShift annotations to known {@link ServerConfig} related entities. */ + /** Helps to deserialize Kuberbetes annotations to known {@link ServerConfig} related entities. */ public static class Deserializer { private final Map annotations; diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/Constants.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/Constants.java similarity index 80% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/Constants.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/Constants.java index 842ee23dd2b..f0b568417e9 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/Constants.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/Constants.java @@ -8,10 +8,10 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift; +package org.eclipse.che.workspace.infrastructure.kubernetes; /** - * Constants for OpenShift implementation of spi. + * Constants for Kubernetes implementation of spi. * * @author Sergii Leshchenko */ @@ -20,10 +20,10 @@ public final class Constants { /** * The label that contains a value with original object name. * - *

Names of OpenShift objects should be modified to avoid collision with objects of other + *

Names of Kubernetes objects should be modified to avoid collision with objects of other * workspaces. * - * @see org.eclipse.che.workspace.infrastructure.openshift.provision.UniqueNamesProvisioner + * @see org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner */ public static final String CHE_ORIGINAL_NAME_LABEL = "che.original_name"; diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesClientFactory.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesClientFactory.java new file mode 100644 index 00000000000..2d2fb632838 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesClientFactory.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import static com.google.common.base.Strings.isNullOrEmpty; + +import io.fabric8.kubernetes.client.Config; +import io.fabric8.kubernetes.client.ConfigBuilder; +import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClient; +import javax.annotation.PreDestroy; +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.commons.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Sergii Leshchenko + * @author Anton Korneta + */ +@Singleton +public class KubernetesClientFactory { + + private static final Logger LOG = LoggerFactory.getLogger(KubernetesClientFactory.class); + + private final UnclosableKubernetesClient client; + + @Inject + public KubernetesClientFactory( + @Nullable @Named("che.infra.kubernetes.master_url") String masterUrl, + @Nullable @Named("che.infra.kubernetes.username") String username, + @Nullable @Named("che.infra.kubernetes.password") String password, + @Nullable @Named("che.infra.kubernetes.oauth_token") String oauthToken, + @Nullable @Named("che.infra.kubernetes.trust_certs") Boolean doTrustCerts) { + ConfigBuilder configBuilder = new ConfigBuilder(); + if (!isNullOrEmpty(masterUrl)) { + configBuilder.withMasterUrl(masterUrl); + } + + if (!isNullOrEmpty(username)) { + configBuilder.withUsername(username); + } + + if (!isNullOrEmpty(password)) { + configBuilder.withPassword(password); + } + + if (!isNullOrEmpty(oauthToken)) { + configBuilder.withOauthToken(oauthToken); + } + + if (doTrustCerts != null) { + configBuilder.withTrustCerts(doTrustCerts); + } + this.client = new UnclosableKubernetesClient(configBuilder.build()); + } + + /** + * Creates instance of {@link KubernetesClient}. + * + * @throws InfrastructureException if any error occurs on client instance creation. + */ + public KubernetesClient create() throws InfrastructureException { + return client; + } + + @PreDestroy + private void cleanup() { + try { + client.doClose(); + } catch (RuntimeException ex) { + LOG.error(ex.getMessage()); + } + } + + /** + * Decorates the {@link DefaultKubernetesClient} so that it can not be closed from the outside. + */ + private static class UnclosableKubernetesClient extends DefaultKubernetesClient { + + public UnclosableKubernetesClient(Config config) { + super(config); + } + + @Override + public void close() {} + + void doClose() { + super.close(); + } + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java new file mode 100644 index 00000000000..b12d250d8ca --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.InstallerServersPortProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.RamLimitProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.restartpolicy.RestartPolicyRewriter; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.server.ServersConverter; + +/** + * Applies the set of configurations to the Kubernetes environment and environment configuration + * with the desired order, which corresponds to the needs of the Kubernetes infrastructure. + * + * @author Anton Korneta + * @author Alexander Garagatyi + */ +@Singleton +public class KubernetesEnvironmentProvisioner { + + private final boolean pvcEnabled; + private final WorkspaceVolumesStrategy volumesStrategy; + private final UniqueNamesProvisioner uniqueNamesProvisioner; + private final ServersConverter serversConverter; + private final EnvVarsConverter envVarsConverter; + private final RestartPolicyRewriter restartPolicyRewriter; + private final RamLimitProvisioner ramLimitProvisioner; + private final InstallerServersPortProvisioner installerServersPortProvisioner; + private final LogsVolumeMachineProvisioner logsVolumeMachineProvisioner; + + @Inject + public KubernetesEnvironmentProvisioner( + @Named("che.infra.kubernetes.pvc.enabled") boolean pvcEnabled, + UniqueNamesProvisioner uniqueNamesProvisioner, + ServersConverter serversConverter, + EnvVarsConverter envVarsConverter, + RestartPolicyRewriter restartPolicyRewriter, + WorkspaceVolumesStrategy volumesStrategy, + RamLimitProvisioner ramLimitProvisioner, + InstallerServersPortProvisioner installerServersPortProvisioner, + LogsVolumeMachineProvisioner logsVolumeMachineProvisioner) { + this.pvcEnabled = pvcEnabled; + this.volumesStrategy = volumesStrategy; + this.uniqueNamesProvisioner = uniqueNamesProvisioner; + this.serversConverter = serversConverter; + this.envVarsConverter = envVarsConverter; + this.restartPolicyRewriter = restartPolicyRewriter; + this.ramLimitProvisioner = ramLimitProvisioner; + this.installerServersPortProvisioner = installerServersPortProvisioner; + this.logsVolumeMachineProvisioner = logsVolumeMachineProvisioner; + } + + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) + throws InfrastructureException { + // 1 stage - update environment according Infrastructure specific + installerServersPortProvisioner.provision(k8sEnv, identity); + if (pvcEnabled) { + logsVolumeMachineProvisioner.provision(k8sEnv, identity); + } + + // 2 stage - converting Che model env to Kubernetes env + serversConverter.provision(k8sEnv, identity); + envVarsConverter.provision(k8sEnv, identity); + if (pvcEnabled) { + volumesStrategy.provision(k8sEnv, identity); + } + + // 3 stage - add Kubernetes env items + restartPolicyRewriter.provision(k8sEnv, identity); + uniqueNamesProvisioner.provision(k8sEnv, identity); + ramLimitProvisioner.provision(k8sEnv, identity); + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java new file mode 100644 index 00000000000..7a422e1d7b6 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy.COMMON_STRATEGY; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy.UNIQUE_STRATEGY; + +import com.google.inject.AbstractModule; +import com.google.inject.assistedinject.FactoryModuleBuilder; +import com.google.inject.multibindings.MapBinder; +import com.google.inject.multibindings.Multibinder; +import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure; +import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironmentFactory; +import org.eclipse.che.api.workspace.server.spi.provision.env.CheApiEnvVarProvider; +import org.eclipse.che.api.workspace.server.spi.provision.env.EnvVarProvider; +import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironment; +import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironmentFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironmentFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.RemoveNamespaceOnWorkspaceRemove; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspacePVCCleaner; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumeStrategyProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiEnvVarProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.LogsRootEnvVariableProvider; + +/** @author Sergii Leshchenko */ +public class KubernetesInfraModule extends AbstractModule { + @Override + protected void configure() { + MapBinder factories = + MapBinder.newMapBinder(binder(), String.class, InternalEnvironmentFactory.class); + + factories.addBinding(KubernetesEnvironment.TYPE).to(KubernetesEnvironmentFactory.class); + factories.addBinding(DockerImageEnvironment.TYPE).to(DockerImageEnvironmentFactory.class); + + bind(RuntimeInfrastructure.class).to(KubernetesInfrastructure.class); + + install(new FactoryModuleBuilder().build(KubernetesRuntimeContextFactory.class)); + + install(new FactoryModuleBuilder().build(KubernetesRuntimeFactory.class)); + install(new FactoryModuleBuilder().build(KubernetesBootstrapperFactory.class)); + bind(WorkspacePVCCleaner.class).asEagerSingleton(); + bind(RemoveNamespaceOnWorkspaceRemove.class).asEagerSingleton(); + + bind(CheApiEnvVarProvider.class).to(KubernetesCheApiEnvVarProvider.class); + + MapBinder volumesStrategies = + MapBinder.newMapBinder(binder(), String.class, WorkspaceVolumesStrategy.class); + volumesStrategies.addBinding(COMMON_STRATEGY).to(CommonPVCStrategy.class); + volumesStrategies.addBinding(UNIQUE_STRATEGY).to(UniqueWorkspacePVCStrategy.class); + bind(WorkspaceVolumesStrategy.class).toProvider(WorkspaceVolumeStrategyProvider.class); + + Multibinder envVarProviders = + Multibinder.newSetBinder(binder(), EnvVarProvider.class); + envVarProviders.addBinding().to(LogsRootEnvVariableProvider.class); + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfrastructure.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfrastructure.java new file mode 100644 index 00000000000..e0d552bfaf9 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfrastructure.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import static java.lang.String.format; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import javax.inject.Inject; +import javax.inject.Singleton; +import org.eclipse.che.api.core.ValidationException; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; +import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure; +import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironment; +import org.eclipse.che.api.workspace.server.spi.provision.InternalEnvironmentProvisioner; +import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.convert.DockerImageEnvironmentConverter; + +/** @author Sergii Leshchenko */ +@Singleton +public class KubernetesInfrastructure extends RuntimeInfrastructure { + + public static final String NAME = "kubernetes"; + + private final DockerImageEnvironmentConverter dockerImageEnvConverter; + private final KubernetesRuntimeContextFactory runtimeContextFactory; + private final KubernetesEnvironmentProvisioner k8sEnvProvisioner; + + @Inject + public KubernetesInfrastructure( + EventService eventService, + KubernetesRuntimeContextFactory runtimeContextFactory, + KubernetesEnvironmentProvisioner k8sEnvProvisioner, + Set internalEnvProvisioners, + DockerImageEnvironmentConverter dockerImageEnvConverter) { + super( + NAME, + ImmutableSet.of(KubernetesEnvironment.TYPE, DockerImageEnvironment.TYPE), + eventService, + internalEnvProvisioners); + this.runtimeContextFactory = runtimeContextFactory; + this.k8sEnvProvisioner = k8sEnvProvisioner; + this.dockerImageEnvConverter = dockerImageEnvConverter; + } + + @Override + protected KubernetesRuntimeContext internalPrepare( + RuntimeIdentity id, InternalEnvironment environment) + throws ValidationException, InfrastructureException { + final KubernetesEnvironment kubernetesEnvironment = asKubernetesEnv(environment); + + k8sEnvProvisioner.provision(kubernetesEnvironment, id); + + return runtimeContextFactory.create(kubernetesEnvironment, id, this); + } + + private KubernetesEnvironment asKubernetesEnv(InternalEnvironment source) + throws ValidationException, InfrastructureException { + if (source instanceof KubernetesEnvironment) { + return (KubernetesEnvironment) source; + } + if (source instanceof DockerImageEnvironment) { + return dockerImageEnvConverter.convert((DockerImageEnvironment) source); + } + throw new InternalInfrastructureException( + format( + "Environment type '%s' is not supported. Supported environment types: %s", + source.getRecipe().getType(), KubernetesEnvironment.TYPE)); + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java new file mode 100644 index 00000000000..7e446e37768 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java @@ -0,0 +1,443 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import static java.lang.String.format; +import static java.util.Collections.emptyMap; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.assistedinject.Assisted; +import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.client.Watcher.Action; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; +import javax.inject.Inject; +import javax.inject.Named; +import org.eclipse.che.api.core.model.workspace.Warning; +import org.eclipse.che.api.core.model.workspace.runtime.Machine; +import org.eclipse.che.api.core.model.workspace.runtime.MachineStatus; +import org.eclipse.che.api.core.model.workspace.runtime.Server; +import org.eclipse.che.api.core.model.workspace.runtime.ServerStatus; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.workspace.server.DtoConverter; +import org.eclipse.che.api.workspace.server.URLRewriter.NoOpURLRewriter; +import org.eclipse.che.api.workspace.server.hc.ServersChecker; +import org.eclipse.che.api.workspace.server.hc.ServersCheckerFactory; +import org.eclipse.che.api.workspace.server.hc.probe.ProbeResult; +import org.eclipse.che.api.workspace.server.hc.probe.ProbeResult.ProbeStatus; +import org.eclipse.che.api.workspace.server.hc.probe.ProbeScheduler; +import org.eclipse.che.api.workspace.server.hc.probe.WorkspaceProbesFactory; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; +import org.eclipse.che.api.workspace.server.spi.InternalRuntime; +import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; +import org.eclipse.che.api.workspace.shared.dto.event.MachineLogEvent; +import org.eclipse.che.api.workspace.shared.dto.event.MachineStatusEvent; +import org.eclipse.che.api.workspace.shared.dto.event.RuntimeStatusEvent; +import org.eclipse.che.api.workspace.shared.dto.event.ServerStatusEvent; +import org.eclipse.che.dto.server.DtoFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.ContainerEvent; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.ContainerEventHandler; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodActionHandler; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Sergii Leshchenko + * @author Anton Korneta + */ +public class KubernetesInternalRuntime< + T extends KubernetesRuntimeContext> + extends InternalRuntime { + + private static final Logger LOG = LoggerFactory.getLogger(KubernetesInternalRuntime.class); + + private static final String RUNTIME_STOPPED_STATE = "STOPPED"; + private static final String RUNTIME_RUNNING_STATE = "RUNNING"; + private static final String POD_FAILED_STATUS = "Failed"; + + private final EventService eventService; + private final ServersCheckerFactory serverCheckerFactory; + private final KubernetesBootstrapperFactory bootstrapperFactory; + private final int machineStartTimeoutMin; + private final ProbeScheduler probeScheduler; + private final WorkspaceProbesFactory probesFactory; + private final KubernetesNamespace namespace; + private final WorkspaceVolumesStrategy volumesStrategy; + protected final Map machines; + + @Inject + public KubernetesInternalRuntime( + @Named("che.infra.kubernetes.machine_start_timeout_min") int machineStartTimeoutMin, + NoOpURLRewriter urlRewriter, + EventService eventService, + KubernetesBootstrapperFactory bootstrapperFactory, + ServersCheckerFactory serverCheckerFactory, + WorkspaceVolumesStrategy volumesStrategy, + ProbeScheduler probeScheduler, + WorkspaceProbesFactory probesFactory, + @Assisted T context, + @Assisted KubernetesNamespace namespace, + @Assisted List warnings) { + super(context, urlRewriter, warnings, false); + this.eventService = eventService; + this.bootstrapperFactory = bootstrapperFactory; + this.serverCheckerFactory = serverCheckerFactory; + this.volumesStrategy = volumesStrategy; + this.machineStartTimeoutMin = machineStartTimeoutMin; + this.probeScheduler = probeScheduler; + this.probesFactory = probesFactory; + this.namespace = namespace; + this.machines = new ConcurrentHashMap<>(); + } + + @Override + protected void internalStart(Map startOptions) throws InfrastructureException { + KubernetesRuntimeContext context = getContext(); + String workspaceId = context.getIdentity().getWorkspaceId(); + try { + final KubernetesEnvironment k8sEnv = context.getEnvironment(); + volumesStrategy.prepare(k8sEnv, workspaceId); + + startMachines(); + + // TODO Rework it to parallel waiting https://github.com/eclipse/che/issues/7067 + for (KubernetesMachine machine : machines.values()) { + try { + machine.waitRunning(machineStartTimeoutMin); + machine.setStatus(MachineStatus.RUNNING); + sendRunningEvent(machine.getName()); + bootstrapMachine(machine); + checkMachineServers(machine); + } catch (InfrastructureException rethrow) { + sendFailedEvent(machine.getName(), rethrow.getMessage()); + throw rethrow; + } + } + } catch (InfrastructureException | RuntimeException | InterruptedException e) { + LOG.warn( + "Failed to start Kubernetes runtime of workspace {}. Cause: {}", + workspaceId, + e.getMessage()); + boolean interrupted = Thread.interrupted() || e instanceof InterruptedException; + // Cancels workspace servers probes if any + probeScheduler.cancel(workspaceId); + try { + namespace.cleanUp(); + } catch (InfrastructureException ignored) { + } + if (interrupted) { + throw new InfrastructureException("Kubernetes environment start was interrupted"); + } + try { + throw e; + } catch (InfrastructureException rethrow) { + throw rethrow; + } catch (Exception wrap) { + throw new InternalInfrastructureException(e.getMessage(), wrap); + } + } + } + + @Override + public Map getInternalMachines() { + return ImmutableMap.copyOf(machines); + } + + @Override + protected void internalStop(Map stopOptions) throws InfrastructureException { + // Cancels workspace servers probes if any + probeScheduler.cancel(getContext().getIdentity().getWorkspaceId()); + namespace.cleanUp(); + } + + @Override + public Map getProperties() { + return emptyMap(); + } + + /** + * Create all machine related objects and start machines. + * + * @throws InfrastructureException when any error occurs while creating Kubernetes objects + */ + protected void startMachines() throws InfrastructureException { + KubernetesEnvironment k8sEnv = getContext().getEnvironment(); + List createdServices = new ArrayList<>(); + for (Service service : k8sEnv.getServices().values()) { + createdServices.add(namespace.services().create(service)); + } + + // needed for resolution later on, even though n routes are actually created by ingress + // /workspace{wsid}/server-{port} => service({wsid}):server-port => pod({wsid}):{port} + List readyIngresses = createAndWaitReady(k8sEnv.getIngresses().values()); + + // TODO https://github.com/eclipse/che/issues/7653 + // namespace.pods().watch(new AbnormalStopHandler()); + // namespace.pods().watchContainers(new MachineLogsPublisher()); + + final KubernetesServerResolver serverResolver = + new KubernetesServerResolver(createdServices, readyIngresses); + + doStartMachine(serverResolver); + } + + /** + * Creates Kubernetes pods and resolves servers using the specified serverResolver. + * + * @param serverResolver server resolver that provide servers by container + * @throws InfrastructureException when any error occurs while creating Kubernetes pods + */ + protected void doStartMachine(KubernetesServerResolver serverResolver) + throws InfrastructureException { + final KubernetesEnvironment environment = getContext().getEnvironment(); + final Map machineConfigs = environment.getMachines(); + for (Pod toCreate : environment.getPods().values()) { + final Pod createdPod = namespace.pods().create(toCreate); + final ObjectMeta podMetadata = createdPod.getMetadata(); + for (Container container : createdPod.getSpec().getContainers()) { + String machineName = Names.machineName(toCreate, container); + KubernetesMachine machine = + new KubernetesMachine( + machineName, + podMetadata.getName(), + container.getName(), + serverResolver.resolve(machineName), + namespace, + MachineStatus.STARTING, + machineConfigs.get(machineName).getAttributes()); + machines.put(machine.getName(), machine); + sendStartingEvent(machine.getName()); + } + } + } + + private List createAndWaitReady(Collection ingresses) + throws InfrastructureException { + List createdIngresses = new ArrayList<>(); + for (Ingress ingress : ingresses) { + createdIngresses.add(namespace.ingresses().create(ingress)); + } + + // wait for LB ip + List readyIngresses = new ArrayList<>(); + for (Ingress ingress : createdIngresses) { + Ingress actualIngress = + namespace + .ingresses() + .wait( + ingress.getMetadata().getName(), + machineStartTimeoutMin, + p -> (!p.getStatus().getLoadBalancer().getIngress().isEmpty())); + readyIngresses.add(actualIngress); + } + + return readyIngresses; + } + + /** + * Bootstraps machine. + * + * @param machine the Kubernetes machine instance to bootstrap + * @throws InfrastructureException when any error occurs while bootstrapping machine + * @throws InterruptedException when machine bootstrapping was interrupted + */ + private void bootstrapMachine(KubernetesMachine machine) + throws InfrastructureException, InterruptedException { + InternalMachineConfig machineConfig = + getContext().getEnvironment().getMachines().get(machine.getName()); + if (!machineConfig.getInstallers().isEmpty()) { + bootstrapperFactory + .create(getContext().getIdentity(), machineConfig.getInstallers(), machine) + .bootstrap(); + } + } + + /** + * Checks whether machine servers are ready. + * + * @param machine the Kubernetes machine instance + * @throws InfrastructureException when any error while server checks occur + * @throws InterruptedException when process of server check was interrupted + */ + private void checkMachineServers(KubernetesMachine machine) + throws InfrastructureException, InterruptedException { + final ServersChecker check = + serverCheckerFactory.create( + getContext().getIdentity(), machine.getName(), machine.getServers()); + check.startAsync(new ServerReadinessHandler(machine.getName())); + check.await(); + + probeScheduler.schedule( + probesFactory.getProbes( + getContext().getIdentity().getWorkspaceId(), machine.getName(), machine.getServers()), + new ServerLivenessHandler()); + } + + private class ServerReadinessHandler implements Consumer { + + private String machineName; + + ServerReadinessHandler(String machineName) { + this.machineName = machineName; + } + + @Override + public void accept(String serverRef) { + final KubernetesMachine machine = machines.get(machineName); + if (machine == null) { + // Probably machine was removed from the list during server check start due to some reason + return; + } + + machine.setServerStatus(serverRef, ServerStatus.RUNNING); + + eventService.publish( + DtoFactory.newDto(ServerStatusEvent.class) + .withIdentity(DtoConverter.asDto(getContext().getIdentity())) + .withMachineName(machineName) + .withServerName(serverRef) + .withStatus(ServerStatus.RUNNING) + .withServerUrl(machine.getServers().get(serverRef).getUrl())); + } + } + + private class ServerLivenessHandler implements Consumer { + + @Override + public void accept(ProbeResult probeResult) { + String machineName = probeResult.getMachineName(); + KubernetesMachine machine = machines.get(machineName); + if (machine == null) { + // Probably machine was removed from the list during server check start due to some reason + return; + } + String serverName = probeResult.getServerName(); + ProbeStatus probeStatus = probeResult.getStatus(); + Server server = machine.getServers().get(serverName); + ServerStatus oldServerStatus = server.getStatus(); + ServerStatus serverStatus; + + if (probeStatus == ProbeStatus.FAILED && oldServerStatus == ServerStatus.RUNNING) { + serverStatus = ServerStatus.STOPPED; + } else if (probeStatus == ProbeStatus.PASSED && (oldServerStatus != ServerStatus.RUNNING)) { + serverStatus = ServerStatus.RUNNING; + } else { + return; + } + + machine.setServerStatus(serverName, serverStatus); + sendServerStatusEvent(machineName, serverName, machine.getServers().get(serverName)); + } + } + + protected void sendStartingEvent(String machineName) { + eventService.publish( + DtoFactory.newDto(MachineStatusEvent.class) + .withIdentity(DtoConverter.asDto(getContext().getIdentity())) + .withEventType(MachineStatus.STARTING) + .withMachineName(machineName)); + } + + private void sendRunningEvent(String machineName) { + eventService.publish( + DtoFactory.newDto(MachineStatusEvent.class) + .withIdentity(DtoConverter.asDto(getContext().getIdentity())) + .withEventType(MachineStatus.RUNNING) + .withMachineName(machineName)); + } + + private void sendFailedEvent(String machineName, String message) { + eventService.publish( + DtoFactory.newDto(MachineStatusEvent.class) + .withIdentity(DtoConverter.asDto(getContext().getIdentity())) + .withEventType(MachineStatus.FAILED) + .withMachineName(machineName) + .withError(message)); + } + + private void sendRuntimeStoppedEvent(String errorMsg) { + eventService.publish( + DtoFactory.newDto(RuntimeStatusEvent.class) + .withIdentity(DtoConverter.asDto(getContext().getIdentity())) + .withStatus(RUNTIME_STOPPED_STATE) + .withPrevStatus(RUNTIME_RUNNING_STATE) + .withFailed(true) + .withError(errorMsg)); + } + + private void sendServerStatusEvent(String machineName, String serverName, Server server) { + eventService.publish( + DtoFactory.newDto(ServerStatusEvent.class) + .withIdentity(DtoConverter.asDto(getContext().getIdentity())) + .withMachineName(machineName) + .withServerName(serverName) + .withStatus(server.getStatus()) + .withServerUrl(server.getUrl())); + } + + /** Listens container's events and publish them as machine logs. */ + class MachineLogsPublisher implements ContainerEventHandler { + + @Override + public void handle(ContainerEvent event) { + final String podName = event.getPodName(); + final String containerName = event.getContainerName(); + for (Entry entry : machines.entrySet()) { + final KubernetesMachine machine = entry.getValue(); + if (machine.getPodName().equals(podName) + && machine.getContainerName().equals(containerName)) { + eventService.publish( + DtoFactory.newDto(MachineLogEvent.class) + .withMachineName(entry.getKey()) + .withRuntimeId(DtoConverter.asDto(getContext().getIdentity())) + .withText(event.getMessage()) + .withTime(event.getTime())); + return; + } + } + } + } + + /** Stops runtime if one of the pods was abnormally stopped. */ + class AbnormalStopHandler implements PodActionHandler { + + @Override + public void handle(Action action, Pod pod) { + // Cancels workspace servers probes if any + probeScheduler.cancel(getContext().getIdentity().getWorkspaceId()); + if (pod.getStatus() != null && POD_FAILED_STATUS.equals(pod.getStatus().getPhase())) { + try { + internalStop(emptyMap()); + } catch (InfrastructureException ex) { + LOG.error("Kubernetes environment stop failed cause '{}'", ex.getMessage()); + } finally { + sendRuntimeStoppedEvent( + format("Pod '%s' was abnormally stopped", pod.getMetadata().getName())); + } + } + } + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftMachine.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesMachine.java similarity index 81% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftMachine.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesMachine.java index 20f8b9ce692..4c029dde54d 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftMachine.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesMachine.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift; +package org.eclipse.che.workspace.infrastructure.kubernetes; import com.google.common.collect.ImmutableMap; import java.util.Collections; @@ -19,11 +19,11 @@ import org.eclipse.che.api.core.model.workspace.runtime.ServerStatus; import org.eclipse.che.api.workspace.server.model.impl.ServerImpl; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; /** @author Sergii Leshchenko */ -public class OpenShiftMachine implements Machine { - private static final String OPENSHIFT_POD_STATUS_RUNNING = "Running"; +public class KubernetesMachine implements Machine { + private static final String KUBERNETES_POD_STATUS_RUNNING = "Running"; // TODO Make timeout configurable private static final int EXEC_TIMEOUT_MIN = 5; @@ -32,16 +32,16 @@ public class OpenShiftMachine implements Machine { private final String containerName; private final Map attributes; private final Map ref2Server; - private final OpenShiftProject project; + private final KubernetesNamespace namespace; private MachineStatus status; - public OpenShiftMachine( + public KubernetesMachine( String machineName, String podName, String containerName, Map ref2Server, - OpenShiftProject project, + KubernetesNamespace namespace, MachineStatus status, Map attributes) { this.machineName = machineName; @@ -57,7 +57,7 @@ public OpenShiftMachine( } else { this.attributes = Collections.emptyMap(); } - this.project = project; + this.namespace = namespace; this.status = status; } @@ -92,7 +92,7 @@ public void setStatus(MachineStatus status) { this.status = status; } - void setServerStatus(String serverRef, ServerStatus status) { + public void setServerStatus(String serverRef, ServerStatus status) { ServerImpl server = ref2Server.get(serverRef); if (server == null) { throw new IllegalArgumentException( @@ -102,15 +102,15 @@ void setServerStatus(String serverRef, ServerStatus status) { } public void exec(String... command) throws InfrastructureException { - project.pods().exec(podName, containerName, EXEC_TIMEOUT_MIN, command); + namespace.pods().exec(podName, containerName, EXEC_TIMEOUT_MIN, command); } public void waitRunning(int timeoutMin) throws InfrastructureException { - project + namespace .pods() .wait( podName, timeoutMin, - p -> (OPENSHIFT_POD_STATUS_RUNNING.equals(p.getStatus().getPhase()))); + p -> (KUBERNETES_POD_STATUS_RUNNING.equals(p.getStatus().getPhase()))); } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeContext.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeContext.java new file mode 100644 index 00000000000..3b4e316f3f7 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeContext.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import com.google.inject.assistedinject.Assisted; +import java.net.URI; +import javax.inject.Inject; +import javax.inject.Named; +import org.eclipse.che.api.core.ValidationException; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; +import org.eclipse.che.api.workspace.server.spi.RuntimeContext; +import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; + +/** @author Sergii Leshchenko */ +public class KubernetesRuntimeContext extends RuntimeContext { + + private final KubernetesRuntimeFactory runtimeFactory; + private final KubernetesNamespaceFactory namespaceFactory; + private final String websocketOutputEndpoint; + + @Inject + public KubernetesRuntimeContext( + @Named("che.websocket.endpoint") String cheWebsocketEndpoint, + KubernetesNamespaceFactory namespaceFactory, + KubernetesRuntimeFactory runtimeFactory, + @Assisted T kubernetesEnvironment, + @Assisted RuntimeIdentity identity, + @Assisted RuntimeInfrastructure infrastructure) + throws ValidationException, InfrastructureException { + super(kubernetesEnvironment, identity, infrastructure); + this.namespaceFactory = namespaceFactory; + this.runtimeFactory = runtimeFactory; + this.websocketOutputEndpoint = cheWebsocketEndpoint; + } + + @Override + public URI getOutputChannel() throws InfrastructureException { + try { + return URI.create(websocketOutputEndpoint); + } catch (IllegalArgumentException ex) { + throw new InternalInfrastructureException( + "Failed to get the output channel. " + ex.getMessage()); + } + } + + @Override + public KubernetesInternalRuntime getRuntime() throws InfrastructureException { + return runtimeFactory.create( + this, + namespaceFactory.create(getIdentity().getWorkspaceId()), + getEnvironment().getWarnings()); + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeContextFactory.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeContextFactory.java new file mode 100644 index 00000000000..abe74854e94 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeContextFactory.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; + +/** @author Sergii Leshchenko */ +public interface KubernetesRuntimeContextFactory { + KubernetesRuntimeContext create( + KubernetesEnvironment kubernetesEnvironment, + RuntimeIdentity identity, + RuntimeInfrastructure infrastructure); +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeFactory.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeFactory.java new file mode 100644 index 00000000000..31dbd64d239 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeFactory.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import java.util.List; +import org.eclipse.che.api.core.model.workspace.Warning; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; + +/** @author Sergii Leshchenko */ +public interface KubernetesRuntimeFactory { + KubernetesInternalRuntime> create( + KubernetesRuntimeContext context, + KubernetesNamespace namespace, + List warnings); +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/Names.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/Names.java similarity index 86% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/Names.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/Names.java index e05a7742d40..8fa0de2992e 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/Names.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/Names.java @@ -8,11 +8,11 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift; +package org.eclipse.che.workspace.infrastructure.kubernetes; import static java.lang.String.format; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_ORIGINAL_NAME_LABEL; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.MACHINE_NAME_ANNOTATION_FMT; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_ORIGINAL_NAME_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.MACHINE_NAME_ANNOTATION_FMT; import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.Pod; @@ -20,7 +20,7 @@ import org.eclipse.che.commons.lang.NameGenerator; /** - * Helps to work with OpenShift objects names. + * Helps to work with Kubernetes objects names. * * @author Sergii Leshchenko */ @@ -28,7 +28,6 @@ public class Names { static final char WORKSPACE_ID_PREFIX_SEPARATOR = '.'; - public static final String ROUTE_NAME_PREFIX = "route"; static final int GENERATED_PART_SIZE = 8; /** diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/bootstrapper/OpenShiftBootstrapper.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java similarity index 77% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/bootstrapper/OpenShiftBootstrapper.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java index 1f22c2015ea..54d4f567110 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/bootstrapper/OpenShiftBootstrapper.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.bootstrapper; +package org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -21,17 +21,17 @@ import org.eclipse.che.api.installer.shared.model.Installer; import org.eclipse.che.api.workspace.server.bootstrap.AbstractBootstrapper; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftMachine; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesMachine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Bootstraps installers in OpenShift machine. + * Bootstraps installers in Kubernetes machine. * * @author Sergii Leshchenko */ -public class OpenShiftBootstrapper extends AbstractBootstrapper { - private static final Logger LOG = LoggerFactory.getLogger(OpenShiftBootstrapper.class); +public class KubernetesBootstrapper extends AbstractBootstrapper { + private static final Logger LOG = LoggerFactory.getLogger(KubernetesBootstrapper.class); private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); @@ -44,26 +44,26 @@ public class OpenShiftBootstrapper extends AbstractBootstrapper { private final List installers; private final int serverCheckPeriodSeconds; private final int installerTimeoutSeconds; - private final OpenShiftMachine openShiftMachine; + private final KubernetesMachine kubernetesMachine; private final String bootstrapperBinaryUrl; private final String bootstrapperLogsFolder; private final String bootstrapperLogsFile; @Inject - public OpenShiftBootstrapper( + public KubernetesBootstrapper( @Assisted RuntimeIdentity runtimeIdentity, @Assisted List installers, - @Assisted OpenShiftMachine openShiftMachine, + @Assisted KubernetesMachine kubernetesMachine, @Named("che.websocket.endpoint") String cheWebsocketEndpoint, - @Named("che.infra.openshift.bootstrapper.binary_url") String bootstrapperBinaryUrl, - @Named("che.infra.openshift.bootstrapper.timeout_min") int bootstrappingTimeoutMinutes, - @Named("che.infra.openshift.bootstrapper.installer_timeout_sec") int installerTimeoutSeconds, - @Named("che.infra.openshift.bootstrapper.server_check_period_sec") + @Named("che.infra.kubernetes.bootstrapper.binary_url") String bootstrapperBinaryUrl, + @Named("che.infra.kubernetes.bootstrapper.timeout_min") int bootstrappingTimeoutMinutes, + @Named("che.infra.kubernetes.bootstrapper.installer_timeout_sec") int installerTimeoutSeconds, + @Named("che.infra.kubernetes.bootstrapper.server_check_period_sec") int serverCheckPeriodSeconds, @Named("che.workspace.logs.root_dir") String logsRootPath, EventService eventService) { super( - openShiftMachine.getName(), + kubernetesMachine.getName(), runtimeIdentity, bootstrappingTimeoutMinutes, cheWebsocketEndpoint, @@ -74,7 +74,7 @@ public OpenShiftBootstrapper( this.installers = installers; this.serverCheckPeriodSeconds = serverCheckPeriodSeconds; this.installerTimeoutSeconds = installerTimeoutSeconds; - this.openShiftMachine = openShiftMachine; + this.kubernetesMachine = kubernetesMachine; this.bootstrapperLogsFolder = logsRootPath + "/bootstrapper"; this.bootstrapperLogsFile = bootstrapperLogsFolder + "/bootstrapper.log"; } @@ -84,13 +84,13 @@ protected void doBootstrapAsync(String installerWebsocketEndpoint, String output throws InfrastructureException { injectBootstrapper(); - openShiftMachine.exec( + kubernetesMachine.exec( "sh", "-c", BOOTSTRAPPER_DIR + BOOTSTRAPPER_FILE + " -machine-name " - + openShiftMachine.getName() + + kubernetesMachine.getName() + " -runtime-id " + String.format( "%s:%s:%s", @@ -117,17 +117,17 @@ protected void doBootstrapAsync(String installerWebsocketEndpoint, String output } private void injectBootstrapper() throws InfrastructureException { - String machineName = openShiftMachine.getName(); + String machineName = kubernetesMachine.getName(); LOG.debug( "Bootstrapping {}:{}. Creating folder for bootstrapper", runtimeIdentity, machineName); - openShiftMachine.exec("mkdir", "-p", BOOTSTRAPPER_DIR, bootstrapperLogsFolder); + kubernetesMachine.exec("mkdir", "-p", BOOTSTRAPPER_DIR, bootstrapperLogsFolder); LOG.debug("Bootstrapping {}:{}. Downloading bootstrapper binary", runtimeIdentity, machineName); - openShiftMachine.exec( + kubernetesMachine.exec( "curl", "-o", BOOTSTRAPPER_DIR + BOOTSTRAPPER_FILE, bootstrapperBinaryUrl); - openShiftMachine.exec("chmod", "+x", BOOTSTRAPPER_DIR + BOOTSTRAPPER_FILE); + kubernetesMachine.exec("chmod", "+x", BOOTSTRAPPER_DIR + BOOTSTRAPPER_FILE); LOG.debug("Bootstrapping {}:{}. Creating config file", runtimeIdentity, machineName); - openShiftMachine.exec( + kubernetesMachine.exec( "sh", "-c", "cat > " diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/bootstrapper/OpenShiftBootstrapperFactory.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapperFactory.java similarity index 71% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/bootstrapper/OpenShiftBootstrapperFactory.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapperFactory.java index 0011af6162d..95d5c2d3dd5 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/bootstrapper/OpenShiftBootstrapperFactory.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapperFactory.java @@ -8,18 +8,18 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.bootstrapper; +package org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper; import com.google.inject.assistedinject.Assisted; import java.util.List; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.installer.shared.model.Installer; -import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftMachine; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesMachine; /** @author Sergii Leshchenko */ -public interface OpenShiftBootstrapperFactory { - OpenShiftBootstrapper create( +public interface KubernetesBootstrapperFactory { + KubernetesBootstrapper create( @Assisted RuntimeIdentity runtimeIdentity, @Assisted List agents, - @Assisted OpenShiftMachine openShiftMachine); + @Assisted KubernetesMachine kubernetesMachine); } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironment.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironment.java new file mode 100644 index 00000000000..11502fe3c3d --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironment.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.environment; + +import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.eclipse.che.api.core.model.workspace.Warning; +import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironment; +import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; +import org.eclipse.che.api.workspace.server.spi.environment.InternalRecipe; + +/** + * Holds objects of Kubernetes environment. + * + * @author Sergii Leshchenko + */ +public class KubernetesEnvironment extends InternalEnvironment { + + public static final String TYPE = "kubernetes"; + + private final Map pods; + private final Map services; + private final Map ingresses; + private final Map persistentVolumeClaims; + + public KubernetesEnvironment(KubernetesEnvironment k8sEnv) { + this( + k8sEnv.getRecipe(), + k8sEnv.getMachines(), + k8sEnv.getWarnings(), + k8sEnv.getPods(), + k8sEnv.getServices(), + k8sEnv.getIngresses(), + k8sEnv.getPersistentVolumeClaims()); + } + + public static Builder builder() { + return new Builder(); + } + + protected KubernetesEnvironment( + InternalRecipe internalRecipe, + Map machines, + List warnings, + Map pods, + Map services, + Map ingresses, + Map persistentVolumeClaims) { + super(internalRecipe, machines, warnings); + this.pods = pods; + this.services = services; + this.ingresses = ingresses; + this.persistentVolumeClaims = persistentVolumeClaims; + } + + /** Returns pods that should be created when environment starts. */ + public Map getPods() { + return pods; + } + + /** Returns services that should be created when environment starts. */ + public Map getServices() { + return services; + } + + /** Returns ingresses that should be created when environment starts. */ + public Map getIngresses() { + return ingresses; + } + + /** Returns PVCs that should be created when environment starts. */ + public Map getPersistentVolumeClaims() { + return persistentVolumeClaims; + } + + public static class Builder { + protected InternalRecipe internalRecipe; + protected final Map machines = new HashMap<>(); + protected final List warnings = new ArrayList<>(); + protected final Map pods = new HashMap<>(); + protected final Map services = new HashMap<>(); + protected final Map ingresses = new HashMap<>(); + protected final Map persistentVolumeClaims = new HashMap<>(); + + protected Builder() {} + + public Builder setInternalRecipe(InternalRecipe internalRecipe) { + this.internalRecipe = internalRecipe; + return this; + } + + public Builder setMachines(Map machines) { + this.machines.putAll(machines); + return this; + } + + public Builder setWarnings(List warnings) { + this.warnings.addAll(warnings); + return this; + } + + public Builder setPods(Map pods) { + this.pods.putAll(pods); + return this; + } + + public Builder setServices(Map services) { + this.services.putAll(services); + return this; + } + + public Builder setIngresses(Map ingresses) { + this.ingresses.putAll(ingresses); + return this; + } + + public Builder setPersistentVolumeClaims(Map pvcs) { + this.persistentVolumeClaims.putAll(pvcs); + return this; + } + + public KubernetesEnvironment build() { + return new KubernetesEnvironment( + internalRecipe, machines, warnings, pods, services, ingresses, persistentVolumeClaims); + } + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentFactory.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentFactory.java new file mode 100644 index 00000000000..bb0663b9fbd --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentFactory.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.environment; + +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.lang.String.format; +import static org.eclipse.che.api.core.model.workspace.config.MachineConfig.MEMORY_LIMIT_ATTRIBUTE; + +import com.google.common.annotations.VisibleForTesting; +import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.KubernetesList; +import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.inject.Inject; +import javax.inject.Named; +import org.eclipse.che.api.core.ValidationException; +import org.eclipse.che.api.core.model.workspace.Warning; +import org.eclipse.che.api.installer.server.InstallerRegistry; +import org.eclipse.che.api.workspace.server.model.impl.WarningImpl; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironment; +import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironmentFactory; +import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; +import org.eclipse.che.api.workspace.server.spi.environment.InternalRecipe; +import org.eclipse.che.api.workspace.server.spi.environment.MachineConfigsValidator; +import org.eclipse.che.api.workspace.server.spi.environment.RecipeRetriever; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.util.Containers; + +/** + * Parses {@link InternalEnvironment} into {@link KubernetesEnvironment}. + * + * @author Sergii Leshchenko + */ +public class KubernetesEnvironmentFactory + extends InternalEnvironmentFactory { + + static final int INGRESSES_IGNORED_WARNING_CODE = 4100; + static final String INGRESSES_IGNORED_WARNING_MESSAGE = + "Ingresses specified in Kubernetes recipe are ignored. " + + "To expose ports please define servers in machine configuration."; + + static final int PVC_IGNORED_WARNING_CODE = 4101; + static final String PVC_IGNORED_WARNING_MESSAGE = + "Persistent volume claims specified in Kubernetes recipe are ignored."; + + private final KubernetesClientFactory clientFactory; + private final KubernetesEnvironmentValidator envValidator; + private final String defaultMachineMemorySizeAttribute; + + @Inject + public KubernetesEnvironmentFactory( + InstallerRegistry installerRegistry, + RecipeRetriever recipeRetriever, + MachineConfigsValidator machinesValidator, + KubernetesClientFactory clientFactory, + KubernetesEnvironmentValidator envValidator, + @Named("che.workspace.default_memory_mb") long defaultMachineMemorySizeMB) { + super(installerRegistry, recipeRetriever, machinesValidator); + this.clientFactory = clientFactory; + this.envValidator = envValidator; + this.defaultMachineMemorySizeAttribute = + String.valueOf(defaultMachineMemorySizeMB * 1024 * 1024); + } + + @Override + protected KubernetesEnvironment doCreate( + InternalRecipe recipe, + Map machines, + List sourceWarnings) + throws InfrastructureException, ValidationException { + List warnings = new ArrayList<>(); + if (sourceWarnings != null) { + warnings.addAll(sourceWarnings); + } + String content = recipe.getContent(); + String contentType = recipe.getContentType(); + checkNotNull(contentType, "Kubernetes Recipe content type should not be null"); + + switch (contentType) { + case "application/x-yaml": + case "text/yaml": + case "text/x-yaml": + break; + default: + throw new ValidationException( + "Provided environment recipe content type '" + + contentType + + "' is unsupported. Supported values are: " + + "application/x-yaml, text/yaml, text/x-yaml"); + } + + final KubernetesList list = + clientFactory.create().lists().load(new ByteArrayInputStream(content.getBytes())).get(); + + Map pods = new HashMap<>(); + Map services = new HashMap<>(); + boolean isAnyIngressPresent = false; + boolean isAnyPVCPresent = false; + for (HasMetadata object : list.getItems()) { + if (object instanceof Pod) { + Pod pod = (Pod) object; + pods.put(pod.getMetadata().getName(), pod); + } else if (object instanceof Service) { + Service service = (Service) object; + services.put(service.getMetadata().getName(), service); + } else if (object instanceof Ingress) { + isAnyIngressPresent = true; + } else if (object instanceof PersistentVolumeClaim) { + isAnyPVCPresent = true; + } else { + throw new ValidationException( + format("Found unknown object type '%s'", object.getMetadata())); + } + } + + if (isAnyIngressPresent) { + warnings.add( + new WarningImpl(INGRESSES_IGNORED_WARNING_CODE, INGRESSES_IGNORED_WARNING_MESSAGE)); + } + + if (isAnyPVCPresent) { + warnings.add(new WarningImpl(PVC_IGNORED_WARNING_CODE, PVC_IGNORED_WARNING_MESSAGE)); + } + + addRamLimitAttribute(machines, pods.values()); + + KubernetesEnvironment k8sEnv = + KubernetesEnvironment.builder() + .setInternalRecipe(recipe) + .setMachines(machines) + .setWarnings(warnings) + .setPods(pods) + .setServices(services) + .setIngresses(new HashMap<>()) + .setPersistentVolumeClaims(new HashMap<>()) + .build(); + + envValidator.validate(k8sEnv); + + return k8sEnv; + } + + @VisibleForTesting + void addRamLimitAttribute(Map machines, Collection pods) { + for (Pod pod : pods) { + for (Container container : pod.getSpec().getContainers()) { + final String machineName = Names.machineName(pod, container); + InternalMachineConfig machineConfig; + if ((machineConfig = machines.get(machineName)) == null) { + machineConfig = new InternalMachineConfig(); + machines.put(machineName, machineConfig); + } + final Map attributes = machineConfig.getAttributes(); + if (isNullOrEmpty(attributes.get(MEMORY_LIMIT_ATTRIBUTE))) { + final long ramLimit = Containers.getRamLimit(container); + if (ramLimit > 0) { + attributes.put(MEMORY_LIMIT_ATTRIBUTE, String.valueOf(ramLimit)); + } else { + attributes.put(MEMORY_LIMIT_ATTRIBUTE, defaultMachineMemorySizeAttribute); + } + } + } + } + } + + private void checkNotNull(Object object, String errorMessage) throws ValidationException { + if (object == null) { + throw new ValidationException(errorMessage); + } + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentValidator.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentValidator.java similarity index 77% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentValidator.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentValidator.java index 4275628607e..3784ca07d80 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentValidator.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentValidator.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.environment; +package org.eclipse.che.workspace.infrastructure.kubernetes.environment; import static java.lang.String.format; @@ -18,22 +18,22 @@ import java.util.HashSet; import java.util.Set; import org.eclipse.che.api.core.ValidationException; -import org.eclipse.che.workspace.infrastructure.openshift.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; /** - * Validates {@link OpenShiftEnvironment}. + * Validates {@link KubernetesEnvironment}. * * @author Sergii Leshchenko */ -class OpenShiftEnvironmentValidator { +public class KubernetesEnvironmentValidator { /** - * Validates {@link OpenShiftEnvironment}. + * Validates {@link KubernetesEnvironment}. * * @param env environment to perform validation - * @throws ValidationException if the specified {@link OpenShiftEnvironment} is invalid + * @throws ValidationException if the specified {@link KubernetesEnvironment} is invalid */ - void validate(OpenShiftEnvironment env) throws ValidationException { + public void validate(KubernetesEnvironment env) throws ValidationException { checkArgument(!env.getPods().isEmpty(), "Environment should contain at least 1 pod"); Set missingMachines = new HashSet<>(env.getMachines().keySet()); @@ -48,7 +48,7 @@ void validate(OpenShiftEnvironment env) throws ValidationException { missingMachines.isEmpty(), "Environment contains machines that are missing in recipe: %s", Joiner.on(", ").join(missingMachines)); - // TODO Implement validation OpenShift objects https://github.com/eclipse/che/issues/7381 + // TODO Implement validation Kubernetes objects https://github.com/eclipse/che/issues/7381 } private static void checkArgument(boolean expression, String error) throws ValidationException { diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/convert/DockerImageEnvironmentConverter.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/convert/DockerImageEnvironmentConverter.java similarity index 83% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/convert/DockerImageEnvironmentConverter.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/convert/DockerImageEnvironmentConverter.java index 19bc6fb082a..0f548e02bb0 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/convert/DockerImageEnvironmentConverter.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/convert/DockerImageEnvironmentConverter.java @@ -8,10 +8,10 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.environment.convert; +package org.eclipse.che.workspace.infrastructure.kubernetes.environment.convert; import static java.lang.String.format; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.MACHINE_NAME_ANNOTATION_FMT; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.MACHINE_NAME_ANNOTATION_FMT; import com.google.common.collect.ImmutableMap; import io.fabric8.kubernetes.api.model.ContainerBuilder; @@ -24,10 +24,10 @@ import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; /** - * Converts {@link DockerImageEnvironment} to {@link OpenShiftEnvironment}. + * Converts {@link DockerImageEnvironment} to {@link KubernetesEnvironment}. * * @author Sergii Leshchenko * @author Anton Korneta @@ -38,7 +38,7 @@ public class DockerImageEnvironmentConverter { static final String POD_NAME = "dockerimage"; static final String CONTAINER_NAME = "container"; - public OpenShiftEnvironment convert(DockerImageEnvironment environment) + public KubernetesEnvironment convert(DockerImageEnvironment environment) throws InfrastructureException { final Iterator iterator = environment.getMachines().keySet().iterator(); if (!iterator.hasNext()) { @@ -61,7 +61,7 @@ public OpenShiftEnvironment convert(DockerImageEnvironment environment) new ContainerBuilder().withImage(dockerImage).withName(CONTAINER_NAME).build()) .endSpec() .build(); - return OpenShiftEnvironment.builder() + return KubernetesEnvironment.builder() .setMachines(environment.getMachines()) .setInternalRecipe(environment.getRecipe()) .setWarnings(environment.getWarnings()) diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesIngresses.java similarity index 52% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesIngresses.java index fb95d13e333..55bd06ecda4 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/KubernetesIngress.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesIngresses.java @@ -8,7 +8,10 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; + +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_WORKSPACE_ID_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putLabel; import io.fabric8.kubernetes.api.model.extensions.DoneableIngress; import io.fabric8.kubernetes.api.model.extensions.Ingress; @@ -22,21 +25,29 @@ import java.util.concurrent.TimeoutException; import java.util.function.Predicate; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; + +/** + * Defines an internal API for managing {@link Ingress} instances in {@link + * KubernetesIngresses#namespace predefined namespace}. + * + * @author Sergii Leshchenko + * @author Guy Daich + */ +public class KubernetesIngresses { -/** Created by I313632 on 05/01/2018. */ -public class KubernetesIngress { private final String namespace; private final String workspaceId; - private final OpenShiftClientFactory clientFactory; + private final KubernetesClientFactory clientFactory; - KubernetesIngress(String namespace, String workspaceId, OpenShiftClientFactory clientFactory) { + KubernetesIngresses(String namespace, String workspaceId, KubernetesClientFactory clientFactory) { this.namespace = namespace; this.workspaceId = workspaceId; this.clientFactory = clientFactory; } public Ingress create(Ingress ingress) throws InfrastructureException { + putLabel(ingress, CHE_WORKSPACE_ID_LABEL, workspaceId); try { return clientFactory .create() @@ -55,7 +66,6 @@ public Ingress wait(String name, int timeoutMin, Predicate predicate) CompletableFuture future = new CompletableFuture<>(); Watch watch = null; try { - Resource ingressResource = clientFactory.create().extensions().ingresses().inNamespace(namespace).withName(name); @@ -103,75 +113,17 @@ public void onClose(KubernetesClientException cause) { } } - /** - * Creates specified route. - * - * @param routes - List of OS routes to create as an ingress - * @return created route - * @throws InfrastructureException when any exception occurs - */ - // public Ingress create(List routes) throws InfrastructureException { - // try { - // List httpIngressPaths = new ArrayList<>(); - // HashSet servers = new HashSet<>(); - // for (Route route : routes) { - // String server = route.getSpec().getPort().getTargetPort().getStrVal(); - // if (!servers.contains(server)) { - // IngressBackend ingressBackend = - // new IngressBackendBuilder() - // .withServiceName(route.getSpec().getTo().getName()) - // .withNewServicePort(server) - // .build(); - // - // String serverPath = "/" + workspaceId + "/" + server; - // - // HTTPIngressPath httpIngressPath = - // new HTTPIngressPathBuilder() - // // .withPath(route.getSpec().getPath()) - // .withPath(serverPath) - // .withBackend(ingressBackend) - // .build(); - // servers.add(server); - // httpIngressPaths.add(httpIngressPath); - // } - // } - // - // HTTPIngressRuleValue httpIngressRuleValue = - // new HTTPIngressRuleValueBuilder().withPaths(httpIngressPaths).build(); - // IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build(); - // IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build(); - // Map ingressAnontations = new HashMap<>(); - // ingressAnontations.put("ingress.kubernetes.io/rewrite-target", "/"); - // ingressAnontations.put("ingress.kubernetes.io/ssl-redirect", "false"); - // ingressAnontations.put("kubernetes.io/ingress.class", "nginx"); - // Ingress ingress = - // new IngressBuilder() - // .withSpec(ingressSpec) - // .withMetadata( - // new ObjectMetaBuilder() - // .withName(workspaceId + "-ingress") - // .withAnnotations(ingressAnontations) - // .build()) - // .build(); - // return clientFactory - // .create() - // .extensions() - // .ingresses() - // .inNamespace(namespace) - // .withName(workspaceId + "-ingress") - // .create(ingress); - // } catch (KubernetesClientException e) { - // throw new InfrastructureException(e.getMessage(), e); - // } - // } - public void delete() throws InfrastructureException { - clientFactory - .create() - .extensions() - .ingresses() - .inNamespace(namespace) - .withName(workspaceId + "-ingress") - .delete(); + try { + clientFactory + .create() + .extensions() + .ingresses() + .inNamespace(namespace) + .withLabel(CHE_WORKSPACE_ID_LABEL, workspaceId) + .delete(); + } catch (KubernetesClientException e) { + throw new InfrastructureException(e.getMessage(), e); + } } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespace.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespace.java new file mode 100644 index 00000000000..e51b4f5d013 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespace.java @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; + +import com.google.common.annotations.VisibleForTesting; +import io.fabric8.kubernetes.api.model.Namespace; +import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientException; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Defines an internal API for managing subset of objects inside {@link Namespace} instance. + * + * @author Sergii Leshchenko + */ +public class KubernetesNamespace { + + private static final Logger LOG = LoggerFactory.getLogger(KubernetesNamespace.class); + + private final String workspaceId; + + private final KubernetesPods pods; + private final KubernetesServices services; + private final KubernetesPersistentVolumeClaims pvcs; + private final KubernetesIngresses ingresses; + + @VisibleForTesting + protected KubernetesNamespace( + String workspaceId, + KubernetesPods pods, + KubernetesServices services, + KubernetesPersistentVolumeClaims pvcs, + KubernetesIngresses kubernetesIngresses) { + this.workspaceId = workspaceId; + this.pods = pods; + this.services = services; + this.pvcs = pvcs; + this.ingresses = kubernetesIngresses; + } + + public KubernetesNamespace(KubernetesClientFactory clientFactory, String name, String workspaceId) + throws InfrastructureException { + this.workspaceId = workspaceId; + this.pods = new KubernetesPods(name, workspaceId, clientFactory); + this.services = new KubernetesServices(name, workspaceId, clientFactory); + this.pvcs = new KubernetesPersistentVolumeClaims(name, clientFactory); + this.ingresses = new KubernetesIngresses(name, workspaceId, clientFactory); + final KubernetesClient client = clientFactory.create(); + doPrepare(name, client); + } + + protected void doPrepare(String name, KubernetesClient client) throws InfrastructureException { + if (get(name, client) == null) { + create(name, client); + } + } + + /** Returns object for managing {@link Pod} instances inside namespace. */ + public KubernetesPods pods() { + return pods; + } + + /** Returns object for managing {@link Service} instances inside namespace. */ + public KubernetesServices services() { + return services; + } + + /** Returns object for managing {@link PersistentVolumeClaim} instances inside namespace. */ + public KubernetesPersistentVolumeClaims persistentVolumeClaims() { + return pvcs; + } + + /** Returns object for managing {@link Ingress} instances inside namespace. */ + public KubernetesIngresses ingresses() { + return ingresses; + } + + /** Removes all object except persistent volume claims inside namespace. */ + public void cleanUp() throws InfrastructureException { + doRemove(ingresses::delete, services::delete, pods::delete); + } + + /** + * Performs all the specified operations and throw exception with composite message if errors + * occurred while any operation execution + */ + protected void doRemove(RemoveOperation... operations) throws InfrastructureException { + StringBuilder errors = new StringBuilder(); + for (RemoveOperation operation : operations) { + try { + operation.perform(); + } catch (InternalInfrastructureException e) { + LOG.warn( + "Internal infra error occurred while cleaning up the namespace for workspace with id " + + workspaceId, + e); + errors.append(" ").append(e.getMessage()); + } catch (InfrastructureException e) { + errors.append(" ").append(e.getMessage()); + } + } + + if (errors.length() > 0) { + throw new InfrastructureException( + "Error(s) occurs while cleaning up the namespace." + errors.toString()); + } + } + + private void create(String namespaceName, KubernetesClient client) + throws InfrastructureException { + try { + client + .namespaces() + .createNew() + .withNewMetadata() + .withName(namespaceName) + .endMetadata() + .done(); + } catch (KubernetesClientException e) { + throw new InfrastructureException(e.getMessage(), e); + } + } + + private Namespace get(String namespaceName, KubernetesClient client) + throws InfrastructureException { + try { + return client.namespaces().withName(namespaceName).get(); + } catch (KubernetesClientException e) { + if (e.getCode() == 403) { + // namespace is foreign or doesn't exist + return null; + } else { + throw new InfrastructureException(e.getMessage(), e); + } + } + } + + protected interface RemoveOperation { + void perform() throws InfrastructureException; + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespaceFactory.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespaceFactory.java new file mode 100644 index 00000000000..782f6827734 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespaceFactory.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; + +import static com.google.common.base.Strings.isNullOrEmpty; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import javax.inject.Named; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.commons.annotation.Nullable; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; + +/** + * Helps to create {@link KubernetesNamespace} instances. + * + * @author Anton Korneta + */ +@Singleton +public class KubernetesNamespaceFactory { + + private final String namespaceName; + private final KubernetesClientFactory clientFactory; + + @Inject + public KubernetesNamespaceFactory( + @Nullable @Named("che.infra.kubernetes.namespace") String namespaceName, + KubernetesClientFactory clientFactory) { + this.namespaceName = namespaceName; + this.clientFactory = clientFactory; + } + + public KubernetesNamespace create(String workspaceId) throws InfrastructureException { + final String namespaceName = + isNullOrEmpty(this.namespaceName) ? workspaceId : this.namespaceName; + return new KubernetesNamespace(clientFactory, namespaceName, workspaceId); + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftObjectUtil.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesObjectUtil.java similarity index 82% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftObjectUtil.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesObjectUtil.java index 0a92a3ccb30..f48d20c894d 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftObjectUtil.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesObjectUtil.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; import com.google.common.collect.ImmutableMap; import io.fabric8.kubernetes.api.model.HasMetadata; @@ -28,15 +28,15 @@ import java.util.Map; /** - * Helps to work with OpenShift objects. + * Helps to work with Kubernetes objects. * * @author Anton Korneta */ -public class OpenShiftObjectUtil { +public class KubernetesObjectUtil { private static final String STORAGE_PARAM = "storage"; - /** Adds label to target OpenShift object. */ + /** Adds label to target Kubernetes object. */ public static void putLabel(HasMetadata target, String key, String value) { ObjectMeta metadata = target.getMetadata(); @@ -52,7 +52,23 @@ public static void putLabel(HasMetadata target, String key, String value) { labels.put(key, value); } - /** Adds selector into target OpenShift service. */ + /** Adds annotation to target Kubernetes object. */ + public static void putAnnotation(HasMetadata target, String key, String value) { + ObjectMeta metadata = target.getMetadata(); + + if (metadata == null) { + target.setMetadata(metadata = new ObjectMeta()); + } + + Map annotations = metadata.getAnnotations(); + if (annotations == null) { + metadata.setAnnotations(annotations = new HashMap<>()); + } + + annotations.put(key, value); + } + + /** Adds selector into target Kubernetes service. */ public static void putSelector(Service target, String key, String value) { ServiceSpec spec = target.getSpec(); diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPersistentVolumeClaims.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPersistentVolumeClaims.java similarity index 86% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPersistentVolumeClaims.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPersistentVolumeClaims.java index ba54628a7ec..1b8444165ff 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPersistentVolumeClaims.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPersistentVolumeClaims.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; import static java.util.stream.Collectors.toSet; @@ -18,19 +18,19 @@ import java.util.List; import java.util.Set; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; /** * Defines an internal API for managing {@link PersistentVolumeClaim} instances in {@link - * OpenShiftPersistentVolumeClaims#namespace predefined namespace}. + * KubernetesPersistentVolumeClaims#namespace predefined namespace}. * * @author Sergii Leshchenko */ -public class OpenShiftPersistentVolumeClaims { +public class KubernetesPersistentVolumeClaims { private final String namespace; - private final OpenShiftClientFactory clientFactory; + private final KubernetesClientFactory clientFactory; - OpenShiftPersistentVolumeClaims(String namespace, OpenShiftClientFactory clientFactory) { + KubernetesPersistentVolumeClaims(String namespace, KubernetesClientFactory clientFactory) { this.namespace = namespace; this.clientFactory = clientFactory; } @@ -92,7 +92,7 @@ public List getByLabel(String labelName, String labelValu } /** - * Creates all PVCs which are not present in current OpenShift project. + * Creates all PVCs which are not present in current Kubernetes namespace. * * @param toCreate collection of PVCs to create * @throws InfrastructureException when any error occurs while creation diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPods.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPods.java similarity index 92% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPods.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPods.java index 03c9a1b35d2..cd0e1814008 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftPods.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesPods.java @@ -8,11 +8,11 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; import static java.util.concurrent.CompletableFuture.allOf; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_WORKSPACE_ID_LABEL; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.putLabel; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_WORKSPACE_ID_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putLabel; import io.fabric8.kubernetes.api.model.DoneablePod; import io.fabric8.kubernetes.api.model.Event; @@ -40,23 +40,23 @@ import java.util.regex.Pattern; import okhttp3.Response; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; -import org.eclipse.che.workspace.infrastructure.openshift.project.event.ContainerEvent; -import org.eclipse.che.workspace.infrastructure.openshift.project.event.ContainerEventHandler; -import org.eclipse.che.workspace.infrastructure.openshift.project.event.PodActionHandler; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.ContainerEvent; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.ContainerEventHandler; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodActionHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Defines an internal API for managing {@link Pod} instances in {@link OpenShiftPods#namespace + * Defines an internal API for managing {@link Pod} instances in {@link KubernetesPods#namespace * predefined namespace}. * * @author Sergii Leshchenko * @author Anton Korneta */ -public class OpenShiftPods { +public class KubernetesPods { - private static final Logger LOG = LoggerFactory.getLogger(OpenShiftPods.class); + private static final Logger LOG = LoggerFactory.getLogger(KubernetesPods.class); private static final String CONTAINER_NAME_GROUP = "name"; // when event is related to container `fieldPath` field contains @@ -70,14 +70,14 @@ public class OpenShiftPods { private static final String POD_OBJECT_KIND = "Pod"; private final String namespace; - private final OpenShiftClientFactory clientFactory; + private final KubernetesClientFactory clientFactory; private final ConcurrentLinkedQueue podActionHandlers; private final ConcurrentLinkedQueue containerEventsHandlers; private final String workspaceId; private Watch podWatch; private Watch containerWatch; - OpenShiftPods(String namespace, String workspaceId, OpenShiftClientFactory clientFactory) { + KubernetesPods(String namespace, String workspaceId, KubernetesClientFactory clientFactory) { this.namespace = namespace; this.workspaceId = workspaceId; this.clientFactory = clientFactory; @@ -199,7 +199,7 @@ public void onClose(KubernetesClientException cause) { } /** - * Starts watching the pods inside OpenShift namespace and registers a specified handler for such + * Starts watching the pods inside Kubernetes namespace and registers a specified handler for such * events. Note that watcher can be started only once so two times invocation of this method will * not produce new watcher and just register the event handlers. * @@ -279,7 +279,7 @@ public void onClose(KubernetesClientException ignored) {} containerEventsHandlers.add(handler); } - /** Stops watching the pods inside OpenShift namespace. */ + /** Stops watching the pods inside Kubernetes namespace. */ void stopWatch() { try { if (podWatch != null) { @@ -343,7 +343,7 @@ public void exec(String podName, String containerName, int timeoutMin, String[] /** * Deletes pod with given name. * - *

Note that this method will mark OpenShift pod as interrupted and then will wait until pod + *

Note that this method will mark Kubernetes pod as interrupted and then will wait until pod * will be killed. * * @param name name of pod to remove @@ -369,7 +369,7 @@ public void delete(String name) throws InfrastructureException { /** * Deletes all existing pods. * - *

Note that this method will mark OpenShift pods as interrupted and then will wait until all + *

Note that this method will mark Kubernetes pods as interrupted and then will wait until all * pods will be killed. * * @throws InfrastructureException when {@link Thread} is interrupted while command executing diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftServices.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesServices.java similarity index 77% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftServices.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesServices.java index 58e0834b042..b3fdc2c44f3 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftServices.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesServices.java @@ -8,30 +8,30 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_WORKSPACE_ID_LABEL; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.putLabel; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.putSelector; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_WORKSPACE_ID_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putLabel; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putSelector; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.client.KubernetesClientException; import java.util.List; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; /** * Defines an internal API for managing {@link Service} instances in {@link - * OpenShiftServices#namespace predefined namespace}. + * KubernetesServices#namespace predefined namespace}. * * @author Sergii Leshchenko */ -public class OpenShiftServices { +public class KubernetesServices { private final String namespace; private final String workspaceId; - private final OpenShiftClientFactory clientFactory; + private final KubernetesClientFactory clientFactory; - OpenShiftServices(String namespace, String workspaceId, OpenShiftClientFactory clientFactory) { + KubernetesServices(String namespace, String workspaceId, KubernetesClientFactory clientFactory) { this.namespace = namespace; this.workspaceId = workspaceId; this.clientFactory = clientFactory; diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java new file mode 100644 index 00000000000..c02c2c5dec7 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; + +import static com.google.common.base.Strings.isNullOrEmpty; + +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import javax.inject.Named; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.core.notification.EventSubscriber; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.api.workspace.shared.event.WorkspaceRemovedEvent; +import org.eclipse.che.commons.annotation.Nullable; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Listener for removing Kubernetes namespace on {@code WorkspaceRemovedEvent}. + * + * @author Sergii Leshchenko + */ +@Singleton +public class RemoveNamespaceOnWorkspaceRemove implements EventSubscriber { + private static final Logger LOG = LoggerFactory.getLogger(RemoveNamespaceOnWorkspaceRemove.class); + + private final KubernetesClientFactory clientFactory; + private final String namespaceName; + + @Inject + public RemoveNamespaceOnWorkspaceRemove( + @Nullable @Named("che.infra.kubernetes.namespace") String namespaceName, + KubernetesClientFactory clientFactory) { + this.namespaceName = namespaceName; + this.clientFactory = clientFactory; + } + + @Inject + public void subscribe(EventService eventService) { + if (isNullOrEmpty(namespaceName)) { + eventService.subscribe(this); + } + } + + @Override + public void onEvent(WorkspaceRemovedEvent event) { + try { + doRemoveNamespace(event.getWorkspace().getId()); + } catch (InfrastructureException e) { + LOG.warn( + "Fail to remove Kubernetes namespace for workspace with id {}. Cause: {}", + event.getWorkspace().getId(), + e.getMessage()); + } + } + + @VisibleForTesting + void doRemoveNamespace(String namespaceName) throws InfrastructureException { + clientFactory.create().namespaces().withName(namespaceName).delete(); + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/event/ContainerEvent.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/ContainerEvent.java similarity index 96% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/event/ContainerEvent.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/ContainerEvent.java index 95f233b3dc3..39e6e143bef 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/event/ContainerEvent.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/ContainerEvent.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.event; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event; import java.util.Objects; diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/event/ContainerEventHandler.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/ContainerEventHandler.java similarity index 78% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/event/ContainerEventHandler.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/ContainerEventHandler.java index 9280f395f1b..22bda88f862 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/event/ContainerEventHandler.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/ContainerEventHandler.java @@ -8,10 +8,10 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.event; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event; /** - * Defines the handling mechanism for OpenShift container events. + * Defines the handling mechanism for Kubernetes container events. * * @author Sergii Leshchenko */ diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/event/PodActionHandler.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/PodActionHandler.java similarity index 80% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/event/PodActionHandler.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/PodActionHandler.java index 2851de4d0c7..b9dfff26444 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/event/PodActionHandler.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/event/PodActionHandler.java @@ -8,13 +8,13 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.event; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.Watcher.Action; /** - * Defines the handling mechanism for OpenShift pod action events. + * Defines the handling mechanism for Kubernetes pod action events. * * @author Anton Korneta */ diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/CommonPVCStrategy.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/CommonPVCStrategy.java similarity index 75% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/CommonPVCStrategy.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/CommonPVCStrategy.java index 1cfec94330a..237280892fe 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/CommonPVCStrategy.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/CommonPVCStrategy.java @@ -8,14 +8,14 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; import static java.lang.String.format; import static java.util.stream.Collectors.toSet; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.newPVC; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.newVolume; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.newVolumeMount; -import static org.eclipse.che.workspace.infrastructure.openshift.provision.LogsVolumeMachineProvisioner.LOGS_VOLUME_NAME; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newPVC; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newVolume; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newVolumeMount; +import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner.LOGS_VOLUME_NAME; import com.google.inject.Inject; import io.fabric8.kubernetes.api.model.Container; @@ -32,14 +32,14 @@ import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.workspace.infrastructure.openshift.Names; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftPersistentVolumeClaims; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProjectFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPersistentVolumeClaims; /** - * Provides common PVC for each workspace in one OpenShift project. + * Provides common PVC for each workspace in one Kubernetes namespace. * *

This strategy uses subpaths for resolving backed up data paths collisions.
* Subpaths evaluated as following: '{workspaceId}/{workspace data folder}'. Workspace data folder @@ -72,16 +72,16 @@ public class CommonPVCStrategy implements WorkspaceVolumesStrategy { private final String pvcName; private final String pvcAccessMode; private final PVCSubPathHelper pvcSubPathHelper; - private final OpenShiftProjectFactory factory; + private final KubernetesNamespaceFactory factory; @Inject public CommonPVCStrategy( - @Named("che.infra.openshift.pvc.name") String pvcName, - @Named("che.infra.openshift.pvc.quantity") String pvcQuantity, - @Named("che.infra.openshift.pvc.access_mode") String pvcAccessMode, - @Named("che.infra.openshift.pvc.precreate_subpaths") boolean preCreateDirs, + @Named("che.infra.kubernetes.pvc.name") String pvcName, + @Named("che.infra.kubernetes.pvc.quantity") String pvcQuantity, + @Named("che.infra.kubernetes.pvc.access_mode") String pvcAccessMode, + @Named("che.infra.kubernetes.pvc.precreate_subpaths") boolean preCreateDirs, PVCSubPathHelper pvcSubPathHelper, - OpenShiftProjectFactory factory) { + KubernetesNamespaceFactory factory) { this.pvcName = pvcName; this.pvcQuantity = pvcQuantity; this.pvcAccessMode = pvcAccessMode; @@ -91,17 +91,17 @@ public CommonPVCStrategy( } @Override - public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) throws InfrastructureException { final String workspaceId = identity.getWorkspaceId(); final Set subPaths = new HashSet<>(); final PersistentVolumeClaim pvc = newPVC(pvcName, pvcAccessMode, pvcQuantity); - osEnv.getPersistentVolumeClaims().put(pvcName, pvc); - for (Pod pod : osEnv.getPods().values()) { + k8sEnv.getPersistentVolumeClaims().put(pvcName, pvc); + for (Pod pod : k8sEnv.getPods().values()) { PodSpec podSpec = pod.getSpec(); for (Container container : podSpec.getContainers()) { String machineName = Names.machineName(pod, container); - InternalMachineConfig machineConfig = osEnv.getMachines().get(machineName); + InternalMachineConfig machineConfig = k8sEnv.getMachines().get(machineName); addMachineVolumes(workspaceId, subPaths, pod, container, machineConfig.getVolumes()); } } @@ -113,12 +113,12 @@ public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) } @Override - public void prepare(OpenShiftEnvironment osEnv, String workspaceId) + public void prepare(KubernetesEnvironment k8sEnv, String workspaceId) throws InfrastructureException { - final Collection claims = osEnv.getPersistentVolumeClaims().values(); + final Collection claims = k8sEnv.getPersistentVolumeClaims().values(); if (!claims.isEmpty()) { - final OpenShiftProject project = factory.create(workspaceId); - final OpenShiftPersistentVolumeClaims pvcs = project.persistentVolumeClaims(); + final KubernetesNamespace namespace = factory.create(workspaceId); + final KubernetesPersistentVolumeClaims pvcs = namespace.persistentVolumeClaims(); final Set existing = pvcs.get().stream().map(p -> p.getMetadata().getName()).collect(toSet()); for (PersistentVolumeClaim pvc : claims) { diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/PVCSubPathHelper.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java similarity index 90% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/PVCSubPathHelper.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java index f3052ef2087..90997a6b39a 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/PVCSubPathHelper.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java @@ -8,12 +8,12 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; import static java.util.Collections.singletonMap; import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.newVolume; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.newVolumeMount; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newVolume; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newVolumeMount; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; @@ -35,8 +35,8 @@ import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; import org.eclipse.che.commons.lang.concurrent.ThreadLocalPropagateContext; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftPods; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProjectFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPods; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,15 +73,15 @@ public class PVCSubPathHelper { private final String pvcName; private final String jobImage; private final String jobMemoryLimit; - private final OpenShiftProjectFactory factory; + private final KubernetesNamespaceFactory factory; private final ExecutorService executor; @Inject PVCSubPathHelper( - @Named("che.infra.openshift.pvc.name") String pvcName, - @Named("che.infra.openshift.pvc.jobs.memorylimit") String jobMemoryLimit, - @Named("che.infra.openshift.pvc.jobs.image") String jobImage, - OpenShiftProjectFactory factory) { + @Named("che.infra.kubernetes.pvc.name") String pvcName, + @Named("che.infra.kubernetes.pvc.jobs.memorylimit") String jobMemoryLimit, + @Named("che.infra.kubernetes.pvc.jobs.image") String jobImage, + KubernetesNamespaceFactory factory) { this.pvcName = pvcName; this.jobMemoryLimit = jobMemoryLimit; this.jobImage = jobImage; @@ -130,7 +130,7 @@ void execute(String workspaceId, String[] commandBase, String... arguments) { final String podName = jobName + '-' + workspaceId; final String[] command = buildCommand(commandBase, arguments); final Pod pod = newPod(podName, command); - OpenShiftPods pods = null; + KubernetesPods pods = null; try { pods = factory.create(workspaceId).pods(); pods.create(pod); diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/UniqueWorkspacePVCStrategy.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/UniqueWorkspacePVCStrategy.java similarity index 72% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/UniqueWorkspacePVCStrategy.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/UniqueWorkspacePVCStrategy.java index 97ec1906b88..f2efd7a5ee3 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/UniqueWorkspacePVCStrategy.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/UniqueWorkspacePVCStrategy.java @@ -8,15 +8,15 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_VOLUME_NAME_LABEL; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_WORKSPACE_ID_LABEL; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.newPVC; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.newVolume; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.newVolumeMount; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.putLabel; -import static org.eclipse.che.workspace.infrastructure.openshift.provision.LogsVolumeMachineProvisioner.LOGS_VOLUME_NAME; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_VOLUME_NAME_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_WORKSPACE_ID_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newPVC; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newVolume; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newVolumeMount; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putLabel; +import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner.LOGS_VOLUME_NAME; import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.ObjectMeta; @@ -33,20 +33,21 @@ import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.commons.annotation.Nullable; -import org.eclipse.che.workspace.infrastructure.openshift.Names; -import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftPersistentVolumeClaims; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProjectFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPersistentVolumeClaims; /** * Provides a unique PVC for each workspace. * *

Names for PVCs are evaluated as: '{configured_prefix}' + '-' +'{generated_8_chars}' to avoid - * naming collisions inside of one OpenShift project. + * naming collisions inside of one Kubernetes namespace. * *

Note that for this strategy count of simultaneously running workspaces and workspaces with - * backed up data is always the same and equal to the count of available PVCs in OpenShift project. + * backed up data is always the same and equal to the count of available PVCs in Kubernetes + * namespace. * *

The usage of PVCs for this strategy is next: one PVC per volume, but for volumes that are * provided by Che there a small exception:
@@ -64,32 +65,32 @@ public class UniqueWorkspacePVCStrategy implements WorkspaceVolumesStrategy { public static final String UNIQUE_STRATEGY = "unique"; private final String pvcNamePrefix; - private final String projectName; + private final String namespaceName; private final String pvcQuantity; private final String pvcAccessMode; - private final OpenShiftClientFactory clientFactory; - private final OpenShiftProjectFactory factory; + private final KubernetesClientFactory clientFactory; + private final KubernetesNamespaceFactory factory; @Inject public UniqueWorkspacePVCStrategy( - @Nullable @Named("che.infra.openshift.project") String projectName, - @Named("che.infra.openshift.pvc.name") String pvcNamePrefix, - @Named("che.infra.openshift.pvc.quantity") String pvcQuantity, - @Named("che.infra.openshift.pvc.access_mode") String pvcAccessMode, - OpenShiftProjectFactory factory, - OpenShiftClientFactory clientFactory) { + @Nullable @Named("che.infra.kubernetes.namespace") String namespaceName, + @Named("che.infra.kubernetes.pvc.name") String pvcNamePrefix, + @Named("che.infra.kubernetes.pvc.quantity") String pvcQuantity, + @Named("che.infra.kubernetes.pvc.access_mode") String pvcAccessMode, + KubernetesNamespaceFactory factory, + KubernetesClientFactory clientFactory) { this.pvcNamePrefix = pvcNamePrefix; this.pvcQuantity = pvcQuantity; - this.projectName = projectName; + this.namespaceName = namespaceName; this.pvcAccessMode = pvcAccessMode; this.clientFactory = clientFactory; this.factory = factory; } @Override - public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) throws InfrastructureException { - final Map claims = osEnv.getPersistentVolumeClaims(); + final Map claims = k8sEnv.getPersistentVolumeClaims(); final String workspaceId = identity.getWorkspaceId(); // fetches all existing PVCs related to given workspace and groups them by volume name final Map volumeName2PVC = @@ -98,24 +99,24 @@ public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) .create(workspaceId) .persistentVolumeClaims() .getByLabel(CHE_WORKSPACE_ID_LABEL, workspaceId)); - for (Pod pod : osEnv.getPods().values()) { + for (Pod pod : k8sEnv.getPods().values()) { final PodSpec podSpec = pod.getSpec(); for (Container container : podSpec.getContainers()) { final String machineName = Names.machineName(pod, container); - Map volumes = osEnv.getMachines().get(machineName).getVolumes(); + Map volumes = k8sEnv.getMachines().get(machineName).getVolumes(); addMachineVolumes(workspaceId, claims, volumeName2PVC, pod, container, volumes); } } } @Override - public void prepare(OpenShiftEnvironment osEnv, String workspaceId) + public void prepare(KubernetesEnvironment k8sEnv, String workspaceId) throws InfrastructureException { - if (!osEnv.getPersistentVolumeClaims().isEmpty()) { - final OpenShiftPersistentVolumeClaims osClaims = + if (!k8sEnv.getPersistentVolumeClaims().isEmpty()) { + final KubernetesPersistentVolumeClaims k8sClaims = factory.create(workspaceId).persistentVolumeClaims(); - for (PersistentVolumeClaim pvc : osEnv.getPersistentVolumeClaims().values()) { - osClaims.create(pvc); + for (PersistentVolumeClaim pvc : k8sEnv.getPersistentVolumeClaims().values()) { + k8sClaims.create(pvc); } } } @@ -175,7 +176,7 @@ public void cleanup(String workspaceId) throws InfrastructureException { clientFactory .create() .persistentVolumeClaims() - .inNamespace(projectName) + .inNamespace(namespaceName) .withLabel(CHE_WORKSPACE_ID_LABEL, workspaceId) .delete(); } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspacePVCCleaner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspacePVCCleaner.java similarity index 77% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspacePVCCleaner.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspacePVCCleaner.java index 41bf69df1b4..4a21b7e6d68 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspacePVCCleaner.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspacePVCCleaner.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; import static com.google.common.base.Strings.isNullOrEmpty; @@ -24,11 +24,11 @@ import org.slf4j.LoggerFactory; /** - * Cleans the workspace related OpenShift resources after {@code WorkspaceRemovedEvent}. + * Cleans the workspace related Kubernetes resources after {@code WorkspaceRemovedEvent}. * *

Note that depending on a configuration different types of cleaners may be chosen. In case of - * configuration when new OpenShift project created for each workspace, the whole project will be - * removed, after workspace removal. + * configuration when new Kubernetes namespace created for each workspace, the whole namespace will + * be removed, after workspace removal. * * @author Anton Korneta */ @@ -38,22 +38,22 @@ public class WorkspacePVCCleaner { private static final Logger LOG = LoggerFactory.getLogger(WorkspacePVCCleaner.class); private final boolean pvcEnabled; - private final String projectName; + private final String namespaceName; private final WorkspaceVolumesStrategy strategy; @Inject public WorkspacePVCCleaner( - @Named("che.infra.openshift.pvc.enabled") boolean pvcEnabled, - @Nullable @Named("che.infra.openshift.project") String projectName, + @Named("che.infra.kubernetes.pvc.enabled") boolean pvcEnabled, + @Nullable @Named("che.infra.kubernetes.namespace") String namespaceName, WorkspaceVolumesStrategy pvcStrategy) { this.pvcEnabled = pvcEnabled; - this.projectName = projectName; + this.namespaceName = namespaceName; this.strategy = pvcStrategy; } @Inject public void subscribe(EventService eventService) { - if (pvcEnabled && !isNullOrEmpty(projectName)) + if (pvcEnabled && !isNullOrEmpty(namespaceName)) eventService.subscribe( new EventSubscriber() { @Override diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspaceVolumeStrategyProvider.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspaceVolumeStrategyProvider.java similarity index 90% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspaceVolumeStrategyProvider.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspaceVolumeStrategyProvider.java index 57a0d2f409e..28a8ed45f85 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspaceVolumeStrategyProvider.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspaceVolumeStrategyProvider.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; import static java.lang.String.format; @@ -30,7 +30,7 @@ public class WorkspaceVolumeStrategyProvider implements Provider strategies) { final WorkspaceVolumesStrategy volumeStrategy = strategies.get(strategy); if (volumeStrategy != null) { diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspaceVolumesStrategy.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspaceVolumesStrategy.java similarity index 74% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspaceVolumesStrategy.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspaceVolumesStrategy.java index 494fa0af794..6d5d707e46f 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspaceVolumesStrategy.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspaceVolumesStrategy.java @@ -8,12 +8,12 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.provision.ConfigurationProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ConfigurationProvisioner; /** * Defines a basic set of operations for workspace volume provisioning strategies. @@ -25,23 +25,23 @@ public interface WorkspaceVolumesStrategy extends ConfigurationProvisioner { /** * Configures the workspace PVCs, volumes, claim bindings with a strategy specific options. * - * @param osEnv OpenShift environment + * @param k8sEnv Kubernetes environment * @param identity runtime identity * @throws InfrastructureException when any error occurs while provisioning volumes */ @Override - void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) + void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) throws InfrastructureException; /** * Prepares volumes for backup of workspace data on a specific machine in a strategy specific way. * Note that this step, depending on the strategy, may take some time. * - * @param osEnv OpenShift environment that changes as a result of preparation + * @param k8sEnv Kubernetes environment that changes as a result of preparation * @param workspaceId the workspace identifier for which volumes will be prepared * @throws InfrastructureException when any error while preparation occurs */ - void prepare(OpenShiftEnvironment osEnv, String workspaceId) throws InfrastructureException; + void prepare(KubernetesEnvironment k8sEnv, String workspaceId) throws InfrastructureException; /** * Cleanups workspace backed up data in a strategy specific way. diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/ConfigurationProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/ConfigurationProvisioner.java new file mode 100644 index 00000000000..f8377b85fd6 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/ConfigurationProvisioner.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; + +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInfrastructure; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; + +/** + * Modifies workspace environment configuration and Kubernetes environment with everything needed + * for some logical part of {@link KubernetesInfrastructure}. + * + * @author Anton Korneta + */ +public interface ConfigurationProvisioner { + + /** + * Configures the Kubernetes environment and workspace environment with infrastructure needs. + * + * @param k8sEnv Kubernetes environment + * @param identity runtime identity + * @throws InfrastructureException when any error occurs + */ + void provision(T k8sEnv, RuntimeIdentity identity) throws InfrastructureException; +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/InstallerServersPortProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/InstallerServersPortProvisioner.java similarity index 91% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/InstallerServersPortProvisioner.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/InstallerServersPortProvisioner.java index a0826b41cc5..79a1547c64b 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/InstallerServersPortProvisioner.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/InstallerServersPortProvisioner.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; import static java.util.stream.Collectors.toList; @@ -32,8 +32,8 @@ import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; import org.eclipse.che.commons.lang.Pair; -import org.eclipse.che.workspace.infrastructure.openshift.Names; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; /** * Fixes installers servers ports if conflicts are present. @@ -60,24 +60,24 @@ public class InstallerServersPortProvisioner implements ConfigurationProvisioner @Inject public InstallerServersPortProvisioner( - @Named("che.infra.openshift.installer_server_min_port") int minPort, - @Named("che.infra.openshift.installer_server_max_port") int maxPort) { + @Named("che.infra.kubernetes.installer_server_min_port") int minPort, + @Named("che.infra.kubernetes.installer_server_max_port") int maxPort) { this.minPort = minPort; this.maxPort = maxPort; } @Override - public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) throws InfrastructureException { - for (Pod pod : osEnv.getPods().values()) { + for (Pod pod : k8sEnv.getPods().values()) { // it is needed to detect conflicts between all containers in a pod // because they use the same ports List podMachines = pod.getSpec() .getContainers() .stream() - .map(container -> osEnv.getMachines().get(Names.machineName(pod, container))) + .map(container -> k8sEnv.getMachines().get(Names.machineName(pod, container))) .collect(toList()); fixInstallersPortsConflicts(podMachines); diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftCheApiEnvVarProvider.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/KubernetesCheApiEnvVarProvider.java similarity index 76% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftCheApiEnvVarProvider.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/KubernetesCheApiEnvVarProvider.java index 0aa91bffcda..cae2118e986 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftCheApiEnvVarProvider.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/KubernetesCheApiEnvVarProvider.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; import javax.inject.Inject; import javax.inject.Named; @@ -18,16 +18,16 @@ import org.eclipse.che.commons.lang.Pair; /** - * Provides env variable to OpenShift machine with url of Che API. + * Provides env variable to Kubernetes machine with url of Che API. * * @author Sergii Leshchenko */ -public class OpenShiftCheApiEnvVarProvider implements CheApiEnvVarProvider { +public class KubernetesCheApiEnvVarProvider implements CheApiEnvVarProvider { private final String cheServerEndpoint; @Inject - public OpenShiftCheApiEnvVarProvider(@Named("che.api") String cheServerEndpoint) { + public KubernetesCheApiEnvVarProvider(@Named("che.api") String cheServerEndpoint) { this.cheServerEndpoint = cheServerEndpoint; } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/LogsVolumeMachineProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/LogsVolumeMachineProvisioner.java similarity index 84% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/LogsVolumeMachineProvisioner.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/LogsVolumeMachineProvisioner.java index e8ddf4f9dd3..7baefd719a2 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/LogsVolumeMachineProvisioner.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/LogsVolumeMachineProvisioner.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; import com.google.inject.Inject; import javax.inject.Named; @@ -16,7 +16,7 @@ import org.eclipse.che.api.workspace.server.model.impl.VolumeImpl; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; /** * Adds to each machine inside environment volume with logs root. @@ -34,7 +34,7 @@ public LogsVolumeMachineProvisioner(@Named("che.workspace.logs.root_dir") String } @Override - public void provision(OpenShiftEnvironment environment, RuntimeIdentity identity) + public void provision(KubernetesEnvironment environment, RuntimeIdentity identity) throws InfrastructureException { for (InternalMachineConfig machine : environment.getMachines().values()) { machine.getVolumes().put(LOGS_VOLUME_NAME, new VolumeImpl().withPath(logsRootPath)); diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/UniqueNamesProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/UniqueNamesProvisioner.java new file mode 100644 index 00000000000..f16b8865036 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/UniqueNamesProvisioner.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; + +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putLabel; + +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import java.util.HashSet; +import java.util.Set; +import javax.inject.Singleton; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.workspace.infrastructure.kubernetes.Constants; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; + +/** + * Makes names of Kubernetes pods and ingresses unique whole namespace by {@link Names}. + * + *

Original names will be stored in {@link Constants#CHE_ORIGINAL_NAME_LABEL} label of renamed + * object. + * + * @author Anton Korneta + * @see Names#uniquePodName(String, String) + * @see Names#generateName(String) + */ +@Singleton +public class UniqueNamesProvisioner + implements ConfigurationProvisioner { + + @Override + public void provision(T k8sEnv, RuntimeIdentity identity) throws InfrastructureException { + final String workspaceId = identity.getWorkspaceId(); + final Set pods = new HashSet<>(k8sEnv.getPods().values()); + k8sEnv.getPods().clear(); + for (Pod pod : pods) { + final ObjectMeta podMeta = pod.getMetadata(); + putLabel(pod, Constants.CHE_ORIGINAL_NAME_LABEL, podMeta.getName()); + final String podName = Names.uniquePodName(podMeta.getName(), workspaceId); + podMeta.setName(podName); + k8sEnv.getPods().put(podName, pod); + } + final Set ingresses = new HashSet<>(k8sEnv.getIngresses().values()); + k8sEnv.getIngresses().clear(); + for (Ingress ingress : ingresses) { + final ObjectMeta ingressMeta = ingress.getMetadata(); + putLabel(ingress, Constants.CHE_ORIGINAL_NAME_LABEL, ingressMeta.getName()); + final String ingressName = Names.generateName("ingress"); + ingressMeta.setName(ingressName); + k8sEnv.getIngresses().put(ingressName, ingress); + } + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/env/EnvVarsConverter.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/env/EnvVarsConverter.java similarity index 69% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/env/EnvVarsConverter.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/env/EnvVarsConverter.java index 6b20d0bf0b6..a1c493e81e1 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/env/EnvVarsConverter.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/env/EnvVarsConverter.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.env; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision.env; import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.EnvVar; @@ -18,25 +18,25 @@ import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.workspace.infrastructure.openshift.Names; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.provision.ConfigurationProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ConfigurationProvisioner; /** - * Converts environment variables in {@link MachineConfig} to OpenShift environment variables. + * Converts environment variables in {@link MachineConfig} to Kubernetes environment variables. * * @author Alexander Garagatyi */ @Singleton public class EnvVarsConverter implements ConfigurationProvisioner { @Override - public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) throws InfrastructureException { - for (Pod pod : osEnv.getPods().values()) { + for (Pod pod : k8sEnv.getPods().values()) { for (Container container : pod.getSpec().getContainers()) { String machineName = Names.machineName(pod, container); - InternalMachineConfig machineConf = osEnv.getMachines().get(machineName); + InternalMachineConfig machineConf = k8sEnv.getMachines().get(machineName); machineConf .getEnv() .forEach( diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/env/LogsRootEnvVariableProvider.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/env/LogsRootEnvVariableProvider.java similarity index 94% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/env/LogsRootEnvVariableProvider.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/env/LogsRootEnvVariableProvider.java index 0604a27e98a..f8cf8a7cd7b 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/env/LogsRootEnvVariableProvider.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/env/LogsRootEnvVariableProvider.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.env; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision.env; import javax.inject.Inject; import javax.inject.Named; diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/limits/ram/RamLimitProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/limits/ram/RamLimitProvisioner.java similarity index 65% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/limits/ram/RamLimitProvisioner.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/limits/ram/RamLimitProvisioner.java index 8272a1a83e0..f20067e15e9 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/limits/ram/RamLimitProvisioner.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/limits/ram/RamLimitProvisioner.java @@ -8,10 +8,10 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.limits.ram; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram; import static org.eclipse.che.api.core.model.workspace.config.MachineConfig.MEMORY_LIMIT_ATTRIBUTE; -import static org.eclipse.che.workspace.infrastructure.openshift.Names.machineName; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Names.machineName; import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.Pod; @@ -19,22 +19,22 @@ import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.provision.ConfigurationProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.util.Containers; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ConfigurationProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.util.Containers; /** - * Sets Ram limit to OpenShift machine. + * Sets Ram limit to Kubernetes machine. * * @author Anton Korneta */ public class RamLimitProvisioner implements ConfigurationProvisioner { @Override - public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) throws InfrastructureException { - final Map machines = osEnv.getMachines(); - for (Pod pod : osEnv.getPods().values()) { + final Map machines = k8sEnv.getMachines(); + for (Pod pod : k8sEnv.getPods().values()) { for (Container container : pod.getSpec().getContainers()) { final Map attributes = machines.get(machineName(pod, container)).getAttributes(); diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/restartpolicy/RestartPolicyRewriter.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/restartpolicy/RestartPolicyRewriter.java similarity index 76% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/restartpolicy/RestartPolicyRewriter.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/restartpolicy/RestartPolicyRewriter.java index e40d543d63d..fa67d567cb5 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/restartpolicy/RestartPolicyRewriter.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/restartpolicy/RestartPolicyRewriter.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.restartpolicy; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision.restartpolicy; import static java.lang.String.format; @@ -17,8 +17,8 @@ import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.model.impl.WarningImpl; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.provision.ConfigurationProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ConfigurationProvisioner; /** * Rewrites restart policy to supported one - 'Never'. @@ -29,17 +29,17 @@ public class RestartPolicyRewriter implements ConfigurationProvisioner { static final String DEFAULT_RESTART_POLICY = "Never"; @Override - public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) throws InfrastructureException { - for (Pod podConfig : osEnv.getPods().values()) { + for (Pod podConfig : k8sEnv.getPods().values()) { final String podName = podConfig.getMetadata().getName(); final PodSpec podSpec = podConfig.getSpec(); - rewriteRestartPolicy(podSpec, podName, osEnv); + rewriteRestartPolicy(podSpec, podName, k8sEnv); } } - private void rewriteRestartPolicy(PodSpec podSpec, String podName, OpenShiftEnvironment env) { + private void rewriteRestartPolicy(PodSpec podSpec, String podName, KubernetesEnvironment env) { final String restartPolicy = podSpec.getRestartPolicy(); if (restartPolicy != null && !DEFAULT_RESTART_POLICY.equalsIgnoreCase(restartPolicy)) { diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java new file mode 100644 index 00000000000..59e43e95e0f --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.provision.server; + +import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodSpec; +import java.util.Map; +import javax.inject.Singleton; +import org.eclipse.che.api.core.model.workspace.config.ServerConfig; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ConfigurationProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerExposer; + +/** + * Converts {@link ServerConfig} to Kubernetes related objects to add a server into Kubernetes + * runtime. + * + *

Adds Kubernetes objects by calling {@link KubernetesServerExposer#expose(Map)} on each machine + * with servers. + * + * @author Alexander Garagatyi + */ +@Singleton +public class ServersConverter implements ConfigurationProvisioner { + + @Override + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) + throws InfrastructureException { + + for (Pod podConfig : k8sEnv.getPods().values()) { + final PodSpec podSpec = podConfig.getSpec(); + for (Container containerConfig : podSpec.getContainers()) { + String machineName = Names.machineName(podConfig, containerConfig); + InternalMachineConfig machineConfig = k8sEnv.getMachines().get(machineName); + if (!machineConfig.getServers().isEmpty()) { + KubernetesServerExposer kubernetesServerExposer = + new KubernetesServerExposer<>(machineName, podConfig, containerConfig, k8sEnv); + kubernetesServerExposer.expose(machineConfig.getServers()); + } + } + } + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposer.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java similarity index 76% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposer.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java index 5f9a39b787f..1a7bf0387f0 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposer.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java @@ -8,13 +8,13 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift; +package org.eclipse.che.workspace.infrastructure.kubernetes.server; import static java.lang.Integer.parseInt; import static java.util.stream.Collectors.toMap; import static org.eclipse.che.api.core.model.workspace.config.ServerConfig.INTERNAL_SERVER_ATTRIBUTE; import static org.eclipse.che.commons.lang.NameGenerator.generate; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_ORIGINAL_NAME_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_ORIGINAL_NAME_LABEL; import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.ContainerPort; @@ -32,12 +32,10 @@ import io.fabric8.kubernetes.api.model.extensions.Ingress; import io.fabric8.kubernetes.api.model.extensions.IngressBackend; import io.fabric8.kubernetes.api.model.extensions.IngressBackendBuilder; -import io.fabric8.kubernetes.api.model.extensions.IngressBuilder; import io.fabric8.kubernetes.api.model.extensions.IngressRule; import io.fabric8.kubernetes.api.model.extensions.IngressRuleBuilder; import io.fabric8.kubernetes.api.model.extensions.IngressSpec; import io.fabric8.kubernetes.api.model.extensions.IngressSpecBuilder; -import io.fabric8.openshift.api.model.Route; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -49,21 +47,23 @@ import java.util.stream.Collectors; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.provision.UniqueNamesProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; +import org.eclipse.che.workspace.infrastructure.kubernetes.Constants; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; /** - * Helps to modify {@link OpenShiftEnvironment} to make servers that are configured by {@link + * Helps to modify {@link KubernetesEnvironment} to make servers that are configured by {@link * ServerConfig} publicly or workspace-wide accessible. * *

To make server accessible it is needed to make sure that container port is declared, create * {@link Service}. To make it also publicly accessible it is needed to create corresponding {@link - * Route} for exposing this port. + * Ingress} for exposing this port. * - *

Created services and routes will have serialized servers which are exposed by the + *

Created services and ingresses will have serialized servers which are exposed by the * corresponding object and machine name to which these servers belongs to. * - *

Container, service and route are linked in the following way: + *

Container, service and ingress are linked in the following way: * * 

  * Pod
@@ -96,61 +96,59 @@
  *       protocol: TCP                ---->> Pod.spec.ports[0].protocol
  *
* - * Then corresponding route expose one of the service's port: + * Then corresponding ingress expose one of the service's port: * * 
- * Route
+ * Ingress
  * ...
  * spec:
- *   to:
- *     name: dev-machine              ---->> Service.metadata.name
- *     targetPort: [8080|web-app]     ---->> Service.spec.ports[0].[port|name]
+ *   rules:
+ *     - http:
+ *         paths:
+ *           - path: service123/webapp        ---->> Service.metadata.name + / + Service.spec.ports[0].name
+ *             backend:
+ *               serviceName: service123      ---->> Service.metadata.name
+ *               servicePort: [8080|web-app]  ---->> Service.spec.ports[0].[port|name]
  *
* - *

For accessing publicly accessible server user will use route host. For accessing - * workspace-wide accessible server user will use service name. Information about servers that are - * exposed by route and/or service are stored in annotations of a route or service. + *

For accessing publicly accessible server user will use ingress host or its load balancer IP. + * For accessing workspace-wide accessible server user will use service name. Information about + * servers that are exposed by ingress and/or service are stored in annotations of a ingress or + * service. * * @author Sergii Leshchenko * @author Alexander Garagatyi * @see Annotations */ -public class ServerExposer { +public class KubernetesServerExposer { public static final int SERVER_UNIQUE_PART_SIZE = 8; public static final String SERVER_PREFIX = "server"; - private final String machineName; - private final Container container; - private final Pod pod; - private final OpenShiftEnvironment openShiftEnvironment; - private final String host; - - public ServerExposer( - String machineName, - Pod pod, - Container container, - OpenShiftEnvironment openShiftEnvironment, - String host) { + protected final String machineName; + protected final Container container; + protected final Pod pod; + protected final T kubernetesEnvironment; + + public KubernetesServerExposer( + String machineName, Pod pod, Container container, T kubernetesEnvironment) { this.machineName = machineName; this.pod = pod; this.container = container; - this.openShiftEnvironment = openShiftEnvironment; - this.host = host; + this.kubernetesEnvironment = kubernetesEnvironment; } /** * Exposes specified servers. * - *

Note that created OpenShift objects will select the corresponding pods by {@link + *

Note that created Kubernetes objects will select the corresponding pods by {@link * Constants#CHE_ORIGINAL_NAME_LABEL} label. That should be added by {@link * UniqueNamesProvisioner}. * * @param servers servers to expose - * @see UniqueNamesProvisioner#provision(OpenShiftEnvironment, RuntimeIdentity) + * @see UniqueNamesProvisioner#provision(KubernetesEnvironment, RuntimeIdentity) */ public void expose(Map servers) { - Map portToServicePort = exposePort(servers.values()); Map internalServers = new HashMap<>(); Map externalServers = new HashMap<>(); @@ -163,6 +161,7 @@ public void expose(Map servers) { } }); + Map portToServicePort = exposePort(servers.values()); Service service = new ServiceBuilder() .withName(generate(SERVER_PREFIX, SERVER_UNIQUE_PART_SIZE) + '-' + machineName) @@ -173,12 +172,19 @@ public void expose(Map servers) { .build(); String serviceName = service.getMetadata().getName(); - openShiftEnvironment.getServices().put(serviceName, service); + kubernetesEnvironment.getServices().put(serviceName, service); + + exposeExternalServers(serviceName, portToServicePort, externalServers); + } + protected void exposeExternalServers( + String serviceName, + Map portToServicePort, + Map externalServers) { for (ServicePort servicePort : portToServicePort.values()) { int port = servicePort.getTargetPort().getIntVal(); - Map routesServers = + Map ingressesServers = externalServers .entrySet() .stream() @@ -189,13 +195,12 @@ public void expose(Map servers) { new IngressBuilder() .withName(serviceName + '-' + servicePort.getName()) .withMachineName(machineName) - .withTargetPort(servicePort.getName()) - .withServers(routesServers) - .withTo(serviceName) - .withHost(host) + .withServiceName(serviceName) + .withServicePort(servicePort.getName()) + .withServers(ingressesServers) .build(); - openShiftEnvironment.getIngresses().put(ingress.getMetadata().getName(), ingress); + kubernetesEnvironment.getIngresses().put(ingress.getMetadata().getName(), ingress); } } @@ -291,23 +296,22 @@ public ServiceBuilder withMachineName(String machineName) { private static class IngressBuilder { private String name; private String serviceName; - private IntOrString targetPort; + private IntOrString servicePort; private Map serversConfigs; private String machineName; - private String host; private IngressBuilder withName(String name) { this.name = name; return this; } - private IngressBuilder withTo(String serviceName) { + private IngressBuilder withServiceName(String serviceName) { this.serviceName = serviceName; return this; } - private IngressBuilder withTargetPort(String targetPortName) { - this.targetPort = new IntOrString(targetPortName); + private IngressBuilder withServicePort(String targetPortName) { + this.servicePort = new IntOrString(targetPortName); return this; } @@ -321,27 +325,21 @@ public IngressBuilder withMachineName(String machineName) { return this; } - public IngressBuilder withHost(String host) { - this.host = host; - return this; - } - private Ingress build() { IngressBackend ingressBackend = new IngressBackendBuilder() .withServiceName(serviceName) - .withNewServicePort(targetPort.getStrVal()) + .withNewServicePort(servicePort.getStrVal()) .build(); - String serverPath = "/" + serviceName + "/" + targetPort.getStrVal(); + String serverPath = "/" + serviceName + "/" + servicePort.getStrVal(); HTTPIngressPath httpIngressPath = new HTTPIngressPathBuilder().withPath(serverPath).withBackend(ingressBackend).build(); HTTPIngressRuleValue httpIngressRuleValue = new HTTPIngressRuleValueBuilder().withPaths(httpIngressPath).build(); IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build(); - // new IngressRuleBuilder().withHttp(httpIngressRuleValue).withHost(host).build(); IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build(); Map ingressAnnotations = new HashMap<>(); @@ -357,10 +355,7 @@ private Ingress build() { return new io.fabric8.kubernetes.api.model.extensions.IngressBuilder() .withSpec(ingressSpec) .withMetadata( - new ObjectMetaBuilder() - .withName(serviceName + targetPort.getStrVal() + "-ingress") - .withAnnotations(ingressAnnotations) - .build()) + new ObjectMetaBuilder().withName(name).withAnnotations(ingressAnnotations).build()) .build(); } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerResolver.java similarity index 67% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerResolver.java index 9e14f10515e..e98f71fe924 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolver.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerResolver.java @@ -8,40 +8,41 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift; +package org.eclipse.che.workspace.infrastructure.kubernetes.server; + +import static com.google.common.base.Strings.isNullOrEmpty; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.extensions.Ingress; import io.fabric8.kubernetes.api.model.extensions.IngressRule; -import io.fabric8.openshift.api.model.Route; import java.util.HashMap; import java.util.List; import java.util.Map; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.core.model.workspace.runtime.ServerStatus; import org.eclipse.che.api.workspace.server.model.impl.ServerImpl; +import org.eclipse.che.commons.annotation.Nullable; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; /** - * Helps to resolve {@link ServerImpl servers} by machine name according to specified {@link Route - * routes} and {@link Service services}. + * Helps to resolve {@link ServerImpl servers} by machine name according to specified {@link Ingress + * ingresses} and {@link Service services}. * - *

Objects annotations are used to check if {@link Service service} or {@link Route route} + *

Objects annotations are used to check if {@link Service service} or {@link Ingress ingress} * exposes the specified machine servers. * * @author Sergii Leshchenko * @author Alexander Garagatyi - * @see ServerExposer + * @see KubernetesServerExposer * @see Annotations */ -public class ServerResolver { +public class KubernetesServerResolver { private final Multimap services; - // private final Multimap routes; private final Multimap ingresses; - // private ServerResolver(List services, List routes) { - private ServerResolver(List services, List ingresses) { + public KubernetesServerResolver(List services, List ingresses) { this.services = ArrayListMultimap.create(); for (Service service : services) { String machineName = @@ -49,13 +50,6 @@ private ServerResolver(List services, List ingresses) { this.services.put(machineName, service); } - // this.routes = ArrayListMultimap.create(); - // for (Route route : routes) { - // String machineName = - // Annotations.newDeserializer(route.getMetadata().getAnnotations()).machineName(); - // this.routes.put(machineName, route); - // } - this.ingresses = ArrayListMultimap.create(); for (Ingress ingress : ingresses) { String machineName = @@ -64,11 +58,6 @@ private ServerResolver(List services, List ingresses) { } } - // public static ServerResolver of(List services, List routes) { - public static ServerResolver of(List services, List ingresses) { - return new ServerResolver(services, ingresses); - } - /** * Resolves servers by the specified machine name. * @@ -77,10 +66,17 @@ public static ServerResolver of(List services, List ingresses) */ public Map resolve(String machineName) { Map servers = new HashMap<>(); + fillInternalServers(machineName, servers); + fillExternalServers(machineName, servers); + return servers; + } + + protected void fillInternalServers(String machineName, Map servers) { services.get(machineName).forEach(service -> fillServiceServers(service, servers)); - // routes.get(machineName).forEach(route -> fillRouteServers(route, servers)); + } + + protected void fillExternalServers(String machineName, Map servers) { ingresses.get(machineName).forEach(ingress -> fillIngressServers(ingress, servers)); - return servers; } private void fillServiceServers(Service service, Map servers) { @@ -98,22 +94,6 @@ private void fillServiceServers(Service service, Map servers config.getAttributes()))); } - private void fillRouteServers(Route route, Map servers) { - Annotations.newDeserializer(route.getMetadata().getAnnotations()) - .servers() - .forEach( - (name, config) -> - servers.put( - name, - newServer( - config.getProtocol(), - route.getSpec().getHost(), - null, - // config.getPath(), - route.getSpec().getPath(), - config.getAttributes()))); - } - private void fillIngressServers(Ingress ingress, Map servers) { IngressRule ingressRule = ingress.getSpec().getRules().get(0); @@ -126,19 +106,34 @@ private void fillIngressServers(Ingress ingress, Map servers Annotations.newDeserializer(ingress.getMetadata().getAnnotations()) .servers() .forEach( - (name, config) -> - servers.put( - name, - newServer( - config.getProtocol(), - host, - null, - ingressRule.getHttp().getPaths().get(0).getPath() + "/" + config.getPath(), - config.getAttributes()))); + (name, config) -> { + String path = + buildPath(ingressRule.getHttp().getPaths().get(0).getPath(), config.getPath()); + servers.put( + name, newServer(config.getProtocol(), host, null, path, config.getAttributes())); + }); + } + + private String buildPath(String fragment1, @Nullable String fragment2) { + StringBuilder sb = new StringBuilder(fragment1); + + if (!isNullOrEmpty(fragment2)) { + if (!fragment1.endsWith("/")) { + sb.append('/'); + } + + if (fragment2.startsWith("/")) { + sb.append(fragment2.substring(1)); + } else { + sb.append(fragment2); + } + } + + return sb.toString(); } /** Constructs {@link ServerImpl} instance from provided parameters. */ - private ServerImpl newServer( + protected ServerImpl newServer( String protocol, String host, String port, String path, Map attributes) { StringBuilder ub = new StringBuilder(); if (protocol != null) { diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/util/Containers.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/Containers.java similarity index 96% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/util/Containers.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/Containers.java index 4f8cdf018f6..81ed99554f0 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/util/Containers.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/Containers.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.util; +package org.eclipse.che.workspace.infrastructure.kubernetes.util; import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.Quantity; diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/util/KubernetesSize.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSize.java similarity index 97% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/util/KubernetesSize.java rename to infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSize.java index 7692be1d59b..88515a32dc3 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/util/KubernetesSize.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSize.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.util; +package org.eclipse.che.workspace.infrastructure.kubernetes.util; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/AnnotationsTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/AnnotationsTest.java similarity index 98% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/AnnotationsTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/AnnotationsTest.java index ea335d4daa2..9beb9fe93e6 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/AnnotationsTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/AnnotationsTest.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift; +package org.eclipse.che.workspace.infrastructure.kubernetes; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java new file mode 100644 index 00000000000..0fe7217f4d1 --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; + +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.InstallerServersPortProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.RamLimitProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.restartpolicy.RestartPolicyRewriter; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.server.ServersConverter; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.testng.MockitoTestNGListener; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +/** + * Tests {@link KubernetesEnvironmentProvisioner}. + * + * @author Anton Korneta + */ +@Listeners(MockitoTestNGListener.class) +public class KubernetesEnvironmentProvisionerTest { + + @Mock private WorkspaceVolumesStrategy volumesStrategy; + @Mock private InstallerServersPortProvisioner installerServersPortProvisioner; + @Mock private UniqueNamesProvisioner uniqueNamesProvisioner; + @Mock private KubernetesEnvironment k8sEnv; + @Mock private RuntimeIdentity runtimeIdentity; + @Mock private EnvVarsConverter envVarsProvisioner; + @Mock private ServersConverter serversProvisioner; + @Mock private RestartPolicyRewriter restartPolicyRewriter; + @Mock private RamLimitProvisioner ramLimitProvisioner; + @Mock private LogsVolumeMachineProvisioner logsVolumeMachineProvisioner; + + private KubernetesEnvironmentProvisioner osInfraProvisioner; + + private InOrder provisionOrder; + + @BeforeMethod + public void setUp() { + osInfraProvisioner = + new KubernetesEnvironmentProvisioner( + true, + uniqueNamesProvisioner, + serversProvisioner, + envVarsProvisioner, + restartPolicyRewriter, + volumesStrategy, + ramLimitProvisioner, + installerServersPortProvisioner, + logsVolumeMachineProvisioner); + provisionOrder = + inOrder( + installerServersPortProvisioner, + logsVolumeMachineProvisioner, + volumesStrategy, + uniqueNamesProvisioner, + serversProvisioner, + envVarsProvisioner, + restartPolicyRewriter, + ramLimitProvisioner); + } + + @Test + public void performsOrderedProvisioning() throws Exception { + osInfraProvisioner.provision(k8sEnv, runtimeIdentity); + + provisionOrder + .verify(installerServersPortProvisioner) + .provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verify(logsVolumeMachineProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verify(serversProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verify(envVarsProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verify(volumesStrategy).provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verify(restartPolicyRewriter).provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verify(uniqueNamesProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verify(ramLimitProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verifyNoMoreInteractions(); + } +} diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java new file mode 100644 index 00000000000..6b6426f2b48 --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java @@ -0,0 +1,496 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.eclipse.che.api.core.model.workspace.runtime.MachineStatus.FAILED; +import static org.eclipse.che.api.core.model.workspace.runtime.MachineStatus.RUNNING; +import static org.eclipse.che.api.core.model.workspace.runtime.MachineStatus.STARTING; +import static org.eclipse.che.dto.server.DtoFactory.newDto; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_ORIGINAL_NAME_LABEL; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.ContainerPort; +import io.fabric8.kubernetes.api.model.ContainerPortBuilder; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.IntOrStringBuilder; +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodSpec; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServicePort; +import io.fabric8.kubernetes.api.model.ServicePortBuilder; +import io.fabric8.kubernetes.api.model.ServiceSpec; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.api.model.extensions.IngressBackend; +import io.fabric8.kubernetes.api.model.extensions.IngressRule; +import io.fabric8.kubernetes.api.model.extensions.IngressSpec; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.eclipse.che.api.core.model.workspace.runtime.MachineStatus; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.installer.server.model.impl.InstallerImpl; +import org.eclipse.che.api.workspace.server.DtoConverter; +import org.eclipse.che.api.workspace.server.URLRewriter; +import org.eclipse.che.api.workspace.server.hc.ServersChecker; +import org.eclipse.che.api.workspace.server.hc.ServersCheckerFactory; +import org.eclipse.che.api.workspace.server.hc.probe.ProbeScheduler; +import org.eclipse.che.api.workspace.server.hc.probe.WorkspaceProbes; +import org.eclipse.che.api.workspace.server.hc.probe.WorkspaceProbesFactory; +import org.eclipse.che.api.workspace.server.model.impl.RuntimeIdentityImpl; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; +import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; +import org.eclipse.che.api.workspace.shared.dto.event.MachineLogEvent; +import org.eclipse.che.api.workspace.shared.dto.event.MachineStatusEvent; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInternalRuntime.MachineLogsPublisher; +import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapper; +import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesIngresses; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPods; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesServices; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.ContainerEvent; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerResolver; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests {@link KubernetesInternalRuntime}. + * + * @author Anton Korneta + */ +public class KubernetesInternalRuntimeTest { + private static final int EXPOSED_PORT_1 = 4401; + private static final int EXPOSED_PORT_2 = 8081; + private static final int INTERNAL_PORT = 4411; + + private static final String WORKSPACE_ID = "workspace123"; + private static final String POD_NAME = "app"; + private static final String INGRESS_NAME = "test-ingress"; + private static final String SERVICE_NAME = "test-service"; + private static final String POD_SELECTOR = "che.pod.name"; + private static final String CONTAINER_NAME_1 = "test1"; + private static final String CONTAINER_NAME_2 = "test2"; + + private static final String INGRESS_HOST = "localhost"; + + private static final String M1_NAME = POD_NAME + '/' + CONTAINER_NAME_1; + private static final String M2_NAME = POD_NAME + '/' + CONTAINER_NAME_2; + + private static final RuntimeIdentity IDENTITY = + new RuntimeIdentityImpl(WORKSPACE_ID, "env1", "usr1"); + + @Mock private KubernetesRuntimeContext context; + @Mock private EventService eventService; + @Mock private ServersCheckerFactory serverCheckerFactory; + @Mock private ServersChecker serversChecker; + @Mock private KubernetesBootstrapperFactory bootstrapperFactory; + @Mock private KubernetesEnvironment k8sEnv; + @Mock private KubernetesNamespace namespace; + @Mock private KubernetesServices services; + @Mock private KubernetesIngresses ingresses; + @Mock private KubernetesPods pods; + @Mock private KubernetesBootstrapper bootstrapper; + @Mock private WorkspaceVolumesStrategy volumesStrategy; + @Mock private WorkspaceProbesFactory workspaceProbesFactory; + @Mock private ProbeScheduler probesScheduler; + @Mock private WorkspaceProbes workspaceProbes; + @Mock private KubernetesServerResolver kubernetesServerResolver; + + @Captor private ArgumentCaptor machineStatusEventCaptor; + + private KubernetesInternalRuntime internalRuntime; + + private Map allServices; + private Map allIngresses; + + @BeforeMethod + public void setup() throws Exception { + MockitoAnnotations.initMocks(this); + internalRuntime = + new KubernetesInternalRuntime( + 13, + new URLRewriter.NoOpURLRewriter(), + eventService, + bootstrapperFactory, + serverCheckerFactory, + volumesStrategy, + probesScheduler, + workspaceProbesFactory, + context, + namespace, + emptyList()); + when(context.getEnvironment()).thenReturn(k8sEnv); + when(serverCheckerFactory.create(any(), anyString(), any())).thenReturn(serversChecker); + when(context.getIdentity()).thenReturn(IDENTITY); + doNothing().when(namespace).cleanUp(); + when(namespace.services()).thenReturn(services); + when(namespace.ingresses()).thenReturn(ingresses); + when(namespace.pods()).thenReturn(pods); + when(bootstrapperFactory.create(any(), anyList(), any())).thenReturn(bootstrapper); + doReturn( + ImmutableMap.of( + M1_NAME, + mockMachine(mockInstaller("ws-agent")), + M2_NAME, + mockMachine(mockInstaller("terminal")))) + .when(k8sEnv) + .getMachines(); + allServices = ImmutableMap.of(SERVICE_NAME, mockService()); + Ingress ingress = mockIngress(); + allIngresses = ImmutableMap.of(INGRESS_NAME, ingress); + final Container container = mockContainer(CONTAINER_NAME_1, EXPOSED_PORT_1, INTERNAL_PORT); + final ImmutableMap allPods = + ImmutableMap.of(POD_NAME, mockPod(ImmutableList.of(container))); + when(services.create(any())).thenAnswer(a -> a.getArguments()[0]); + when(ingresses.create(any())).thenAnswer(a -> a.getArguments()[0]); + when(ingresses.wait(any(), anyInt(), any())).thenReturn(ingress); + when(pods.create(any())).thenAnswer(a -> a.getArguments()[0]); + when(k8sEnv.getServices()).thenReturn(allServices); + when(k8sEnv.getIngresses()).thenReturn(allIngresses); + when(k8sEnv.getPods()).thenReturn(allPods); + } + + @Test + public void startsKubernetesEnvironment() throws Exception { + final Container container1 = mockContainer(CONTAINER_NAME_1, EXPOSED_PORT_1); + final Container container2 = mockContainer(CONTAINER_NAME_2, EXPOSED_PORT_2, INTERNAL_PORT); + final ImmutableMap allPods = + ImmutableMap.of(POD_NAME, mockPod(ImmutableList.of(container1, container2))); + when(k8sEnv.getPods()).thenReturn(allPods); + + internalRuntime.internalStart(emptyMap()); + + verify(pods).create(any()); + verify(ingresses).create(any()); + verify(services).create(any()); + verify(bootstrapper, times(2)).bootstrap(); + verify(eventService, times(4)).publish(any()); + verifyEventsOrder( + newEvent(M1_NAME, STARTING), + newEvent(M2_NAME, STARTING), + newEvent(M1_NAME, RUNNING), + newEvent(M2_NAME, RUNNING)); + verify(serverCheckerFactory).create(IDENTITY, M1_NAME, emptyMap()); + verify(serverCheckerFactory).create(IDENTITY, M2_NAME, emptyMap()); + verify(serversChecker, times(2)).startAsync(any()); + } + + @Test(expectedExceptions = InternalInfrastructureException.class) + public void throwsInternalInfrastructureExceptionWhenRuntimeErrorOccurs() throws Exception { + doNothing().when(namespace).cleanUp(); + when(k8sEnv.getServices()).thenThrow(new RuntimeException()); + + try { + internalRuntime.internalStart(emptyMap()); + } catch (Exception rethrow) { + verify(namespace).cleanUp(); + verify(namespace, never()).services(); + verify(namespace, never()).ingresses(); + verify(namespace, never()).pods(); + throw rethrow; + } + } + + @Test(expectedExceptions = InfrastructureException.class) + public void stopsWaitingAllMachineStartWhenOneMachineStartFailed() throws Exception { + final Container container1 = mockContainer(CONTAINER_NAME_1, EXPOSED_PORT_1); + final Container container2 = mockContainer(CONTAINER_NAME_2, EXPOSED_PORT_2, INTERNAL_PORT); + final ImmutableMap allPods = + ImmutableMap.of(POD_NAME, mockPod(ImmutableList.of(container1, container2))); + when(k8sEnv.getPods()).thenReturn(allPods); + doThrow(InfrastructureException.class).when(bootstrapper).bootstrap(); + + try { + internalRuntime.internalStart(emptyMap()); + } catch (Exception rethrow) { + verify(pods).create(any()); + verify(ingresses).create(any()); + verify(services).create(any()); + verify(bootstrapper).bootstrap(); + verify(eventService, times(4)).publish(any()); + verifyEventsOrder( + newEvent(M1_NAME, STARTING), + newEvent(M2_NAME, STARTING), + newEvent(M1_NAME, RUNNING), + newEvent(M1_NAME, FAILED)); + throw rethrow; + } + } + + @Test(expectedExceptions = InfrastructureException.class) + public void throwsInfrastructureExceptionWhenErrorOccursAndCleanupFailed() throws Exception { + doNothing().doThrow(InfrastructureException.class).when(namespace).cleanUp(); + when(k8sEnv.getServices()).thenReturn(singletonMap("testService", mock(Service.class))); + when(services.create(any())).thenThrow(new InfrastructureException("service creation failed")); + doThrow(InfrastructureException.class).when(namespace).services(); + + try { + internalRuntime.internalStart(emptyMap()); + } catch (Exception rethrow) { + verify(namespace).cleanUp(); + verify(namespace).services(); + verify(namespace, never()).ingresses(); + verify(namespace, never()).pods(); + throw rethrow; + } + } + + @Test(expectedExceptions = InfrastructureException.class) + public void throwsInfrastructureExceptionWhenBootstrapInterrupted() throws Exception { + doThrow(InterruptedException.class).when(bootstrapper).bootstrap(); + + try { + internalRuntime.internalStart(emptyMap()); + } catch (Exception rethrow) { + verify(namespace).cleanUp(); + verify(pods).create(any()); + verify(ingresses).create(any()); + verify(services).create(any()); + verify(bootstrapper).bootstrap(); + verifyEventsOrder(newEvent(M1_NAME, STARTING), newEvent(M1_NAME, RUNNING)); + throw rethrow; + } + } + + @Test + public void stopsKubernetesEnvironment() throws Exception { + doNothing().when(namespace).cleanUp(); + + internalRuntime.internalStop(emptyMap()); + + verify(namespace).cleanUp(); + } + + @Test(expectedExceptions = InfrastructureException.class) + public void throwsInfrastructureExceptionWhenKubernetesNamespaceCleanupFailed() throws Exception { + doThrow(InfrastructureException.class).when(namespace).cleanUp(); + + internalRuntime.internalStop(emptyMap()); + } + + @Test + public void testRepublishContainerOutputAsMachineLogEvents() throws Exception { + final MachineLogsPublisher logsPublisher = internalRuntime.new MachineLogsPublisher(); + final ContainerEvent out1 = mockContainerEvent("pulling image", "07/07/2007 19:01:22"); + final ContainerEvent out2 = mockContainerEvent("image pulled", "07/07/2007 19:08:53"); + final ArgumentCaptor captor = ArgumentCaptor.forClass(MachineLogEvent.class); + + internalRuntime.doStartMachine(kubernetesServerResolver); + logsPublisher.handle(out1); + logsPublisher.handle(out2); + + verify(eventService, atLeastOnce()).publish(captor.capture()); + final ImmutableList machineLogs = + ImmutableList.of(asMachineLogEvent(out1), asMachineLogEvent(out2)); + assertTrue(captor.getAllValues().containsAll(machineLogs)); + } + + @Test + public void testDoNotPublishForeignMachineOutput() throws Exception { + final MachineLogsPublisher logsPublisher = internalRuntime.new MachineLogsPublisher(); + final ContainerEvent out1 = mockContainerEvent("folder created", "33/03/2033 19:01:06"); + + logsPublisher.handle(out1); + + verify(eventService, never()).publish(any()); + } + + @Test + public void cancelsWsProbesOnRuntimeStop() throws Exception { + doNothing().when(namespace).cleanUp(); + + internalRuntime.internalStop(emptyMap()); + + verify(probesScheduler).cancel(WORKSPACE_ID); + } + + @Test + public void cancelsWsProbesWhenErrorOnRuntimeStartOccurs() throws Exception { + doNothing().when(namespace).cleanUp(); + when(k8sEnv.getServices()).thenThrow(new RuntimeException()); + + try { + internalRuntime.internalStart(emptyMap()); + } catch (Exception e) { + verify(probesScheduler).cancel(WORKSPACE_ID); + return; + } + fail(); + } + + @Test + public void schedulesProbesOnRuntimeStart() throws Exception { + doNothing().when(namespace).cleanUp(); + when(workspaceProbesFactory.getProbes(eq(WORKSPACE_ID), anyString(), any())) + .thenReturn(workspaceProbes); + + internalRuntime.internalStart(emptyMap()); + + verify(probesScheduler).schedule(eq(workspaceProbes), any()); + } + + private static MachineStatusEvent newEvent(String machineName, MachineStatus status) { + return newDto(MachineStatusEvent.class) + .withIdentity(DtoConverter.asDto(IDENTITY)) + .withMachineName(machineName) + .withEventType(status); + } + + private void verifyEventsOrder(MachineStatusEvent... expectedEvents) { + final Iterator actualEvents = captureEvents().iterator(); + for (MachineStatusEvent expected : expectedEvents) { + if (!actualEvents.hasNext()) { + fail("It is expected to receive machine status events"); + } + final MachineStatusEvent actual = actualEvents.next(); + assertEquals(actual, expected); + } + if (actualEvents.hasNext()) { + fail("No more events expected"); + } + } + + private List captureEvents() { + verify(eventService, atLeastOnce()).publish(machineStatusEventCaptor.capture()); + return machineStatusEventCaptor.getAllValues(); + } + + private static Container mockContainer(String name, int... ports) { + final Container container = mock(Container.class); + when(container.getName()).thenReturn(name); + final List containerPorts = new ArrayList<>(ports.length); + for (int port : ports) { + containerPorts.add(new ContainerPortBuilder().withContainerPort(port).build()); + } + when(container.getPorts()).thenReturn(containerPorts); + return container; + } + + private static Pod mockPod(List containers) { + final Pod pod = mock(Pod.class); + final PodSpec spec = mock(PodSpec.class); + mockName(POD_NAME, pod); + when(spec.getContainers()).thenReturn(containers); + when(pod.getSpec()).thenReturn(spec); + when(pod.getMetadata().getLabels()) + .thenReturn(ImmutableMap.of(POD_SELECTOR, POD_NAME, CHE_ORIGINAL_NAME_LABEL, POD_NAME)); + return pod; + } + + private static Service mockService() { + final Service service = mock(Service.class); + final ServiceSpec spec = mock(ServiceSpec.class); + mockName(SERVICE_NAME, service); + when(service.getSpec()).thenReturn(spec); + when(spec.getSelector()).thenReturn(ImmutableMap.of(POD_SELECTOR, POD_NAME)); + final ServicePort sp1 = + new ServicePortBuilder().withTargetPort(intOrString(EXPOSED_PORT_1)).build(); + final ServicePort sp2 = + new ServicePortBuilder().withTargetPort(intOrString(EXPOSED_PORT_2)).build(); + when(spec.getPorts()).thenReturn(ImmutableList.of(sp1, sp2)); + return service; + } + + private static Ingress mockIngress() { + final Ingress ingress = mock(Ingress.class); + mockName(INGRESS_NAME, ingress); + final IngressSpec spec = mock(IngressSpec.class); + + final IngressBackend backend = mock(IngressBackend.class); + when(backend.getServiceName()).thenReturn(SERVICE_NAME); + when(backend.getServicePort()).thenReturn(new IntOrString(EXPOSED_PORT_1)); + when(spec.getBackend()).thenReturn(backend); + + final IngressRule rule = mock(IngressRule.class); + when(rule.getHost()).thenReturn(INGRESS_HOST); + when(spec.getRules()).thenReturn(singletonList(rule)); + + when(ingress.getSpec()).thenReturn(spec); + when(ingress.getMetadata().getLabels()) + .thenReturn(ImmutableMap.of(CHE_ORIGINAL_NAME_LABEL, INGRESS_NAME)); + return ingress; + } + + private static InstallerImpl mockInstaller(String name) { + InstallerImpl installer = mock(InstallerImpl.class); + when(installer.getName()).thenReturn(name); + return installer; + } + + private static InternalMachineConfig mockMachine(InstallerImpl... installers) { + final InternalMachineConfig machine1 = mock(InternalMachineConfig.class); + when(machine1.getInstallers()).thenReturn(Arrays.asList(installers)); + return machine1; + } + + private static ObjectMeta mockName(String name, HasMetadata mock) { + final ObjectMeta metadata = mock(ObjectMeta.class); + when(mock.getMetadata()).thenReturn(metadata); + when(metadata.getName()).thenReturn(name); + return metadata; + } + + private static ContainerEvent mockContainerEvent(String message, String time) { + final ContainerEvent event = mock(ContainerEvent.class); + when(event.getPodName()).thenReturn(POD_NAME); + when(event.getContainerName()).thenReturn(CONTAINER_NAME_1); + when(event.getMessage()).thenReturn(message); + when(event.getTime()).thenReturn(time); + return event; + } + + private static MachineLogEvent asMachineLogEvent(ContainerEvent event) { + return newDto(MachineLogEvent.class) + .withRuntimeId(DtoConverter.asDto(IDENTITY)) + .withText(event.getMessage()) + .withTime(event.getTime()) + .withMachineName(event.getPodName() + '/' + event.getContainerName()); + } + + private static IntOrString intOrString(int port) { + return new IntOrStringBuilder().withIntVal(port).withStrVal(String.valueOf(port)).build(); + } +} diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentFactoryTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentFactoryTest.java new file mode 100644 index 00000000000..1ff97ee45b4 --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentFactoryTest.java @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.environment; + +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static java.util.Arrays.fill; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static org.eclipse.che.api.core.model.workspace.config.MachineConfig.MEMORY_LIMIT_ATTRIBUTE; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.MACHINE_NAME_ANNOTATION_FMT; +import static org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironmentFactory.INGRESSES_IGNORED_WARNING_CODE; +import static org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironmentFactory.INGRESSES_IGNORED_WARNING_MESSAGE; +import static org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironmentFactory.PVC_IGNORED_WARNING_CODE; +import static org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironmentFactory.PVC_IGNORED_WARNING_MESSAGE; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.DoneableKubernetesList; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.KubernetesList; +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodSpec; +import io.fabric8.kubernetes.api.model.Quantity; +import io.fabric8.kubernetes.api.model.ResourceRequirements; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.KubernetesListMixedOperation; +import io.fabric8.kubernetes.client.dsl.RecreateFromServerGettable; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.eclipse.che.api.workspace.server.model.impl.WarningImpl; +import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironment; +import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; +import org.eclipse.che.api.workspace.server.spi.environment.InternalRecipe; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; +import org.mockito.Mock; +import org.mockito.testng.MockitoTestNGListener; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +/** + * Tests {@link KubernetesEnvironmentFactory}. + * + * @author Anton Korneta + */ +@Listeners(MockitoTestNGListener.class) +public class KubernetesEnvironmentFactoryTest { + + private static final String YAML_RECIPE = "application/x-yaml"; + private static final long DEFAULT_RAM_LIMIT_MB = 2048; + public static final String MACHINE_NAME_1 = "machine1"; + public static final String MACHINE_NAME_2 = "machine2"; + + private KubernetesEnvironmentFactory k8sEnvironmentFactory; + + @Mock private KubernetesClientFactory clientFactory; + @Mock private KubernetesEnvironmentValidator k8sEnvValidator; + @Mock private KubernetesClient client; + @Mock private InternalEnvironment internalEnvironment; + @Mock private InternalRecipe internalRecipe; + @Mock private KubernetesListMixedOperation listMixedOperation; + @Mock private KubernetesList validatedObjects; + @Mock private InternalMachineConfig machineConfig1; + @Mock private InternalMachineConfig machineConfig2; + + private Map machines; + + @Mock + private RecreateFromServerGettable + serverGettable; + + @BeforeMethod + public void setup() throws Exception { + k8sEnvironmentFactory = + new KubernetesEnvironmentFactory( + null, null, null, clientFactory, k8sEnvValidator, DEFAULT_RAM_LIMIT_MB); + when(clientFactory.create()).thenReturn(client); + when(client.lists()).thenReturn(listMixedOperation); + when(listMixedOperation.load(any(InputStream.class))).thenReturn(serverGettable); + when(serverGettable.get()).thenReturn(validatedObjects); + when(internalEnvironment.getRecipe()).thenReturn(internalRecipe); + when(internalRecipe.getContentType()).thenReturn(YAML_RECIPE); + when(internalRecipe.getContent()).thenReturn("recipe content"); + machines = ImmutableMap.of(MACHINE_NAME_1, machineConfig1, MACHINE_NAME_2, machineConfig2); + } + + @Test + public void ignoreIgressesWhenRecipeContainsThem() throws Exception { + final List objects = asList(new Ingress(), new Ingress()); + when(validatedObjects.getItems()).thenReturn(objects); + + final KubernetesEnvironment parsed = + k8sEnvironmentFactory.doCreate(internalRecipe, emptyMap(), emptyList()); + + assertTrue(parsed.getIngresses().isEmpty()); + assertEquals(parsed.getWarnings().size(), 1); + assertEquals( + parsed.getWarnings().get(0), + new WarningImpl(INGRESSES_IGNORED_WARNING_CODE, INGRESSES_IGNORED_WARNING_MESSAGE)); + } + + @Test + public void ignorePVCsWhenRecipeContainsThem() throws Exception { + final List pvc = singletonList(new PersistentVolumeClaim()); + when(validatedObjects.getItems()).thenReturn(pvc); + + final KubernetesEnvironment parsed = + k8sEnvironmentFactory.doCreate(internalRecipe, emptyMap(), emptyList()); + + assertTrue(parsed.getPersistentVolumeClaims().isEmpty()); + assertEquals(parsed.getWarnings().size(), 1); + assertEquals( + parsed.getWarnings().get(0), + new WarningImpl(PVC_IGNORED_WARNING_CODE, PVC_IGNORED_WARNING_MESSAGE)); + } + + @Test + public void testSetsRamLimitAttributeFromKubernetesResource() throws Exception { + final long firstMachineRamLimit = 3072; + final long secondMachineRamLimit = 1024; + when(machineConfig1.getAttributes()).thenReturn(new HashMap<>()); + when(machineConfig2.getAttributes()).thenReturn(new HashMap<>()); + final Set pods = + ImmutableSet.of( + mockPod(MACHINE_NAME_1, firstMachineRamLimit), + mockPod(MACHINE_NAME_2, secondMachineRamLimit)); + + k8sEnvironmentFactory.addRamLimitAttribute(machines, pods); + + final long[] actual = machinesRam(machines.values()); + final long[] expected = new long[] {firstMachineRamLimit, secondMachineRamLimit}; + assertTrue(Arrays.equals(actual, expected)); + } + + @Test + public void testDoNotOverrideRamLimitAttributeWhenItAlreadyPresent() throws Exception { + final long customRamLimit = 3072; + final Map attributes = + ImmutableMap.of(MEMORY_LIMIT_ATTRIBUTE, String.valueOf(customRamLimit)); + when(machineConfig1.getAttributes()).thenReturn(attributes); + when(machineConfig2.getAttributes()).thenReturn(attributes); + final Pod pod1 = mockPod(MACHINE_NAME_1, 0); + final Pod pod2 = mockPod(MACHINE_NAME_2, 0); + final Set pods = ImmutableSet.of(pod1, pod2); + + k8sEnvironmentFactory.addRamLimitAttribute(machines, pods); + + final long[] actual = machinesRam(machines.values()); + final long[] expected = new long[actual.length]; + fill(expected, customRamLimit); + assertTrue(Arrays.equals(actual, expected)); + } + + @Test + public void testAddsMachineConfIntoEnvAndSetsRamLimAttributeWhenMachinePresentOnlyInRecipe() + throws Exception { + final long customRamLimit = 2048; + final Map machines = new HashMap<>(); + final Set pods = ImmutableSet.of(mockPod(MACHINE_NAME_2, customRamLimit)); + + k8sEnvironmentFactory.addRamLimitAttribute(machines, pods); + + final long[] actual = machinesRam(machines.values()); + final long[] expected = new long[actual.length]; + fill(expected, customRamLimit); + assertTrue(Arrays.equals(actual, expected)); + } + + private static Pod mockPod(String machineName, long ramLimit) { + final String containerName = "container_" + machineName; + final Container containerMock = mock(Container.class); + final ResourceRequirements resourcesMock = mock(ResourceRequirements.class); + final Quantity quantityMock = mock(Quantity.class); + final Pod podMock = mock(Pod.class); + final PodSpec specMock = mock(PodSpec.class); + final ObjectMeta metadataMock = mock(ObjectMeta.class); + when(podMock.getSpec()).thenReturn(specMock); + when(podMock.getMetadata()).thenReturn(metadataMock); + when(quantityMock.getAmount()).thenReturn(String.valueOf(ramLimit)); + when(resourcesMock.getLimits()).thenReturn(ImmutableMap.of("memory", quantityMock)); + when(containerMock.getName()).thenReturn(containerName); + when(containerMock.getResources()).thenReturn(resourcesMock); + when(metadataMock.getAnnotations()) + .thenReturn( + ImmutableMap.of(format(MACHINE_NAME_ANNOTATION_FMT, containerName), machineName)); + when(specMock.getContainers()).thenReturn(ImmutableList.of(containerMock)); + return podMock; + } + + private static long[] machinesRam(Collection configs) { + return configs + .stream() + .mapToLong(m -> Long.parseLong(m.getAttributes().get(MEMORY_LIMIT_ATTRIBUTE))) + .toArray(); + } +} diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentValidatorTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentValidatorTest.java similarity index 77% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentValidatorTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentValidatorTest.java index 7b01330807a..99de74e5419 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentValidatorTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/KubernetesEnvironmentValidatorTest.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.environment; +package org.eclipse.che.workspace.infrastructure.kubernetes.environment; import static java.util.Collections.emptyMap; import static org.mockito.Mockito.mock; @@ -30,19 +30,19 @@ import org.testng.annotations.Test; /** - * Tests {@link OpenShiftEnvironmentValidator}. + * Tests {@link KubernetesEnvironmentValidator}. * * @author Sergii Leshchenko */ @Listeners(MockitoTestNGListener.class) -public class OpenShiftEnvironmentValidatorTest { - @Mock private OpenShiftEnvironment openShiftEnvironment; +public class KubernetesEnvironmentValidatorTest { + @Mock private KubernetesEnvironment kubernetesEnvironment; - private OpenShiftEnvironmentValidator environmentValidator; + private KubernetesEnvironmentValidator environmentValidator; @BeforeMethod public void setUp() throws Exception { - environmentValidator = new OpenShiftEnvironmentValidator(); + environmentValidator = new KubernetesEnvironmentValidator(); } @Test( @@ -51,10 +51,10 @@ public void setUp() throws Exception { ) public void shouldThrowExceptionWhenEnvDoesNotHaveAnyPods() throws Exception { // given - when(openShiftEnvironment.getPods()).thenReturn(emptyMap()); + when(kubernetesEnvironment.getPods()).thenReturn(emptyMap()); // when - environmentValidator.validate(openShiftEnvironment); + environmentValidator.validate(kubernetesEnvironment); } @Test( @@ -62,16 +62,16 @@ public void shouldThrowExceptionWhenEnvDoesNotHaveAnyPods() throws Exception { expectedExceptionsMessageRegExp = "Environment contains machines that are missing in recipe: pod1/db" ) - public void shouldThrowExceptionWhenMachineIsDeclaredButThereIsNotContainerInOpenShiftRecipe() + public void shouldThrowExceptionWhenMachineIsDeclaredButThereIsNotContainerInKubernetesRecipe() throws Exception { // given Pod pod = createPod("pod1", "main"); - when(openShiftEnvironment.getPods()).thenReturn(ImmutableMap.of("pod1", pod)); - when(openShiftEnvironment.getMachines()) + when(kubernetesEnvironment.getPods()).thenReturn(ImmutableMap.of("pod1", pod)); + when(kubernetesEnvironment.getMachines()) .thenReturn(ImmutableMap.of("pod1/db", mock(InternalMachineConfig.class))); // when - environmentValidator.validate(openShiftEnvironment); + environmentValidator.validate(kubernetesEnvironment); } private Pod createPod(String name, String... containers) { diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/convert/DockerImageEnvironmentConverterTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/convert/DockerImageEnvironmentConverterTest.java similarity index 82% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/convert/DockerImageEnvironmentConverterTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/convert/DockerImageEnvironmentConverterTest.java index ad3873dac77..6caf317e2ef 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/convert/DockerImageEnvironmentConverterTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/environment/convert/DockerImageEnvironmentConverterTest.java @@ -8,13 +8,13 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.environment.convert; +package org.eclipse.che.workspace.infrastructure.kubernetes.environment.convert; import static java.lang.String.format; import static java.util.Collections.emptyMap; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.MACHINE_NAME_ANNOTATION_FMT; -import static org.eclipse.che.workspace.infrastructure.openshift.environment.convert.DockerImageEnvironmentConverter.CONTAINER_NAME; -import static org.eclipse.che.workspace.infrastructure.openshift.environment.convert.DockerImageEnvironmentConverter.POD_NAME; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.MACHINE_NAME_ANNOTATION_FMT; +import static org.eclipse.che.workspace.infrastructure.kubernetes.environment.convert.DockerImageEnvironmentConverter.CONTAINER_NAME; +import static org.eclipse.che.workspace.infrastructure.kubernetes.environment.convert.DockerImageEnvironmentConverter.POD_NAME; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.AssertJUnit.assertEquals; @@ -29,7 +29,7 @@ import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; import org.eclipse.che.api.workspace.server.spi.environment.InternalRecipe; import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; import org.testng.annotations.BeforeMethod; @@ -73,11 +73,11 @@ public void setup() throws Exception { } @Test - public void testConvertsDockerImageEnvironment2OpenShiftEnvironment() throws Exception { + public void testConvertsDockerImageEnvironment2KubernetesEnvironment() throws Exception { when(dockerEnv.getMachines()).thenReturn(machines); when(dockerEnv.getRecipe()).thenReturn(recipe); - final OpenShiftEnvironment actual = converter.convert(dockerEnv); + final KubernetesEnvironment actual = converter.convert(dockerEnv); assertEquals(pod, actual.getPods().values().iterator().next()); assertEquals(recipe, actual.getRecipe()); diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespaceTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespaceTest.java new file mode 100644 index 00000000000..ff18c2c62b2 --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/KubernetesNamespaceTest.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +import io.fabric8.kubernetes.api.model.DoneableNamespace; +import io.fabric8.kubernetes.api.model.Namespace; +import io.fabric8.kubernetes.api.model.NamespaceFluent.MetadataNested; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; +import io.fabric8.kubernetes.client.dsl.Resource; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; +import org.mockito.Mock; +import org.mockito.testng.MockitoTestNGListener; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +/** + * Tests {@link KubernetesNamespace} + * + * @author Sergii Leshchenko + */ +@Listeners(MockitoTestNGListener.class) +public class KubernetesNamespaceTest { + + public static final String NAMESPACE = "testNamespace"; + public static final String WORKSPACE_ID = "workspace123"; + + @Mock private KubernetesPods pods; + @Mock private KubernetesServices services; + @Mock private KubernetesIngresses ingresses; + @Mock private KubernetesPersistentVolumeClaims pvcs; + @Mock private KubernetesClientFactory clientFactory; + @Mock private KubernetesClient kubernetesClient; + @Mock private NonNamespaceOperation namespaceOperation; + + private KubernetesNamespace k8sNamespace; + + @BeforeMethod + public void setUp() throws Exception { + when(clientFactory.create()).thenReturn(kubernetesClient); + + doReturn(namespaceOperation).when(kubernetesClient).namespaces(); + + k8sNamespace = new KubernetesNamespace(WORKSPACE_ID, pods, services, pvcs, ingresses); + } + + @Test + public void testKubernetesNamespaceCreationWhenNamespaceExists() throws Exception { + // given + prepareNamespace(NAMESPACE); + + // when + new KubernetesNamespace(clientFactory, NAMESPACE, WORKSPACE_ID); + } + + @Test + public void testKubernetesNamespaceCreationWhenNamespaceDoesNotExist() throws Exception { + // given + MetadataNested namespaceMeta = prepareCreateNamespaceRequest(); + + Resource resource = prepareNamespaceResource(NAMESPACE); + doThrow(new KubernetesClientException("error", 403, null)).when(resource).get(); + + // when + KubernetesNamespace namespace = new KubernetesNamespace(clientFactory, NAMESPACE, WORKSPACE_ID); + + // then + verify(namespaceMeta).withName(NAMESPACE); + } + + @Test + public void testKubernetesNamespaceCleaningUp() throws Exception { + // when + k8sNamespace.cleanUp(); + + verify(ingresses).delete(); + verify(services).delete(); + verify(pods).delete(); + } + + @Test + public void testKubernetesNamespaceCleaningUpIfExceptionsOccurs() throws Exception { + doThrow(new InfrastructureException("err1.")).when(services).delete(); + doThrow(new InfrastructureException("err2.")).when(pods).delete(); + + InfrastructureException error = null; + // when + try { + k8sNamespace.cleanUp(); + + } catch (InfrastructureException e) { + error = e; + } + + // then + assertNotNull(error); + String message = error.getMessage(); + assertEquals(message, "Error(s) occurs while cleaning up the namespace. err1. err2."); + verify(ingresses).delete(); + } + + private MetadataNested prepareCreateNamespaceRequest() { + DoneableNamespace namespace = mock(DoneableNamespace.class); + MetadataNested metadataNested = mock(MetadataNested.class); + + doReturn(namespace).when(namespaceOperation).createNew(); + doReturn(metadataNested).when(namespace).withNewMetadata(); + doReturn(metadataNested).when(metadataNested).withName(anyString()); + doReturn(namespace).when(metadataNested).endMetadata(); + return metadataNested; + } + + private Resource prepareNamespaceResource(String namespaceName) { + Resource namespaceResource = mock(Resource.class); + doReturn(namespaceResource).when(namespaceOperation).withName(namespaceName); + kubernetesClient.namespaces().withName(namespaceName).get(); + return namespaceResource; + } + + private void prepareNamespace(String namespaceName) { + Namespace namespace = mock(Namespace.class); + Resource namespaceResource = prepareNamespaceResource(namespaceName); + doReturn(namespace).when(namespaceResource).get(); + } +} diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemoveTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemoveTest.java new file mode 100644 index 00000000000..bd210a98e16 --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemoveTest.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.eclipse.che.api.core.model.workspace.Workspace; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.workspace.shared.event.WorkspaceRemovedEvent; +import org.mockito.Mock; +import org.mockito.testng.MockitoTestNGListener; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +/** + * Test for {@link RemoveNamespaceOnWorkspaceRemove}. + * + * @author Sergii Leshchenko + */ +@Listeners(MockitoTestNGListener.class) +public class RemoveNamespaceOnWorkspaceRemoveTest { + + private static final String WORKSPACE_ID = "workspace123"; + + @Mock private Workspace workspace; + + private RemoveNamespaceOnWorkspaceRemove removeNamespaceOnWorkspaceRemove; + + @BeforeMethod + public void setUp() throws Exception { + removeNamespaceOnWorkspaceRemove = spy(new RemoveNamespaceOnWorkspaceRemove(null, null)); + + doNothing().when(removeNamespaceOnWorkspaceRemove).doRemoveNamespace(anyString()); + + when(workspace.getId()).thenReturn(WORKSPACE_ID); + } + + @Test + public void shouldSubscribeListenerToEventService() { + EventService eventService = mock(EventService.class); + + removeNamespaceOnWorkspaceRemove.subscribe(eventService); + + verify(eventService).subscribe(removeNamespaceOnWorkspaceRemove); + } + + @Test + public void shouldRemoveNamespaceOnWorkspaceRemovedEvent() throws Exception { + removeNamespaceOnWorkspaceRemove.onEvent(new WorkspaceRemovedEvent(workspace)); + + verify(removeNamespaceOnWorkspaceRemove).doRemoveNamespace(WORKSPACE_ID); + } +} diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/CommonPVCStrategyTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/CommonPVCStrategyTest.java similarity index 79% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/CommonPVCStrategyTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/CommonPVCStrategyTest.java index ce940b1647f..a28b6a2dc1e 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/CommonPVCStrategyTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/CommonPVCStrategyTest.java @@ -8,14 +8,14 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; import static java.lang.String.format; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static org.eclipse.che.workspace.infrastructure.openshift.project.pvc.CommonPVCStrategy.SUBPATHS_PROPERTY_FMT; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy.SUBPATHS_PROPERTY_FMT; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doNothing; @@ -45,10 +45,10 @@ import org.eclipse.che.api.workspace.server.model.impl.VolumeImpl; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftPersistentVolumeClaims; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProjectFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPersistentVolumeClaims; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; import org.testng.annotations.BeforeMethod; @@ -90,11 +90,11 @@ public class CommonPVCStrategyTest { @Mock private Container container; @Mock private Container container2; @Mock private Container container3; - @Mock private OpenShiftEnvironment osEnv; + @Mock private KubernetesEnvironment k8sEnv; @Mock private PVCSubPathHelper pvcSubPathHelper; - @Mock private OpenShiftProjectFactory factory; - @Mock private OpenShiftProject osProject; - @Mock private OpenShiftPersistentVolumeClaims pvcs; + @Mock private KubernetesNamespaceFactory factory; + @Mock private KubernetesNamespace k8sNamespace; + @Mock private KubernetesPersistentVolumeClaims pvcs; private CommonPVCStrategy commonPVCStrategy; @@ -121,12 +121,12 @@ public void setup() throws Exception { volumes3.put(VOLUME_1_NAME, new VolumeImpl().withPath("/path")); when(machine3.getVolumes()).thenReturn(volumes3); machines.put(MACHINE_3_NAME, machine3); - when(osEnv.getMachines()).thenReturn(machines); + when(k8sEnv.getMachines()).thenReturn(machines); Map pods = new HashMap<>(); pods.put(POD_NAME, pod); pods.put(POD_NAME_2, pod2); - when(osEnv.getPods()).thenReturn(pods); + when(k8sEnv.getPods()).thenReturn(pods); when(pod.getSpec()).thenReturn(podSpec); when(pod2.getSpec()).thenReturn(podSpec2); @@ -142,21 +142,21 @@ public void setup() throws Exception { when(container3.getVolumeMounts()).thenReturn(new ArrayList<>()); doNothing().when(pvcSubPathHelper).execute(any(), any(), any()); - when(osEnv.getPersistentVolumeClaims()).thenReturn(new HashMap<>()); + when(k8sEnv.getPersistentVolumeClaims()).thenReturn(new HashMap<>()); when(pvcSubPathHelper.removeDirsAsync(anyString(), any(String.class))) .thenReturn(CompletableFuture.completedFuture(null)); - when(factory.create(WORKSPACE_ID)).thenReturn(osProject); - when(osProject.persistentVolumeClaims()).thenReturn(pvcs); + when(factory.create(WORKSPACE_ID)).thenReturn(k8sNamespace); + when(k8sNamespace.persistentVolumeClaims()).thenReturn(pvcs); mockName(pod, POD_NAME); mockName(pod2, POD_NAME_2); } @Test - public void testProvisionVolumesIntoOpenShiftEnvironment() throws Exception { - when(osEnv.getPersistentVolumeClaims()).thenReturn(new HashMap<>()); + public void testProvisionVolumesIntoKubernetesEnvironment() throws Exception { + when(k8sEnv.getPersistentVolumeClaims()).thenReturn(new HashMap<>()); - commonPVCStrategy.provision(osEnv, IDENTITY); + commonPVCStrategy.provision(k8sEnv, IDENTITY); // 2 volumes in machine1 verify(container, times(2)).getVolumeMounts(); @@ -171,27 +171,27 @@ public void testProvisionVolumesIntoOpenShiftEnvironment() throws Exception { assertFalse(container.getVolumeMounts().isEmpty()); assertFalse(container2.getVolumeMounts().isEmpty()); assertFalse(container3.getVolumeMounts().isEmpty()); - assertFalse(osEnv.getPersistentVolumeClaims().isEmpty()); - assertTrue(osEnv.getPersistentVolumeClaims().containsKey(PVC_NAME)); + assertFalse(k8sEnv.getPersistentVolumeClaims().isEmpty()); + assertTrue(k8sEnv.getPersistentVolumeClaims().containsKey(PVC_NAME)); } @Test - public void testReplacePVCWhenItsAlreadyInOpenShiftEnvironment() throws Exception { + public void testReplacePVCWhenItsAlreadyInKubernetesEnvironment() throws Exception { final Map claims = new HashMap<>(); final PersistentVolumeClaim provisioned = mock(PersistentVolumeClaim.class); claims.put(PVC_NAME, provisioned); - when(osEnv.getPersistentVolumeClaims()).thenReturn(claims); + when(k8sEnv.getPersistentVolumeClaims()).thenReturn(claims); - commonPVCStrategy.provision(osEnv, IDENTITY); + commonPVCStrategy.provision(k8sEnv, IDENTITY); - assertNotEquals(osEnv.getPersistentVolumeClaims().get(PVC_NAME), provisioned); + assertNotEquals(k8sEnv.getPersistentVolumeClaims().get(PVC_NAME), provisioned); } @Test - public void testProvisionVolumesWithSubpathsIntoOpenShiftEnviromnent() throws Exception { - commonPVCStrategy.provision(osEnv, IDENTITY); + public void testProvisionVolumesWithSubpathsIntoKubernetesEnviromnent() throws Exception { + commonPVCStrategy.provision(k8sEnv, IDENTITY); - final Map actual = osEnv.getPersistentVolumeClaims(); + final Map actual = k8sEnv.getPersistentVolumeClaims(); assertFalse(actual.isEmpty()); assertTrue(actual.containsKey(PVC_NAME)); assertEquals( @@ -209,9 +209,9 @@ public void testDoNotAddsSubpathsWhenPreCreationIsNotNeeded() throws Exception { new CommonPVCStrategy( PVC_NAME, PVC_QUANTITY, PVC_ACCESS_MODE, false, pvcSubPathHelper, factory); - commonPVCStrategy.provision(osEnv, IDENTITY); + commonPVCStrategy.provision(k8sEnv, IDENTITY); - final Map actual = osEnv.getPersistentVolumeClaims(); + final Map actual = k8sEnv.getPersistentVolumeClaims(); assertFalse(actual.isEmpty()); assertTrue(actual.containsKey(PVC_NAME)); assertFalse( @@ -224,13 +224,13 @@ public void testDoNotAddsSubpathsWhenPreCreationIsNotNeeded() throws Exception { @Test public void testCreatesPVCsWithSubpathsOnPrepare() throws Exception { final PersistentVolumeClaim pvc = mockName(mock(PersistentVolumeClaim.class), PVC_NAME); - when(osEnv.getPersistentVolumeClaims()).thenReturn(singletonMap(PVC_NAME, pvc)); + when(k8sEnv.getPersistentVolumeClaims()).thenReturn(singletonMap(PVC_NAME, pvc)); final Map subPaths = new HashMap<>(); subPaths.put(format(SUBPATHS_PROPERTY_FMT, WORKSPACE_ID), WORKSPACE_SUBPATHS); when(pvc.getAdditionalProperties()).thenReturn(subPaths); doNothing().when(pvcSubPathHelper).createDirs(WORKSPACE_ID, WORKSPACE_SUBPATHS); - commonPVCStrategy.prepare(osEnv, WORKSPACE_ID); + commonPVCStrategy.prepare(k8sEnv, WORKSPACE_ID); verify(pvcs).get(); verify(pvcs).create(pvc); @@ -239,21 +239,21 @@ public void testCreatesPVCsWithSubpathsOnPrepare() throws Exception { @Test(expectedExceptions = InfrastructureException.class) public void throwsInfrastructureExceptionWhenFailedToGetExistingPVCs() throws Exception { - when(osEnv.getPersistentVolumeClaims()) + when(k8sEnv.getPersistentVolumeClaims()) .thenReturn(singletonMap(PVC_NAME, mock(PersistentVolumeClaim.class))); doThrow(InfrastructureException.class).when(pvcs).get(); - commonPVCStrategy.prepare(osEnv, WORKSPACE_ID); + commonPVCStrategy.prepare(k8sEnv, WORKSPACE_ID); } @Test(expectedExceptions = InfrastructureException.class) public void throwsInfrastructureExceptionWhenPVCCreationFailed() throws Exception { final PersistentVolumeClaim claim = mockName(mock(PersistentVolumeClaim.class), PVC_NAME); - when(osEnv.getPersistentVolumeClaims()).thenReturn(singletonMap(PVC_NAME, claim)); + when(k8sEnv.getPersistentVolumeClaims()).thenReturn(singletonMap(PVC_NAME, claim)); when(pvcs.get()).thenReturn(emptyList()); doThrow(InfrastructureException.class).when(pvcs).create(any()); - commonPVCStrategy.prepare(osEnv, WORKSPACE_ID); + commonPVCStrategy.prepare(k8sEnv, WORKSPACE_ID); } @Test diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/PVCSubPathHelperTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java similarity index 73% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/PVCSubPathHelperTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java index 2aef1ac4ecc..491f2247780 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/PVCSubPathHelperTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java @@ -8,13 +8,13 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; import static java.util.stream.Collectors.toList; -import static org.eclipse.che.workspace.infrastructure.openshift.project.pvc.PVCSubPathHelper.JOB_MOUNT_PATH; -import static org.eclipse.che.workspace.infrastructure.openshift.project.pvc.PVCSubPathHelper.MKDIR_COMMAND_BASE; -import static org.eclipse.che.workspace.infrastructure.openshift.project.pvc.PVCSubPathHelper.POD_PHASE_FAILED; -import static org.eclipse.che.workspace.infrastructure.openshift.project.pvc.PVCSubPathHelper.POD_PHASE_SUCCEEDED; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.PVCSubPathHelper.JOB_MOUNT_PATH; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.PVCSubPathHelper.MKDIR_COMMAND_BASE; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.PVCSubPathHelper.POD_PHASE_FAILED; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.PVCSubPathHelper.POD_PHASE_SUCCEEDED; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; @@ -31,9 +31,9 @@ import java.util.List; import java.util.stream.Stream; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftPods; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProjectFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPods; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; @@ -57,9 +57,9 @@ public class PVCSubPathHelperTest { private static final String PROJECTS_PATH = "/projects"; private static final String M2_PATH = "/.m2"; - @Mock private OpenShiftProjectFactory osProjectFactory; - @Mock private OpenShiftProject osProject; - @Mock private OpenShiftPods osPods; + @Mock private KubernetesNamespaceFactory k8sNamespaceFactory; + @Mock private KubernetesNamespace k8sNamespace; + @Mock private KubernetesPods osPods; @Mock private Pod pod; @Mock private PodStatus podStatus; @@ -69,9 +69,10 @@ public class PVCSubPathHelperTest { @BeforeMethod public void setup() throws Exception { - pvcSubPathHelper = new PVCSubPathHelper(PVC_NAME, jobMemoryLimit, jobImage, osProjectFactory); - when(osProjectFactory.create(anyString())).thenReturn(osProject); - when(osProject.pods()).thenReturn(osPods); + pvcSubPathHelper = + new PVCSubPathHelper(PVC_NAME, jobMemoryLimit, jobImage, k8sNamespaceFactory); + when(k8sNamespaceFactory.create(anyString())).thenReturn(k8sNamespace); + when(k8sNamespace.pods()).thenReturn(osPods); when(pod.getStatus()).thenReturn(podStatus); when(osPods.create(any(Pod.class))).thenReturn(pod); when(osPods.wait(anyString(), anyInt(), any())).thenReturn(pod); @@ -123,25 +124,25 @@ public void testLogErrorWhenJobExecutionFailed() throws Exception { } @Test - public void testLogErrorWhenOpenShiftProjectCreationFailed() throws Exception { - when(osProjectFactory.create(WORKSPACE_ID)) - .thenThrow(new InfrastructureException("OpenShift project creation failed")); + public void testLogErrorWhenKubernetesProjectCreationFailed() throws Exception { + when(k8sNamespaceFactory.create(WORKSPACE_ID)) + .thenThrow(new InfrastructureException("Kubernetes namespace creation failed")); pvcSubPathHelper.execute(WORKSPACE_ID, MKDIR_COMMAND_BASE, WORKSPACE_ID + PROJECTS_PATH); - verify(osProjectFactory).create(WORKSPACE_ID); - verify(osProject, never()).pods(); + verify(k8sNamespaceFactory).create(WORKSPACE_ID); + verify(k8sNamespace, never()).pods(); } @Test - public void testLogErrorWhenOpenShiftPodCreationFailed() throws Exception { + public void testLogErrorWhenKubernetesPodCreationFailed() throws Exception { when(osPods.create(any())) - .thenThrow(new InfrastructureException("OpenShift pod creation failed")); + .thenThrow(new InfrastructureException("Kubernetes pod creation failed")); pvcSubPathHelper.execute(WORKSPACE_ID, MKDIR_COMMAND_BASE, WORKSPACE_ID + PROJECTS_PATH); - verify(osProjectFactory).create(WORKSPACE_ID); - verify(osProject).pods(); + verify(k8sNamespaceFactory).create(WORKSPACE_ID); + verify(k8sNamespace).pods(); verify(osPods).create(any()); verify(osPods, never()).wait(anyString(), anyInt(), any()); } diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/UniqueWorkspacePVCStrategyTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/UniqueWorkspacePVCStrategyTest.java similarity index 80% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/UniqueWorkspacePVCStrategyTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/UniqueWorkspacePVCStrategyTest.java index 9dbcba31a5d..26463756bb4 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/UniqueWorkspacePVCStrategyTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/UniqueWorkspacePVCStrategyTest.java @@ -8,14 +8,14 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_VOLUME_NAME_LABEL; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_WORKSPACE_ID_LABEL; -import static org.eclipse.che.workspace.infrastructure.openshift.project.pvc.CommonPVCStrategyTest.mockName; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_VOLUME_NAME_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_WORKSPACE_ID_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategyTest.mockName; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -33,10 +33,10 @@ import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodSpec; +import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; -import io.fabric8.openshift.client.OpenShiftClient; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -46,11 +46,11 @@ import org.eclipse.che.api.workspace.server.model.impl.VolumeImpl; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftPersistentVolumeClaims; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProjectFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPersistentVolumeClaims; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; import org.testng.annotations.BeforeMethod; @@ -66,7 +66,7 @@ public class UniqueWorkspacePVCStrategyTest { private static final String WORKSPACE_ID = "workspace123"; - private static final String PROJECT_NAME = "che"; + private static final String NAMESPACE_NAME = "che"; private static final String PVC_NAME_PREFIX = "che-claim"; private static final String POD_NAME = "main"; private static final String POD_NAME_2 = "second"; @@ -84,12 +84,12 @@ public class UniqueWorkspacePVCStrategyTest { private static final RuntimeIdentity IDENTITY = new RuntimeIdentityImpl(WORKSPACE_ID, "env1", "usr1"); - @Mock private OpenShiftEnvironment osEnv; - @Mock private OpenShiftClientFactory clientFactory; - @Mock private OpenShiftClient client; - @Mock private OpenShiftProjectFactory factory; - @Mock private OpenShiftProject osProject; - @Mock private OpenShiftPersistentVolumeClaims pvcs; + @Mock private KubernetesEnvironment k8sEnv; + @Mock private KubernetesClientFactory clientFactory; + @Mock private KubernetesClient client; + @Mock private KubernetesNamespaceFactory factory; + @Mock private KubernetesNamespace k8sNamespace; + @Mock private KubernetesPersistentVolumeClaims pvcs; @Mock private Pod pod; @Mock private Pod pod2; @Mock private PodSpec podSpec; @@ -104,7 +104,7 @@ public class UniqueWorkspacePVCStrategyTest { public void setup() throws Exception { strategy = new UniqueWorkspacePVCStrategy( - PROJECT_NAME, PVC_NAME_PREFIX, PVC_QUANTITY, PVC_ACCESS_MODE, factory, clientFactory); + NAMESPACE_NAME, PVC_NAME_PREFIX, PVC_QUANTITY, PVC_ACCESS_MODE, factory, clientFactory); when(clientFactory.create()).thenReturn(client); Map machines = new HashMap<>(); @@ -124,12 +124,12 @@ public void setup() throws Exception { volumes3.put(VOLUME_1_NAME, new VolumeImpl().withPath("/path")); when(machine3.getVolumes()).thenReturn(volumes3); machines.put(MACHINE_NAME_3, machine3); - when(osEnv.getMachines()).thenReturn(machines); + when(k8sEnv.getMachines()).thenReturn(machines); Map pods = new HashMap<>(); pods.put(POD_NAME, pod); pods.put(POD_NAME_2, pod2); - when(osEnv.getPods()).thenReturn(pods); + when(k8sEnv.getPods()).thenReturn(pods); when(pod.getSpec()).thenReturn(podSpec); when(pod2.getSpec()).thenReturn(podSpec2); @@ -144,8 +144,8 @@ public void setup() throws Exception { when(container2.getVolumeMounts()).thenReturn(new ArrayList<>()); when(container3.getVolumeMounts()).thenReturn(new ArrayList<>()); - when(factory.create(WORKSPACE_ID)).thenReturn(osProject); - when(osProject.persistentVolumeClaims()).thenReturn(pvcs); + when(factory.create(WORKSPACE_ID)).thenReturn(k8sNamespace); + when(k8sNamespace.persistentVolumeClaims()).thenReturn(pvcs); mockName(pod, POD_NAME); mockName(pod2, POD_NAME_2); @@ -153,17 +153,17 @@ public void setup() throws Exception { @Test public void testProvisionPVCsForEachVolumeWithUniqueName() throws Exception { - when(osEnv.getPersistentVolumeClaims()).thenReturn(new HashMap<>()); + when(k8sEnv.getPersistentVolumeClaims()).thenReturn(new HashMap<>()); - strategy.provision(osEnv, IDENTITY); + strategy.provision(k8sEnv, IDENTITY); assertEquals(podSpec.getVolumes().size(), 2); assertEquals(podSpec2.getVolumes().size(), 1); assertEquals(container.getVolumeMounts().size(), 2); assertEquals(container2.getVolumeMounts().size(), 1); assertEquals(container3.getVolumeMounts().size(), 1); - assertEquals(osEnv.getPersistentVolumeClaims().size(), 2); - for (PersistentVolumeClaim pvc : osEnv.getPersistentVolumeClaims().values()) { + assertEquals(k8sEnv.getPersistentVolumeClaims().size(), 2); + for (PersistentVolumeClaim pvc : k8sEnv.getPersistentVolumeClaims().values()) { String volumeName = pvc.getMetadata().getLabels().get(CHE_VOLUME_NAME_LABEL); assertNotNull(volumeName); assertTrue(volumeName.equals(VOLUME_1_NAME) || volumeName.equals(VOLUME_2_NAME)); @@ -179,17 +179,17 @@ public void testDoNotProvisionPVCsWhenItIsAlreadyProvisionedForGivenVolumeAndWor final String pvcUniqueName2 = PVC_NAME_PREFIX + "-71333"; PersistentVolumeClaim pvc2 = mockPVC(ImmutableMap.of(CHE_VOLUME_NAME_LABEL, VOLUME_2_NAME), pvcUniqueName2); - when(osEnv.getPersistentVolumeClaims()) + when(k8sEnv.getPersistentVolumeClaims()) .thenReturn(ImmutableMap.of(pvcUniqueName1, pvc1, pvcUniqueName2, pvc2)); - strategy.provision(osEnv, IDENTITY); + strategy.provision(k8sEnv, IDENTITY); assertEquals(podSpec.getVolumes().size(), 2); assertEquals(podSpec2.getVolumes().size(), 1); assertEquals(container.getVolumeMounts().size(), 2); assertEquals(container2.getVolumeMounts().size(), 1); assertEquals(container3.getVolumeMounts().size(), 1); - assertEquals(osEnv.getPersistentVolumeClaims().size(), 2); + assertEquals(k8sEnv.getPersistentVolumeClaims().size(), 2); } @Test @@ -204,24 +204,24 @@ public void testDoNotProvisionPVCsWhenItIsAlreadyExistsForGivenVolumeAndWorkspac when(pvcs.getByLabel(CHE_WORKSPACE_ID_LABEL, WORKSPACE_ID)) .thenReturn(ImmutableList.of(pvc1, pvc2)); - strategy.provision(osEnv, IDENTITY); + strategy.provision(k8sEnv, IDENTITY); assertEquals(podSpec.getVolumes().size(), 2); assertEquals(podSpec2.getVolumes().size(), 1); assertEquals(container.getVolumeMounts().size(), 2); assertEquals(container2.getVolumeMounts().size(), 1); assertEquals(container3.getVolumeMounts().size(), 1); - assertTrue(osEnv.getPersistentVolumeClaims().isEmpty()); + assertTrue(k8sEnv.getPersistentVolumeClaims().isEmpty()); } @Test public void testCreatesProvisionedPVCsOnPrepare() throws Exception { final String uniqueName = PVC_NAME_PREFIX + "-3121"; final PersistentVolumeClaim pvc = mockName(mock(PersistentVolumeClaim.class), uniqueName); - when(osEnv.getPersistentVolumeClaims()).thenReturn(singletonMap(uniqueName, pvc)); + when(k8sEnv.getPersistentVolumeClaims()).thenReturn(singletonMap(uniqueName, pvc)); doReturn(pvc).when(pvcs).create(any()); - strategy.prepare(osEnv, WORKSPACE_ID); + strategy.prepare(k8sEnv, WORKSPACE_ID); verify(pvcs).create(any()); } @@ -229,10 +229,10 @@ public void testCreatesProvisionedPVCsOnPrepare() throws Exception { @Test(expectedExceptions = InfrastructureException.class) public void throwsInfrastructureExceptionWhenFailedToCreatePVCs() throws Exception { final PersistentVolumeClaim pvc = mock(PersistentVolumeClaim.class); - when(osEnv.getPersistentVolumeClaims()).thenReturn(singletonMap(PVC_NAME_PREFIX, pvc)); + when(k8sEnv.getPersistentVolumeClaims()).thenReturn(singletonMap(PVC_NAME_PREFIX, pvc)); doThrow(InfrastructureException.class).when(pvcs).create(any(PersistentVolumeClaim.class)); - strategy.prepare(osEnv, WORKSPACE_ID); + strategy.prepare(k8sEnv, WORKSPACE_ID); } @Test @@ -241,7 +241,7 @@ public void testRemovesPVCWhenCleanupCalled() throws Exception { final NonNamespaceOperation namespace = mock(NonNamespaceOperation.class); final FilterWatchListDeletable filterList = mock(FilterWatchListDeletable.class); doReturn(mixedOperation).when(client).persistentVolumeClaims(); - doReturn(namespace).when(mixedOperation).inNamespace(PROJECT_NAME); + doReturn(namespace).when(mixedOperation).inNamespace(NAMESPACE_NAME); doReturn(filterList).when(namespace).withLabel(CHE_WORKSPACE_ID_LABEL, WORKSPACE_ID); when(filterList.delete()).thenReturn(true); @@ -256,7 +256,7 @@ public void testDoNothingWhenNoPVCFoundInNamespaceOnCleanup() throws Exception { final NonNamespaceOperation namespace = mock(NonNamespaceOperation.class); final FilterWatchListDeletable filterList = mock(FilterWatchListDeletable.class); doReturn(mixedOperation).when(client).persistentVolumeClaims(); - doReturn(namespace).when(mixedOperation).inNamespace(PROJECT_NAME); + doReturn(namespace).when(mixedOperation).inNamespace(NAMESPACE_NAME); doReturn(filterList).when(namespace).withLabel(CHE_WORKSPACE_ID_LABEL, WORKSPACE_ID); when(filterList.delete()).thenReturn(false); diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspacePVCCleanerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspacePVCCleanerTest.java similarity index 90% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspacePVCCleanerTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspacePVCCleanerTest.java index 99b3f8f3443..1ad774f3fda 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/pvc/WorkspacePVCCleanerTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/WorkspacePVCCleanerTest.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.project.pvc; +package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -39,7 +39,7 @@ public class WorkspacePVCCleanerTest { private static final String WORKSPACE_ID = "workspace132"; - private static final String PROJECT_NAME = "che"; + private static final String NAMESPACE_NAME = "che"; @Mock private WorkspaceVolumesStrategy pvcStrategy; @Mock private EventService eventService; @@ -48,12 +48,12 @@ public class WorkspacePVCCleanerTest { @BeforeMethod public void setUp() { - workspacePVCCleaner = new WorkspacePVCCleaner(true, PROJECT_NAME, pvcStrategy); + workspacePVCCleaner = new WorkspacePVCCleaner(true, NAMESPACE_NAME, pvcStrategy); } @Test public void testDoNotSubscribesCleanerWhenPVCDisabled() { - workspacePVCCleaner = spy(new WorkspacePVCCleaner(false, PROJECT_NAME, pvcStrategy)); + workspacePVCCleaner = spy(new WorkspacePVCCleaner(false, NAMESPACE_NAME, pvcStrategy)); workspacePVCCleaner.subscribe(eventService); @@ -61,7 +61,7 @@ public void testDoNotSubscribesCleanerWhenPVCDisabled() { } @Test - public void testSubscribesDeleteOpenShiftProjectOnWorkspaceRemove() throws Exception { + public void testSubscribesDeleteKubernetesOnWorkspaceRemove() throws Exception { workspacePVCCleaner.subscribe(eventService); verify(eventService).subscribe(any(EventSubscriber.class)); diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/InstallerServersPortProvisionerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/InstallerServersPortProvisionerTest.java similarity index 96% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/InstallerServersPortProvisionerTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/InstallerServersPortProvisionerTest.java index 8b5542d389c..52e6bde1d67 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/InstallerServersPortProvisionerTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/InstallerServersPortProvisionerTest.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -43,8 +43,8 @@ import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.provision.InstallerServersPortProvisioner.ServersPorts; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.InstallerServersPortProvisioner.ServersPorts; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; import org.testng.annotations.BeforeMethod; @@ -60,7 +60,7 @@ @Listeners(MockitoTestNGListener.class) public class InstallerServersPortProvisionerTest { - @Mock private OpenShiftEnvironment osEnv; + @Mock private KubernetesEnvironment k8sEnv; private InstallerServersPortProvisioner portProvisioner; @@ -74,7 +74,7 @@ public void shouldGroupMachinesByPodsAndLaunchPortsConflictResolvingOnProvisioni throws Exception { // given doNothing().when(portProvisioner).fixInstallersPortsConflicts(anyList()); - when(osEnv.getPods()) + when(k8sEnv.getPods()) .thenReturn( ImmutableMap.of( "pod1", @@ -86,7 +86,7 @@ public void shouldGroupMachinesByPodsAndLaunchPortsConflictResolvingOnProvisioni InternalMachineConfig machine2 = mock(InternalMachineConfig.class); InternalMachineConfig machine3 = mock(InternalMachineConfig.class); - when(osEnv.getMachines()) + when(k8sEnv.getMachines()) .thenReturn( ImmutableMap.of( "pod1/container1", @@ -97,7 +97,7 @@ public void shouldGroupMachinesByPodsAndLaunchPortsConflictResolvingOnProvisioni machine3)); // when - portProvisioner.provision(osEnv, mock(RuntimeIdentity.class)); + portProvisioner.provision(k8sEnv, mock(RuntimeIdentity.class)); // then verify(portProvisioner).fixInstallersPortsConflicts(asList(machine1, machine2)); diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/installer/OpenShiftCheApiEnvVarProviderTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/KubernetesCheApiEnvVarProviderTest.java similarity index 62% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/installer/OpenShiftCheApiEnvVarProviderTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/KubernetesCheApiEnvVarProviderTest.java index 89dc96aaedd..9f41ba19aec 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/installer/OpenShiftCheApiEnvVarProviderTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/KubernetesCheApiEnvVarProviderTest.java @@ -8,41 +8,40 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.installer; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; import static org.testng.Assert.assertEquals; import org.eclipse.che.commons.lang.Pair; -import org.eclipse.che.workspace.infrastructure.openshift.provision.OpenShiftCheApiEnvVarProvider; import org.mockito.testng.MockitoTestNGListener; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Listeners; import org.testng.annotations.Test; /** - * Tests {@link OpenShiftCheApiEnvVarProvider}. + * Tests {@link KubernetesCheApiEnvVarProvider}. * * @author Sergii Leshchenko */ @Listeners(MockitoTestNGListener.class) -public class OpenShiftCheApiEnvVarProviderTest { +public class KubernetesCheApiEnvVarProviderTest { private static final String CHE_SERVER_ENDPOINT = "localhost:8080"; - private OpenShiftCheApiEnvVarProvider openShiftCheApiEnvVarProvider; + private KubernetesCheApiEnvVarProvider kubernetesCheApiEnvVarProvider; @BeforeMethod public void setUp() throws Exception { - openShiftCheApiEnvVarProvider = new OpenShiftCheApiEnvVarProvider(CHE_SERVER_ENDPOINT); + kubernetesCheApiEnvVarProvider = new KubernetesCheApiEnvVarProvider(CHE_SERVER_ENDPOINT); } @Test public void shouldReturnCheApiEnv() throws Exception { // when - Pair cheApiEnv = openShiftCheApiEnvVarProvider.get(null); + Pair cheApiEnv = kubernetesCheApiEnvVarProvider.get(null); // then - assertEquals(cheApiEnv.first, OpenShiftCheApiEnvVarProvider.API_ENDPOINT_URL_VARIABLE); + assertEquals(cheApiEnv.first, KubernetesCheApiEnvVarProvider.API_ENDPOINT_URL_VARIABLE); assertEquals(cheApiEnv.second, CHE_SERVER_ENDPOINT); } } diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/UniqueNamesProvisionerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/UniqueNamesProvisionerTest.java new file mode 100644 index 00000000000..25dce69378d --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/UniqueNamesProvisionerTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; + +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_ORIGINAL_NAME_LABEL; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; + +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodBuilder; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.api.model.extensions.IngressBuilder; +import java.util.HashMap; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.mockito.Mock; +import org.mockito.testng.MockitoTestNGListener; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +/** + * Tests {@link UniqueNamesProvisioner}. + * + * @author Anton Korneta + */ +@Listeners(MockitoTestNGListener.class) +public class UniqueNamesProvisionerTest { + + private static final String WORKSPACE_ID = "workspace37"; + private static final String POD_NAME = "testPod"; + private static final String INGRESS_NAME = "testIngress"; + + @Mock private KubernetesEnvironment k8sEnv; + @Mock private RuntimeIdentity runtimeIdentity; + + private UniqueNamesProvisioner uniqueNamesProvisioner; + + @BeforeMethod + public void setup() { + uniqueNamesProvisioner = new UniqueNamesProvisioner<>(); + } + + @Test + public void provideUniquePodsNames() throws Exception { + when(runtimeIdentity.getWorkspaceId()).thenReturn(WORKSPACE_ID); + final HashMap pods = new HashMap<>(); + Pod pod = newPod(); + pods.put(POD_NAME, pod); + doReturn(pods).when(k8sEnv).getPods(); + + uniqueNamesProvisioner.provision(k8sEnv, runtimeIdentity); + + ObjectMeta podMetadata = pod.getMetadata(); + assertNotEquals(podMetadata.getName(), POD_NAME); + assertEquals(podMetadata.getLabels().get(CHE_ORIGINAL_NAME_LABEL), POD_NAME); + } + + @Test + public void provideUniqueIngressesNames() throws Exception { + final HashMap ingresses = new HashMap<>(); + Ingress ingress = newIngress(); + ingresses.put(POD_NAME, ingress); + doReturn(ingresses).when(k8sEnv).getIngresses(); + + uniqueNamesProvisioner.provision(k8sEnv, runtimeIdentity); + + final ObjectMeta ingressMetadata = ingress.getMetadata(); + assertNotEquals(ingressMetadata.getName(), INGRESS_NAME); + assertEquals(ingressMetadata.getLabels().get(CHE_ORIGINAL_NAME_LABEL), INGRESS_NAME); + } + + private static Pod newPod() { + return new PodBuilder() + .withMetadata(new ObjectMetaBuilder().withName(POD_NAME).build()) + .build(); + } + + private static Ingress newIngress() { + return new IngressBuilder() + .withMetadata(new ObjectMetaBuilder().withName(INGRESS_NAME).build()) + .build(); + } +} diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/env/LogsRootEnvVariableProviderTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/env/LogsRootEnvVariableProviderTest.java similarity index 87% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/env/LogsRootEnvVariableProviderTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/env/LogsRootEnvVariableProviderTest.java index 9df43013d40..c8d73729dab 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/env/LogsRootEnvVariableProviderTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/env/LogsRootEnvVariableProviderTest.java @@ -8,9 +8,9 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.env; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision.env; -import static org.eclipse.che.workspace.infrastructure.openshift.provision.env.LogsRootEnvVariableProvider.WORKSPACE_LOGS_ROOT_ENV_VAR; +import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.LogsRootEnvVariableProvider.WORKSPACE_LOGS_ROOT_ENV_VAR; import static org.testng.Assert.assertEquals; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/limits/ram/RamLimitProvisionerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/limits/ram/RamLimitProvisionerTest.java similarity index 88% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/limits/ram/RamLimitProvisionerTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/limits/ram/RamLimitProvisionerTest.java index 5278a6987fc..64d223ae328 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/limits/ram/RamLimitProvisionerTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/limits/ram/RamLimitProvisionerTest.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.limits.ram; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -27,7 +27,7 @@ import org.eclipse.che.api.core.model.workspace.config.MachineConfig; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; @@ -49,7 +49,7 @@ public class RamLimitProvisionerTest { public static final String MACHINE_NAME = POD_NAME + '/' + CONTAINER_NAME; public static final String RAM_LIMIT_ATTRIBUTE = "2147483648"; - @Mock private OpenShiftEnvironment osEnv; + @Mock private KubernetesEnvironment k8sEnv; @Mock private RuntimeIdentity identity; @Mock private Pod pod; @Mock private Container container; @@ -62,8 +62,8 @@ public class RamLimitProvisionerTest { @BeforeMethod public void setup() { ramLimitProvisioner = new RamLimitProvisioner(); - when(osEnv.getPods()).thenReturn(ImmutableMap.of(POD_NAME, pod)); - when(osEnv.getMachines()).thenReturn(ImmutableMap.of(MACHINE_NAME, internalMachineConfig)); + when(k8sEnv.getPods()).thenReturn(ImmutableMap.of(POD_NAME, pod)); + when(k8sEnv.getMachines()).thenReturn(ImmutableMap.of(MACHINE_NAME, internalMachineConfig)); when(internalMachineConfig.getAttributes()) .thenReturn(ImmutableMap.of(MachineConfig.MEMORY_LIMIT_ATTRIBUTE, RAM_LIMIT_ATTRIBUTE)); when(container.getName()).thenReturn(CONTAINER_NAME); @@ -77,7 +77,7 @@ public void setup() { @Test public void testProvisionRamLimitAttributeToContainer() throws Exception { - ramLimitProvisioner.provision(osEnv, identity); + ramLimitProvisioner.provision(k8sEnv, identity); verify(container).setResources(resourceCaptor.capture()); final ResourceRequirements captured = resourceCaptor.getValue(); @@ -92,7 +92,7 @@ public void testOverridesContainerRamLimitFromMachineAttribute() throws Exceptio when(containerResource.getLimits()).thenReturn(limits); when(container.getResources()).thenReturn(containerResource); - ramLimitProvisioner.provision(osEnv, identity); + ramLimitProvisioner.provision(k8sEnv, identity); verify(container).setResources(resourceCaptor.capture()); final ResourceRequirements captured = resourceCaptor.getValue(); diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/restartpolicy/RestartPolicyRewriterTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/restartpolicy/RestartPolicyRewriterTest.java similarity index 84% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/restartpolicy/RestartPolicyRewriterTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/restartpolicy/RestartPolicyRewriterTest.java index 47f6f56557c..77ca571aa7e 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/restartpolicy/RestartPolicyRewriterTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/restartpolicy/RestartPolicyRewriterTest.java @@ -8,11 +8,11 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.restartpolicy; +package org.eclipse.che.workspace.infrastructure.kubernetes.provision.restartpolicy; import static java.lang.String.format; import static java.util.Collections.singletonMap; -import static org.eclipse.che.workspace.infrastructure.openshift.provision.restartpolicy.RestartPolicyRewriter.DEFAULT_RESTART_POLICY; +import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.restartpolicy.RestartPolicyRewriter.DEFAULT_RESTART_POLICY; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -31,7 +31,7 @@ import org.eclipse.che.api.core.model.workspace.Warning; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.model.impl.WarningImpl; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.InjectMocks; @@ -47,7 +47,7 @@ public class RestartPolicyRewriterTest { private static final String TEST_POD_NAME = "app"; private static final String ALWAYS_RESTART_POLICY = "Always"; - @Mock private OpenShiftEnvironment osEnv; + @Mock private KubernetesEnvironment k8sEnv; @Mock private RuntimeIdentity runtimeIdentity; @InjectMocks private RestartPolicyRewriter restartPolicyRewriter; @@ -55,13 +55,13 @@ public class RestartPolicyRewriterTest { @Test public void rewritesRestartPolicyWhenItsDifferentWithDefaultOne() throws Exception { - when(osEnv.getPods()) + when(k8sEnv.getPods()) .thenReturn(singletonMap(TEST_POD_NAME, newPod(TEST_POD_NAME, ALWAYS_RESTART_POLICY))); - restartPolicyRewriter.provision(osEnv, runtimeIdentity); + restartPolicyRewriter.provision(k8sEnv, runtimeIdentity); assertEquals( - osEnv.getPods().get(TEST_POD_NAME).getSpec().getRestartPolicy(), DEFAULT_RESTART_POLICY); + k8sEnv.getPods().get(TEST_POD_NAME).getSpec().getRestartPolicy(), DEFAULT_RESTART_POLICY); verifyWarnings( new WarningImpl( 101, @@ -92,7 +92,7 @@ private void verifyWarnings(Warning... expectedWarnings) { } private List captureWarnings() { - verify(osEnv, atLeastOnce()).addWarning(warningCaptor.capture()); + verify(k8sEnv, atLeastOnce()).addWarning(warningCaptor.capture()); return warningCaptor.getAllValues(); } } diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java new file mode 100644 index 00000000000..fdd847f773e --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.server; + +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerExposer.SERVER_PREFIX; +import static org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerExposer.SERVER_UNIQUE_PART_SIZE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import com.google.common.collect.ImmutableMap; +import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.ContainerBuilder; +import io.fabric8.kubernetes.api.model.ContainerPortBuilder; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodBuilder; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServicePort; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.api.model.extensions.IngressBackend; +import io.fabric8.kubernetes.api.model.extensions.IngressRule; +import java.util.ArrayList; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.regex.Pattern; +import org.eclipse.che.api.core.model.workspace.config.ServerConfig; +import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test for {@link KubernetesServerExposer}. + * + * @author Sergii Leshchenko + */ +public class KubernetesServerExposerTest { + + private static final Map ATTRIBUTES_MAP = singletonMap("key", "value"); + private static final Map INTERNAL_SERVER_ATTRIBUTE_MAP = + singletonMap(ServerConfig.INTERNAL_SERVER_ATTRIBUTE, Boolean.TRUE.toString()); + + private static final Pattern SERVER_PREFIX_REGEX = + Pattern.compile('^' + SERVER_PREFIX + "[A-z0-9]{" + SERVER_UNIQUE_PART_SIZE + "}-pod-main$"); + public static final String MACHINE_NAME = "pod/main"; + + private KubernetesServerExposer serverExposer; + private KubernetesEnvironment kubernetesEnvironment; + private Container container; + + @BeforeMethod + public void setUp() throws Exception { + container = new ContainerBuilder().withName("main").build(); + Pod pod = + new PodBuilder() + .withNewMetadata() + .withName("pod") + .endMetadata() + .withNewSpec() + .withContainers(container) + .endSpec() + .build(); + + kubernetesEnvironment = + KubernetesEnvironment.builder().setPods(ImmutableMap.of("pod", pod)).build(); + this.serverExposer = + new KubernetesServerExposer<>(MACHINE_NAME, pod, container, kubernetesEnvironment); + } + + @Test + public void shouldExposeContainerPortAndCreateServiceAndIngressForServer() { + // given + ServerConfigImpl httpServerConfig = + new ServerConfigImpl("8080/tcp", "http", "/api", ATTRIBUTES_MAP); + Map serversToExpose = + ImmutableMap.of("http-server", httpServerConfig); + + // when + serverExposer.expose(serversToExpose); + + // then + assertThatExternalServerIsExposed( + MACHINE_NAME, + "http-server", + "tcp", + 8080, + new ServerConfigImpl(httpServerConfig).withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void + shouldExposeContainerPortAndCreateServiceAndIngressForServerWhenTwoServersHasTheSamePort() { + // given + ServerConfigImpl httpServerConfig = + new ServerConfigImpl("8080/tcp", "http", "/api", ATTRIBUTES_MAP); + ServerConfigImpl wsServerConfig = + new ServerConfigImpl("8080/tcp", "ws", "/connect", ATTRIBUTES_MAP); + Map serversToExpose = + ImmutableMap.of( + "http-server", httpServerConfig, + "ws-server", wsServerConfig); + + // when + serverExposer.expose(serversToExpose); + + // then + assertEquals(kubernetesEnvironment.getServices().size(), 1); + assertEquals(kubernetesEnvironment.getIngresses().size(), 1); + assertThatExternalServerIsExposed( + MACHINE_NAME, + "http-server", + "tcp", + 8080, + new ServerConfigImpl(httpServerConfig).withAttributes(ATTRIBUTES_MAP)); + assertThatExternalServerIsExposed( + MACHINE_NAME, + "ws-server", + "tcp", + 8080, + new ServerConfigImpl(wsServerConfig).withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void + shouldExposeContainerPortsAndCreateServiceAndIngressesForServerWhenTwoServersHasDifferentPorts() { + // given + ServerConfigImpl httpServerConfig = + new ServerConfigImpl("8080/tcp", "http", "/api", ATTRIBUTES_MAP); + ServerConfigImpl wsServerConfig = + new ServerConfigImpl("8081/tcp", "ws", "/connect", ATTRIBUTES_MAP); + Map serversToExpose = + ImmutableMap.of( + "http-server", httpServerConfig, + "ws-server", wsServerConfig); + + // when + serverExposer.expose(serversToExpose); + + // then + assertEquals(kubernetesEnvironment.getServices().size(), 1); + assertEquals(kubernetesEnvironment.getIngresses().size(), 2); + assertThatExternalServerIsExposed( + MACHINE_NAME, + "http-server", + "tcp", + 8080, + new ServerConfigImpl(httpServerConfig).withAttributes(ATTRIBUTES_MAP)); + assertThatExternalServerIsExposed( + MACHINE_NAME, + "ws-server", + "tcp", + 8081, + new ServerConfigImpl(wsServerConfig).withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void + shouldExposeTcpContainerPortsAndCreateServiceAndIngressForServerWhenProtocolIsMissedInPort() { + // given + ServerConfigImpl httpServerConfig = + new ServerConfigImpl("8080", "http", "/api", ATTRIBUTES_MAP); + Map serversToExpose = + ImmutableMap.of("http-server", httpServerConfig); + + // when + serverExposer.expose(serversToExpose); + + // then + assertEquals(kubernetesEnvironment.getServices().size(), 1); + assertEquals(kubernetesEnvironment.getIngresses().size(), 1); + assertThatExternalServerIsExposed( + MACHINE_NAME, + "http-server", + "TCP", + 8080, + new ServerConfigImpl(httpServerConfig).withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void shouldNotAddAdditionalContainerPortWhenItIsAlreadyExposed() { + // given + ServerConfigImpl httpServerConfig = + new ServerConfigImpl("8080/tcp", "http", "/api", ATTRIBUTES_MAP); + Map serversToExpose = + ImmutableMap.of("http-server", httpServerConfig); + container.setPorts( + singletonList( + new ContainerPortBuilder() + .withName("port-8080") + .withContainerPort(8080) + .withProtocol("TCP") + .build())); + + // when + serverExposer.expose(serversToExpose); + + // then + assertThatExternalServerIsExposed( + MACHINE_NAME, + "http-server", + "tcp", + 8080, + new ServerConfigImpl(httpServerConfig).withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void shouldAddAdditionalContainerPortWhenThereIsTheSameButWithDifferentProtocol() { + // given + ServerConfigImpl udpServerConfig = + new ServerConfigImpl("8080/udp", "udp", "/api", ATTRIBUTES_MAP); + Map serversToExpose = ImmutableMap.of("server", udpServerConfig); + container.setPorts( + new ArrayList<>( + singletonList( + new ContainerPortBuilder() + .withName("port-8080") + .withContainerPort(8080) + .withProtocol("TCP") + .build()))); + + // when + serverExposer.expose(serversToExpose); + + // then + assertEquals(container.getPorts().size(), 2); + assertEquals(container.getPorts().get(1).getContainerPort(), new Integer(8080)); + assertEquals(container.getPorts().get(1).getProtocol(), "UDP"); + assertThatExternalServerIsExposed( + MACHINE_NAME, + "server", + "udp", + 8080, + new ServerConfigImpl(udpServerConfig).withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void shouldExposeContainerPortAndCreateServiceForInternalServer() throws Exception { + // given + ServerConfigImpl httpServerConfig = + new ServerConfigImpl("8080/tcp", "http", "/api", INTERNAL_SERVER_ATTRIBUTE_MAP); + Map serversToExpose = + ImmutableMap.of("http-server", httpServerConfig); + + // when + serverExposer.expose(serversToExpose); + + // then + assertThatInternalServerIsExposed( + MACHINE_NAME, + "http-server", + "tcp", + 8080, + new ServerConfigImpl(httpServerConfig).withAttributes(INTERNAL_SERVER_ATTRIBUTE_MAP)); + } + + @Test + public void shouldExposeInternalAndExternalServers() throws Exception { + // given + ServerConfigImpl internalServerConfig = + new ServerConfigImpl("8080/tcp", "http", "/api", INTERNAL_SERVER_ATTRIBUTE_MAP); + ServerConfigImpl externalServerConfig = + new ServerConfigImpl("9090/tcp", "http", "/api", ATTRIBUTES_MAP); + Map serversToExpose = + ImmutableMap.of("int-server", internalServerConfig, "ext-server", externalServerConfig); + + // when + serverExposer.expose(serversToExpose); + + // then + assertThatInternalServerIsExposed( + MACHINE_NAME, + "int-server", + "tcp", + 8080, + new ServerConfigImpl(internalServerConfig).withAttributes(INTERNAL_SERVER_ATTRIBUTE_MAP)); + assertThatExternalServerIsExposed( + MACHINE_NAME, + "ext-server", + "tcp", + 9090, + new ServerConfigImpl(externalServerConfig).withAttributes(ATTRIBUTES_MAP)); + } + + private void assertThatExternalServerIsExposed( + String machineName, + String serverNameRegex, + String portProtocol, + Integer port, + ServerConfigImpl expected) { + // then + assertTrue( + container + .getPorts() + .stream() + .anyMatch( + p -> + p.getContainerPort().equals(port) + && p.getProtocol().equals(portProtocol.toUpperCase()))); + // ensure that service is created + + Service service = null; + for (Entry entry : kubernetesEnvironment.getServices().entrySet()) { + if (SERVER_PREFIX_REGEX.matcher(entry.getKey()).matches()) { + service = entry.getValue(); + break; + } + } + assertNotNull(service); + + // ensure that required service port is exposed + Optional servicePortOpt = + service + .getSpec() + .getPorts() + .stream() + .filter(p -> p.getTargetPort().getIntVal().equals(port)) + .findAny(); + assertTrue(servicePortOpt.isPresent()); + ServicePort servicePort = servicePortOpt.get(); + assertEquals(servicePort.getTargetPort().getIntVal(), port); + assertEquals(servicePort.getPort(), port); + assertEquals(servicePort.getName(), SERVER_PREFIX + "-" + port); + + Annotations.Deserializer serviceAnnotations = + Annotations.newDeserializer(service.getMetadata().getAnnotations()); + assertEquals(serviceAnnotations.machineName(), machineName); + + // ensure that required ingress is created + Ingress ingress = + kubernetesEnvironment + .getIngresses() + .get(service.getMetadata().getName() + "-server-" + port); + IngressRule ingressRule = ingress.getSpec().getRules().get(0); + IngressBackend backend = ingressRule.getHttp().getPaths().get(0).getBackend(); + assertEquals(backend.getServiceName(), service.getMetadata().getName()); + assertEquals(backend.getServicePort().getStrVal(), servicePort.getName()); + + Annotations.Deserializer ingressAnnotations = + Annotations.newDeserializer(ingress.getMetadata().getAnnotations()); + Map servers = ingressAnnotations.servers(); + ServerConfig serverConfig = servers.get(serverNameRegex); + assertEquals(serverConfig, expected); + + assertEquals(ingressAnnotations.machineName(), machineName); + } + + private void assertThatInternalServerIsExposed( + String machineName, + String serverNameRegex, + String portProtocol, + Integer port, + ServerConfigImpl expected) { + // then + assertTrue( + container + .getPorts() + .stream() + .anyMatch( + p -> + p.getContainerPort().equals(port) + && p.getProtocol().equals(portProtocol.toUpperCase()))); + // ensure that service is created + + Service service = null; + for (Entry entry : kubernetesEnvironment.getServices().entrySet()) { + if (SERVER_PREFIX_REGEX.matcher(entry.getKey()).matches()) { + service = entry.getValue(); + break; + } + } + assertNotNull(service); + + // ensure that required service port is exposed + Optional servicePortOpt = + service + .getSpec() + .getPorts() + .stream() + .filter(p -> p.getTargetPort().getIntVal().equals(port)) + .findAny(); + assertTrue(servicePortOpt.isPresent()); + ServicePort servicePort = servicePortOpt.get(); + assertEquals(servicePort.getTargetPort().getIntVal(), port); + assertEquals(servicePort.getPort(), port); + assertEquals(servicePort.getName(), SERVER_PREFIX + "-" + port); + + Annotations.Deserializer serviceAnnotations = + Annotations.newDeserializer(service.getMetadata().getAnnotations()); + assertEquals(serviceAnnotations.machineName(), machineName); + + Map servers = serviceAnnotations.servers(); + ServerConfig serverConfig = servers.get(serverNameRegex); + assertEquals(serverConfig, expected); + } +} diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerResolverTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerResolverTest.java new file mode 100644 index 00000000000..106e97c5158 --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerResolverTest.java @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.server; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServiceBuilder; +import io.fabric8.kubernetes.api.model.ServicePortBuilder; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressPath; +import io.fabric8.kubernetes.api.model.extensions.HTTPIngressRuleValue; +import io.fabric8.kubernetes.api.model.extensions.Ingress; +import io.fabric8.kubernetes.api.model.extensions.IngressBackend; +import io.fabric8.kubernetes.api.model.extensions.IngressBuilder; +import io.fabric8.kubernetes.api.model.extensions.IngressRule; +import java.util.Map; +import org.eclipse.che.api.core.model.workspace.config.ServerConfig; +import org.eclipse.che.api.core.model.workspace.runtime.ServerStatus; +import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; +import org.eclipse.che.api.workspace.server.model.impl.ServerImpl; +import org.eclipse.che.commons.lang.Pair; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations.Serializer; +import org.testng.annotations.Test; + +/** + * Test for {@link KubernetesServerResolver}. + * + * @author Sergii Leshchenko + */ +public class KubernetesServerResolverTest { + + private static final Map ATTRIBUTES_MAP = singletonMap("key", "value"); + private static final int CONTAINER_PORT = 3054; + private static final String INGRESS_IP = "127.0.0.1"; + private static final String INGRESS_RULE_PATH_PREFIX = "/server-8080"; + private static final String INGRESS_PATH_PREFIX = "server-8080"; + + @Test + public void + testResolvingServersWhenThereIsNoTheCorrespondingServiceAndingressForTheSpecifiedMachine() { + // given + Service nonMatchedByPodService = + createService("nonMatched", "foreignMachine", CONTAINER_PORT, null); + Ingress ingress = + createIngress( + "nonMatched", + "foreignMachine", + Pair.of("http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); + + KubernetesServerResolver serverResolver = + new KubernetesServerResolver(singletonList(nonMatchedByPodService), singletonList(ingress)); + + // when + Map resolved = serverResolver.resolve("machine"); + + // then + assertTrue(resolved.isEmpty()); + } + + @Test + public void testResolvingServersWhenThereIsMatchedingressForTheSpecifiedMachine() { + Ingress ingress = + createIngress( + "matched", + "machine", + Pair.of("http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); + + KubernetesServerResolver serverResolver = + new KubernetesServerResolver(emptyList(), singletonList(ingress)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://" + INGRESS_IP + INGRESS_RULE_PATH_PREFIX + "/api") + .withStatus(ServerStatus.UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingServersWhenThereIsMatchedIngressForMachineAndServerPathIsNull() { + Ingress ingress = + createIngress( + "matched", + "machine", + Pair.of("http-server", new ServerConfigImpl("3054", "http", null, ATTRIBUTES_MAP))); + + KubernetesServerResolver serverResolver = + new KubernetesServerResolver(emptyList(), singletonList(ingress)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://" + INGRESS_IP + INGRESS_RULE_PATH_PREFIX) + .withStatus(ServerStatus.UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingServersWhenThereIsMatchedIngressForMachineAndServerPathIsEmpty() { + Ingress ingress = + createIngress( + "matched", + "machine", + Pair.of("http-server", new ServerConfigImpl("3054", "http", "", ATTRIBUTES_MAP))); + + KubernetesServerResolver serverResolver = + new KubernetesServerResolver(emptyList(), singletonList(ingress)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://" + INGRESS_IP + INGRESS_RULE_PATH_PREFIX) + .withStatus(ServerStatus.UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingServersWhenThereIsMatchedingressForMachineAndServerPathIsRelative() { + Ingress ingress = + createIngress( + "matched", + "machine", + Pair.of("http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); + + KubernetesServerResolver serverResolver = + new KubernetesServerResolver(emptyList(), singletonList(ingress)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://" + INGRESS_IP + INGRESS_RULE_PATH_PREFIX + "/api") + .withStatus(ServerStatus.UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingInternalServers() { + Service service = + createService( + "service11", + "machine", + CONTAINER_PORT, + singletonMap( + "http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); + + KubernetesServerResolver serverResolver = + new KubernetesServerResolver(singletonList(service), emptyList()); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://service11:3054/api") + .withStatus(ServerStatus.UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingInternalServersWithPortWithTransportProtocol() { + Service service = + createService( + "service11", + "machine", + CONTAINER_PORT, + singletonMap( + "http-server", new ServerConfigImpl("3054/udp", "xxx", "api", ATTRIBUTES_MAP))); + + KubernetesServerResolver serverResolver = + new KubernetesServerResolver(singletonList(service), emptyList()); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("xxx://service11:3054/api") + .withStatus(ServerStatus.UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + private Service createService( + String name, String machineName, Integer port, Map servers) { + Serializer serializer = Annotations.newSerializer(); + serializer.machineName(machineName); + if (servers != null) { + serializer.servers(servers); + } + + return new ServiceBuilder() + .withNewMetadata() + .withName(name) + .withAnnotations(serializer.annotations()) + .endMetadata() + .withNewSpec() + .withPorts( + new ServicePortBuilder() + .withPort(port) + .withNewTargetPort() + .withIntVal(port) + .endTargetPort() + .build()) + .endSpec() + .build(); + } + + private Ingress createIngress( + String name, String machineName, Pair server) { + Serializer serializer = Annotations.newSerializer(); + serializer.machineName(machineName); + serializer.server(server.first, server.second); + + return new IngressBuilder() + .withNewMetadata() + .withName(name) + .withAnnotations(serializer.annotations()) + .endMetadata() + .withNewSpec() + .withRules( + new IngressRule( + null, + new HTTPIngressRuleValue( + singletonList( + new HTTPIngressPath( + new IngressBackend(name, new IntOrString("8080")), + INGRESS_PATH_PREFIX))))) + .endSpec() + .withNewStatus() + .withNewLoadBalancer() + .addNewIngress() + .withIp("127.0.0.1") + .endIngress() + .endLoadBalancer() + .endStatus() + .build(); + } +} diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/util/ContainersTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/ContainersTest.java similarity index 98% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/util/ContainersTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/ContainersTest.java index 6192a2fc38c..1ee9b608408 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/util/ContainersTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/ContainersTest.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.util; +package org.eclipse.che.workspace.infrastructure.kubernetes.util; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/util/KubernetesSizeTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSizeTest.java similarity index 94% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/util/KubernetesSizeTest.java rename to infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSizeTest.java index edf48817fc0..eb57a864599 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/util/KubernetesSizeTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSizeTest.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.util; +package org.eclipse.che.workspace.infrastructure.kubernetes.util; import static org.testng.Assert.assertEquals; diff --git a/infrastructures/kubernetes/src/test/resources/logback-test.xml b/infrastructures/kubernetes/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..8d0bd11d3eb --- /dev/null +++ b/infrastructures/kubernetes/src/test/resources/logback-test.xml @@ -0,0 +1,35 @@ + + + + + + + %-41(%date[%.15thread]) %-45([%-5level] [%.30logger{30} %L]) - %msg%n + + + + + target/log/log.log + + %-41(%date[%.15thread]) %-45([%-5level] [%.30logger{30} %L]) - %msg%n + + + + + + + + + + diff --git a/infrastructures/openshift/pom.xml b/infrastructures/openshift/pom.xml index 16f358462d5..7adc5d4e701 100644 --- a/infrastructures/openshift/pom.xml +++ b/infrastructures/openshift/pom.xml @@ -22,10 +22,6 @@ infrastructure-openshift Infrastructure :: OpenShift - - com.google.code.gson - gson - com.google.guava guava @@ -42,10 +38,6 @@ com.google.inject.extensions guice-multibindings - - com.squareup.okhttp3 - okhttp - io.fabric8 kubernetes-client @@ -78,10 +70,6 @@ org.eclipse.che.core che-core-api-installer - - org.eclipse.che.core - che-core-api-installer-shared - org.eclipse.che.core che-core-api-model @@ -99,8 +87,8 @@ che-core-commons-annotations - org.eclipse.che.core - che-core-commons-lang + org.eclipse.che.infrastructure + infrastructure-kubernetes org.eclipse.che.infrastructure.docker @@ -110,11 +98,6 @@ org.slf4j slf4j-api - - javax.ws.rs - javax.ws.rs-api - provided - ch.qos.logback logback-classic diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftClientFactory.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftClientFactory.java index 2595bfb3d5b..2f5bddf1055 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftClientFactory.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftClientFactory.java @@ -22,6 +22,7 @@ import javax.inject.Singleton; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.commons.annotation.Nullable; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +31,7 @@ * @author Anton Korneta */ @Singleton -public class OpenShiftClientFactory { +public class OpenShiftClientFactory extends KubernetesClientFactory { private static final Logger LOG = LoggerFactory.getLogger(OpenShiftClientFactory.class); @@ -38,11 +39,12 @@ public class OpenShiftClientFactory { @Inject public OpenShiftClientFactory( - @Nullable @Named("che.infra.openshift.master_url") String masterUrl, - @Nullable @Named("che.infra.openshift.username") String username, - @Nullable @Named("che.infra.openshift.password") String password, - @Nullable @Named("che.infra.openshift.oauth_token") String oauthToken, - @Nullable @Named("che.infra.openshift.trust_certs") Boolean doTrustCerts) { + @Nullable @Named("che.infra.kubernetes.master_url") String masterUrl, + @Nullable @Named("che.infra.kubernetes.username") String username, + @Nullable @Named("che.infra.kubernetes.password") String password, + @Nullable @Named("che.infra.kubernetes.oauth_token") String oauthToken, + @Nullable @Named("che.infra.kubernetes.trust_certs") Boolean doTrustCerts) { + super(masterUrl, username, password, oauthToken, doTrustCerts); OpenShiftConfigBuilder configBuilder = new OpenShiftConfigBuilder(); if (!isNullOrEmpty(masterUrl)) { configBuilder.withMasterUrl(masterUrl); diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java index 8117e628da0..1f503442a83 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java @@ -15,16 +15,17 @@ import javax.inject.Singleton; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.InstallerServersPortProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.RamLimitProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.restartpolicy.RestartPolicyRewriter; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.project.pvc.WorkspaceVolumesStrategy; -import org.eclipse.che.workspace.infrastructure.openshift.provision.InstallerServersPortProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.LogsVolumeMachineProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.UniqueNamesProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.env.EnvVarsConverter; -import org.eclipse.che.workspace.infrastructure.openshift.provision.limits.ram.RamLimitProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.restartpolicy.RestartPolicyRewriter; -import org.eclipse.che.workspace.infrastructure.openshift.provision.route.TlsRouteProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.server.ServersConverter; +import org.eclipse.che.workspace.infrastructure.openshift.provision.OpenShiftServersConverter; +import org.eclipse.che.workspace.infrastructure.openshift.provision.OpenShiftUniqueNamesProvisioner; +import org.eclipse.che.workspace.infrastructure.openshift.provision.RouteTlsProvisioner; /** * Applies the set of configurations to the OpenShift environment and environment configuration with @@ -38,9 +39,9 @@ public class OpenShiftEnvironmentProvisioner { private final boolean pvcEnabled; private final WorkspaceVolumesStrategy volumesStrategy; - private final UniqueNamesProvisioner uniqueNamesProvisioner; - private final TlsRouteProvisioner tlsRouteProvisioner; - private final ServersConverter serversConverter; + private final UniqueNamesProvisioner uniqueNamesProvisioner; + private final RouteTlsProvisioner routeTlsProvisioner; + private final OpenShiftServersConverter openShiftServersConverter; private final EnvVarsConverter envVarsConverter; private final RestartPolicyRewriter restartPolicyRewriter; private final RamLimitProvisioner ramLimitProvisioner; @@ -49,10 +50,10 @@ public class OpenShiftEnvironmentProvisioner { @Inject public OpenShiftEnvironmentProvisioner( - @Named("che.infra.openshift.pvc.enabled") boolean pvcEnabled, - UniqueNamesProvisioner uniqueNamesProvisioner, - TlsRouteProvisioner tlsRouteProvisioner, - ServersConverter serversConverter, + @Named("che.infra.kubernetes.pvc.enabled") boolean pvcEnabled, + OpenShiftUniqueNamesProvisioner uniqueNamesProvisioner, + RouteTlsProvisioner routeTlsProvisioner, + OpenShiftServersConverter openShiftServersConverter, EnvVarsConverter envVarsConverter, RestartPolicyRewriter restartPolicyRewriter, WorkspaceVolumesStrategy volumesStrategy, @@ -62,8 +63,8 @@ public OpenShiftEnvironmentProvisioner( this.pvcEnabled = pvcEnabled; this.volumesStrategy = volumesStrategy; this.uniqueNamesProvisioner = uniqueNamesProvisioner; - this.tlsRouteProvisioner = tlsRouteProvisioner; - this.serversConverter = serversConverter; + this.routeTlsProvisioner = routeTlsProvisioner; + this.openShiftServersConverter = openShiftServersConverter; this.envVarsConverter = envVarsConverter; this.restartPolicyRewriter = restartPolicyRewriter; this.ramLimitProvisioner = ramLimitProvisioner; @@ -80,7 +81,7 @@ public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) } // 2 stage - converting Che model env to OpenShift env - serversConverter.provision(osEnv, identity); + openShiftServersConverter.provision(osEnv, identity); envVarsConverter.provision(osEnv, identity); if (pvcEnabled) { volumesStrategy.provision(osEnv, identity); @@ -89,7 +90,7 @@ public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) // 3 stage - add OpenShift env items restartPolicyRewriter.provision(osEnv, identity); uniqueNamesProvisioner.provision(osEnv, identity); - tlsRouteProvisioner.provision(osEnv, identity); + routeTlsProvisioner.provision(osEnv, identity); ramLimitProvisioner.provision(osEnv, identity); } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfraModule.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfraModule.java index 7fba217434e..f23bb0d20a6 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfraModule.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfraModule.java @@ -10,8 +10,8 @@ */ package org.eclipse.che.workspace.infrastructure.openshift; -import static org.eclipse.che.workspace.infrastructure.openshift.project.pvc.CommonPVCStrategy.COMMON_STRATEGY; -import static org.eclipse.che.workspace.infrastructure.openshift.project.pvc.UniqueWorkspacePVCStrategy.UNIQUE_STRATEGY; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy.COMMON_STRATEGY; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy.UNIQUE_STRATEGY; import com.google.inject.AbstractModule; import com.google.inject.assistedinject.FactoryModuleBuilder; @@ -23,17 +23,19 @@ import org.eclipse.che.api.workspace.server.spi.provision.env.EnvVarProvider; import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironment; import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironmentFactory; -import org.eclipse.che.workspace.infrastructure.openshift.bootstrapper.OpenShiftBootstrapperFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspacePVCCleaner; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumeStrategyProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiEnvVarProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.LogsRootEnvVariableProvider; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironmentFactory; +import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProjectFactory; import org.eclipse.che.workspace.infrastructure.openshift.project.RemoveProjectOnWorkspaceRemove; -import org.eclipse.che.workspace.infrastructure.openshift.project.pvc.CommonPVCStrategy; -import org.eclipse.che.workspace.infrastructure.openshift.project.pvc.UniqueWorkspacePVCStrategy; -import org.eclipse.che.workspace.infrastructure.openshift.project.pvc.WorkspacePVCCleaner; -import org.eclipse.che.workspace.infrastructure.openshift.project.pvc.WorkspaceVolumeStrategyProvider; -import org.eclipse.che.workspace.infrastructure.openshift.project.pvc.WorkspaceVolumesStrategy; -import org.eclipse.che.workspace.infrastructure.openshift.provision.OpenShiftCheApiEnvVarProvider; -import org.eclipse.che.workspace.infrastructure.openshift.provision.env.LogsRootEnvVariableProvider; /** @author Sergii Leshchenko */ public class OpenShiftInfraModule extends AbstractModule { @@ -47,13 +49,16 @@ protected void configure() { bind(RuntimeInfrastructure.class).to(OpenShiftInfrastructure.class); + bind(KubernetesNamespaceFactory.class).to(OpenShiftProjectFactory.class); + install(new FactoryModuleBuilder().build(OpenShiftRuntimeContextFactory.class)); install(new FactoryModuleBuilder().build(OpenShiftRuntimeFactory.class)); - install(new FactoryModuleBuilder().build(OpenShiftBootstrapperFactory.class)); + + install(new FactoryModuleBuilder().build(KubernetesBootstrapperFactory.class)); bind(WorkspacePVCCleaner.class).asEagerSingleton(); bind(RemoveProjectOnWorkspaceRemove.class).asEagerSingleton(); - bind(CheApiEnvVarProvider.class).to(OpenShiftCheApiEnvVarProvider.class); + bind(CheApiEnvVarProvider.class).to(KubernetesCheApiEnvVarProvider.class); MapBinder volumesStrategies = MapBinder.newMapBinder(binder(), String.class, WorkspaceVolumesStrategy.class); diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfrastructure.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfrastructure.java index 4a9380f57fb..88f34001a9b 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfrastructure.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfrastructure.java @@ -25,8 +25,9 @@ import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironment; import org.eclipse.che.api.workspace.server.spi.provision.InternalEnvironmentProvisioner; import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.convert.DockerImageEnvironmentConverter; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.environment.convert.DockerImageEnvironmentConverter; /** @author Sergii Leshchenko */ @Singleton @@ -47,7 +48,8 @@ public OpenShiftInfrastructure( DockerImageEnvironmentConverter dockerImageEnvConverter) { super( NAME, - ImmutableSet.of(OpenShiftEnvironment.TYPE, DockerImageEnvironment.TYPE), + ImmutableSet.of( + OpenShiftEnvironment.TYPE, KubernetesEnvironment.TYPE, DockerImageEnvironment.TYPE), eventService, internalEnvProvisioners); this.runtimeContextFactory = runtimeContextFactory; @@ -71,9 +73,17 @@ private OpenShiftEnvironment asOpenShiftEnv(InternalEnvironment source) if (source instanceof OpenShiftEnvironment) { return (OpenShiftEnvironment) source; } + + if (source instanceof KubernetesEnvironment) { + return new OpenShiftEnvironment((KubernetesEnvironment) source); + } + if (source instanceof DockerImageEnvironment) { - return dockerImageEnvConverter.convert((DockerImageEnvironment) source); + KubernetesEnvironment k8sEnv = + dockerImageEnvConverter.convert((DockerImageEnvironment) source); + return new OpenShiftEnvironment(k8sEnv); } + throw new InternalInfrastructureException( format( "Environment type '%s' is not supported. Supported environment types: %s", diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java index 9a4e22366f1..fd4dc815be3 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java @@ -10,87 +10,41 @@ */ package org.eclipse.che.workspace.infrastructure.openshift; -import static java.lang.String.format; -import static java.util.Collections.emptyMap; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import com.google.inject.assistedinject.Assisted; -import io.fabric8.kubernetes.api.model.Container; -import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; -import io.fabric8.kubernetes.api.model.extensions.Ingress; -import io.fabric8.kubernetes.client.Watcher.Action; +import io.fabric8.openshift.api.model.Route; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Consumer; import javax.inject.Inject; import javax.inject.Named; import org.eclipse.che.api.core.model.workspace.Warning; -import org.eclipse.che.api.core.model.workspace.runtime.Machine; -import org.eclipse.che.api.core.model.workspace.runtime.MachineStatus; -import org.eclipse.che.api.core.model.workspace.runtime.Server; -import org.eclipse.che.api.core.model.workspace.runtime.ServerStatus; import org.eclipse.che.api.core.notification.EventService; -import org.eclipse.che.api.workspace.server.DtoConverter; import org.eclipse.che.api.workspace.server.URLRewriter.NoOpURLRewriter; -import org.eclipse.che.api.workspace.server.hc.ServersChecker; import org.eclipse.che.api.workspace.server.hc.ServersCheckerFactory; -import org.eclipse.che.api.workspace.server.hc.probe.ProbeResult; -import org.eclipse.che.api.workspace.server.hc.probe.ProbeResult.ProbeStatus; import org.eclipse.che.api.workspace.server.hc.probe.ProbeScheduler; import org.eclipse.che.api.workspace.server.hc.probe.WorkspaceProbesFactory; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; -import org.eclipse.che.api.workspace.server.spi.InternalRuntime; -import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.api.workspace.shared.dto.event.MachineLogEvent; -import org.eclipse.che.api.workspace.shared.dto.event.MachineStatusEvent; -import org.eclipse.che.api.workspace.shared.dto.event.RuntimeStatusEvent; -import org.eclipse.che.api.workspace.shared.dto.event.ServerStatusEvent; -import org.eclipse.che.dto.server.DtoFactory; -import org.eclipse.che.workspace.infrastructure.openshift.bootstrapper.OpenShiftBootstrapperFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInternalRuntime; +import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; -import org.eclipse.che.workspace.infrastructure.openshift.project.event.ContainerEvent; -import org.eclipse.che.workspace.infrastructure.openshift.project.event.ContainerEventHandler; -import org.eclipse.che.workspace.infrastructure.openshift.project.event.PodActionHandler; -import org.eclipse.che.workspace.infrastructure.openshift.project.pvc.WorkspaceVolumesStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.eclipse.che.workspace.infrastructure.openshift.server.OpenShiftServerResolver; /** * @author Sergii Leshchenko * @author Anton Korneta */ -public class OpenShiftInternalRuntime extends InternalRuntime { - - private static final Logger LOG = LoggerFactory.getLogger(OpenShiftInternalRuntime.class); +public class OpenShiftInternalRuntime extends KubernetesInternalRuntime { - private static final String RUNTIME_STOPPED_STATE = "STOPPED"; - private static final String RUNTIME_RUNNING_STATE = "RUNNING"; - private static final String POD_FAILED_STATUS = "Failed"; - - private final EventService eventService; - private final ServersCheckerFactory serverCheckerFactory; - private final OpenShiftBootstrapperFactory bootstrapperFactory; - private final Map machines; - private final int machineStartTimeoutMin; - private final ProbeScheduler probeScheduler; - private final WorkspaceProbesFactory probesFactory; private final OpenShiftProject project; - private final WorkspaceVolumesStrategy volumesStrategy; @Inject public OpenShiftInternalRuntime( - @Named("che.infra.openshift.machine_start_timeout_min") int machineStartTimeoutMin, + @Named("che.infra.kubernetes.machine_start_timeout_min") int machineStartTimeoutMin, NoOpURLRewriter urlRewriter, EventService eventService, - OpenShiftBootstrapperFactory bootstrapperFactory, + KubernetesBootstrapperFactory bootstrapperFactory, ServersCheckerFactory serverCheckerFactory, WorkspaceVolumesStrategy volumesStrategy, ProbeScheduler probeScheduler, @@ -98,328 +52,37 @@ public OpenShiftInternalRuntime( @Assisted OpenShiftRuntimeContext context, @Assisted OpenShiftProject project, @Assisted List warnings) { - super(context, urlRewriter, warnings, false); - this.eventService = eventService; - this.bootstrapperFactory = bootstrapperFactory; - this.serverCheckerFactory = serverCheckerFactory; - this.volumesStrategy = volumesStrategy; - this.machineStartTimeoutMin = machineStartTimeoutMin; - this.probeScheduler = probeScheduler; - this.probesFactory = probesFactory; + super( + machineStartTimeoutMin, + urlRewriter, + eventService, + bootstrapperFactory, + serverCheckerFactory, + volumesStrategy, + probeScheduler, + probesFactory, + context, + project, + warnings); this.project = project; - this.machines = new ConcurrentHashMap<>(); - } - - @Override - protected void internalStart(Map startOptions) throws InfrastructureException { - OpenShiftRuntimeContext context = getContext(); - String workspaceId = context.getIdentity().getWorkspaceId(); - try { - final OpenShiftEnvironment osEnv = context.getEnvironment(); - volumesStrategy.prepare(osEnv, workspaceId); - - List createdServices = new ArrayList<>(); - for (Service service : osEnv.getServices().values()) { - createdServices.add(project.services().create(service)); - } - - // List createdRoutes = new ArrayList<>(); - // for (Route route : osEnv.getRoutes().values()) { - // createdRoutes.add(project.routes().create(route)); - // } - // TODO https://github.com/eclipse/che/issues/7653 - // project.pods().watch(new AbnormalStopHandler()); - // project.pods().watchContainers(new MachineLogsPublisher()); - - // needed for resolution later on, even though nroutes are actually created by ingress - // /workspace{wsid}/server-{port} => service({wsid}):server-port => pod({wsid}):{port} - List createdIngresses = new ArrayList<>(); - for (Ingress ingress : osEnv.getIngresses().values()) { - // create - createdIngresses.add(project.kubernetesIngress().create(ingress)); - } - - // wait for LB ip - List actualIngresses = new ArrayList<>(); - for (Ingress ingress : osEnv.getIngresses().values()) { - Ingress actualIngress = - project - .kubernetesIngress() - .wait( - ingress.getMetadata().getName(), - machineStartTimeoutMin, - p -> (p.getStatus().getLoadBalancer().getIngress().size() > 0)); - actualIngresses.add(actualIngress); - } - - createPods(createdServices, actualIngresses); - - // TODO Rework it to parallel waiting https://github.com/eclipse/che/issues/7067 - for (OpenShiftMachine machine : machines.values()) { - try { - machine.waitRunning(machineStartTimeoutMin); - machine.setStatus(MachineStatus.RUNNING); - sendRunningEvent(machine.getName()); - bootstrapMachine(machine); - checkMachineServers(machine); - } catch (InfrastructureException rethrow) { - sendFailedEvent(machine.getName(), rethrow.getMessage()); - throw rethrow; - } - } - } catch (InfrastructureException | RuntimeException | InterruptedException e) { - LOG.warn( - "Failed to start OpenShift runtime of workspace {}. Cause: {}", - workspaceId, - e.getMessage()); - boolean interrupted = Thread.interrupted() || e instanceof InterruptedException; - // Cancels workspace servers probes if any - probeScheduler.cancel(workspaceId); - try { - project.cleanUp(); - } catch (InfrastructureException ignored) { - } - if (interrupted) { - throw new InfrastructureException("OpenShift environment start was interrupted"); - } - try { - throw e; - } catch (InfrastructureException rethrow) { - throw rethrow; - } catch (Exception wrap) { - throw new InternalInfrastructureException(e.getMessage(), wrap); - } - } } @Override - public Map getInternalMachines() { - return ImmutableMap.copyOf(machines); - } - - @Override - protected void internalStop(Map stopOptions) throws InfrastructureException { - // Cancels workspace servers probes if any - probeScheduler.cancel(getContext().getIdentity().getWorkspaceId()); - project.cleanUp(); - } - - @Override - public Map getProperties() { - return emptyMap(); - } - - /** - * Bootstraps machine. - * - * @param machine the OpenShift machine instance to bootstrap - * @throws InfrastructureException when any error occurs while bootstrapping machine - * @throws InterruptedException when machine bootstrapping was interrupted - */ - private void bootstrapMachine(OpenShiftMachine machine) - throws InfrastructureException, InterruptedException { - InternalMachineConfig machineConfig = - getContext().getEnvironment().getMachines().get(machine.getName()); - if (!machineConfig.getInstallers().isEmpty()) - bootstrapperFactory - .create(getContext().getIdentity(), machineConfig.getInstallers(), machine) - .bootstrap(); - } - - /** - * Checks whether machine servers are ready. - * - * @param machine the OpenShift machine instance - * @throws InfrastructureException when any error while server checks occur - * @throws InterruptedException when process of server check was interrupted - */ - private void checkMachineServers(OpenShiftMachine machine) - throws InfrastructureException, InterruptedException { - final ServersChecker check = - serverCheckerFactory.create( - getContext().getIdentity(), machine.getName(), machine.getServers()); - check.startAsync(new ServerReadinessHandler(machine.getName())); - check.await(); - - probeScheduler.schedule( - probesFactory.getProbes( - getContext().getIdentity().getWorkspaceId(), machine.getName(), machine.getServers()), - new ServerLivenessHandler()); - } - - /** - * Creates OpenShift pods and resolves machine servers based on routes and services. - * - * @param services created OpenShift services - * @param ingresses created OpenShift ingresses - * @throws InfrastructureException when any error occurs while creating OpenShift pods - */ - @VisibleForTesting - // void createPods(List services, List routes) throws InfrastructureException { - void createPods(List services, List ingresses) throws InfrastructureException { - final ServerResolver serverResolver = ServerResolver.of(services, ingresses); - final OpenShiftEnvironment environment = getContext().getEnvironment(); - final Map machineConfigs = environment.getMachines(); - for (Pod toCreate : environment.getPods().values()) { - final Pod createdPod = project.pods().create(toCreate); - final ObjectMeta podMetadata = createdPod.getMetadata(); - for (Container container : createdPod.getSpec().getContainers()) { - String machineName = Names.machineName(toCreate, container); - OpenShiftMachine machine = - new OpenShiftMachine( - machineName, - podMetadata.getName(), - container.getName(), - serverResolver.resolve(machineName), - project, - MachineStatus.STARTING, - machineConfigs.get(machineName).getAttributes()); - machines.put(machine.getName(), machine); - sendStartingEvent(machine.getName()); - } - } - } - - private class ServerReadinessHandler implements Consumer { - private String machineName; - - ServerReadinessHandler(String machineName) { - this.machineName = machineName; - } - - @Override - public void accept(String serverRef) { - final OpenShiftMachine machine = machines.get(machineName); - if (machine == null) { - // Probably machine was removed from the list during server check start due to some reason - return; - } - - machine.setServerStatus(serverRef, ServerStatus.RUNNING); - - eventService.publish( - DtoFactory.newDto(ServerStatusEvent.class) - .withIdentity(DtoConverter.asDto(getContext().getIdentity())) - .withMachineName(machineName) - .withServerName(serverRef) - .withStatus(ServerStatus.RUNNING) - .withServerUrl(machine.getServers().get(serverRef).getUrl())); + protected void startMachines() throws InfrastructureException { + OpenShiftEnvironment osEnv = getContext().getEnvironment(); + List createdServices = new ArrayList<>(); + for (Service service : osEnv.getServices().values()) { + createdServices.add(project.services().create(service)); } - } - - private class ServerLivenessHandler implements Consumer { - @Override - public void accept(ProbeResult probeResult) { - String machineName = probeResult.getMachineName(); - OpenShiftMachine machine = machines.get(machineName); - if (machine == null) { - // Probably machine was removed from the list during server check start due to some reason - return; - } - String serverName = probeResult.getServerName(); - ProbeStatus probeStatus = probeResult.getStatus(); - Server server = machine.getServers().get(serverName); - ServerStatus oldServerStatus = server.getStatus(); - ServerStatus serverStatus; - if (probeStatus == ProbeStatus.FAILED && oldServerStatus == ServerStatus.RUNNING) { - serverStatus = ServerStatus.STOPPED; - } else if (probeStatus == ProbeStatus.PASSED && (oldServerStatus != ServerStatus.RUNNING)) { - serverStatus = ServerStatus.RUNNING; - } else { - return; - } - - machine.setServerStatus(serverName, serverStatus); - sendServerStatusEvent(machineName, serverName, machine.getServers().get(serverName)); + List createdRoutes = new ArrayList<>(); + for (Route route : osEnv.getRoutes().values()) { + createdRoutes.add(project.routes().create(route)); } - } + // TODO https://github.com/eclipse/che/issues/7653 + // project.pods().watch(new AbnormalStopHandler()); + // project.pods().watchContainers(new MachineLogsPublisher()); - private void sendStartingEvent(String machineName) { - eventService.publish( - DtoFactory.newDto(MachineStatusEvent.class) - .withIdentity(DtoConverter.asDto(getContext().getIdentity())) - .withEventType(MachineStatus.STARTING) - .withMachineName(machineName)); - } - - private void sendRunningEvent(String machineName) { - eventService.publish( - DtoFactory.newDto(MachineStatusEvent.class) - .withIdentity(DtoConverter.asDto(getContext().getIdentity())) - .withEventType(MachineStatus.RUNNING) - .withMachineName(machineName)); - } - - private void sendFailedEvent(String machineName, String message) { - eventService.publish( - DtoFactory.newDto(MachineStatusEvent.class) - .withIdentity(DtoConverter.asDto(getContext().getIdentity())) - .withEventType(MachineStatus.FAILED) - .withMachineName(machineName) - .withError(message)); - } - - private void sendRuntimeStoppedEvent(String errorMsg) { - eventService.publish( - DtoFactory.newDto(RuntimeStatusEvent.class) - .withIdentity(DtoConverter.asDto(getContext().getIdentity())) - .withStatus(RUNTIME_STOPPED_STATE) - .withPrevStatus(RUNTIME_RUNNING_STATE) - .withFailed(true) - .withError(errorMsg)); - } - - private void sendServerStatusEvent(String machineName, String serverName, Server server) { - eventService.publish( - DtoFactory.newDto(ServerStatusEvent.class) - .withIdentity(DtoConverter.asDto(getContext().getIdentity())) - .withMachineName(machineName) - .withServerName(serverName) - .withStatus(server.getStatus()) - .withServerUrl(server.getUrl())); - } - - /** Listens container's events and publish them as machine logs. */ - class MachineLogsPublisher implements ContainerEventHandler { - - @Override - public void handle(ContainerEvent event) { - final String podName = event.getPodName(); - final String containerName = event.getContainerName(); - for (Entry entry : machines.entrySet()) { - final OpenShiftMachine machine = entry.getValue(); - if (machine.getPodName().equals(podName) - && machine.getContainerName().equals(containerName)) { - eventService.publish( - DtoFactory.newDto(MachineLogEvent.class) - .withMachineName(entry.getKey()) - .withRuntimeId(DtoConverter.asDto(getContext().getIdentity())) - .withText(event.getMessage()) - .withTime(event.getTime())); - return; - } - } - } - } - - /** Stops runtime if one of the pods was abnormally stopped. */ - class AbnormalStopHandler implements PodActionHandler { - - @Override - public void handle(Action action, Pod pod) { - // Cancels workspace servers probes if any - probeScheduler.cancel(getContext().getIdentity().getWorkspaceId()); - if (pod.getStatus() != null && POD_FAILED_STATUS.equals(pod.getStatus().getPhase())) { - try { - internalStop(emptyMap()); - } catch (InfrastructureException ex) { - LOG.error("OpenShift environment stop failed cause '{}'", ex.getMessage()); - } finally { - sendRuntimeStoppedEvent( - format("Pod '%s' was abnormally stopped", pod.getMetadata().getName())); - } - } - } + doStartMachine(new OpenShiftServerResolver(createdServices, createdRoutes)); } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftRuntimeContext.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftRuntimeContext.java index 0e63e8136c1..ead2efc0db6 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftRuntimeContext.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftRuntimeContext.java @@ -11,24 +11,21 @@ package org.eclipse.che.workspace.infrastructure.openshift; import com.google.inject.assistedinject.Assisted; -import java.net.URI; import javax.inject.Inject; import javax.inject.Named; import org.eclipse.che.api.core.ValidationException; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; -import org.eclipse.che.api.workspace.server.spi.RuntimeContext; import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInternalRuntime; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesRuntimeContext; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProjectFactory; /** @author Sergii Leshchenko */ -public class OpenShiftRuntimeContext extends RuntimeContext { - +public class OpenShiftRuntimeContext extends KubernetesRuntimeContext { private final OpenShiftRuntimeFactory runtimeFactory; private final OpenShiftProjectFactory projectFactory; - private final String websocketOutputEndpoint; @Inject public OpenShiftRuntimeContext( @@ -39,24 +36,19 @@ public OpenShiftRuntimeContext( @Assisted RuntimeIdentity identity, @Assisted RuntimeInfrastructure infrastructure) throws ValidationException, InfrastructureException { - super(openShiftEnvironment, identity, infrastructure); - this.projectFactory = projectFactory; + super( + cheWebsocketEndpoint, + projectFactory, + runtimeFactory, + openShiftEnvironment, + identity, + infrastructure); this.runtimeFactory = runtimeFactory; - this.websocketOutputEndpoint = cheWebsocketEndpoint; - } - - @Override - public URI getOutputChannel() throws InfrastructureException { - try { - return URI.create(websocketOutputEndpoint); - } catch (IllegalArgumentException ex) { - throw new InternalInfrastructureException( - "Failed to get the output channel. " + ex.getMessage()); - } + this.projectFactory = projectFactory; } @Override - public OpenShiftInternalRuntime getRuntime() throws InfrastructureException { + public KubernetesInternalRuntime getRuntime() throws InfrastructureException { return runtimeFactory.create( this, projectFactory.create(getIdentity().getWorkspaceId()), diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftRuntimeFactory.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftRuntimeFactory.java index 320bd5929e1..6398484563b 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftRuntimeFactory.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftRuntimeFactory.java @@ -12,10 +12,11 @@ import java.util.List; import org.eclipse.che.api.core.model.workspace.Warning; +import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesRuntimeFactory; import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; /** @author Sergii Leshchenko */ -public interface OpenShiftRuntimeFactory { +public interface OpenShiftRuntimeFactory extends KubernetesRuntimeFactory { OpenShiftInternalRuntime create( OpenShiftRuntimeContext context, OpenShiftProject namespace, List warnings); } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironment.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironment.java index 20f3c13a7aa..45bc0fe7e93 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironment.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironment.java @@ -15,59 +15,45 @@ import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.extensions.Ingress; import io.fabric8.openshift.api.model.Route; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.eclipse.che.api.core.model.workspace.Warning; -import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironment; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; import org.eclipse.che.api.workspace.server.spi.environment.InternalRecipe; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; /** * Holds objects of OpenShift environment. * * @author Sergii Leshchenko */ -public class OpenShiftEnvironment extends InternalEnvironment { +public class OpenShiftEnvironment extends KubernetesEnvironment { public static final String TYPE = "openshift"; - private final Map pods; - private final Map services; private final Map routes; - private final Map ingresses; - private final Map persistentVolumeClaims; + + public OpenShiftEnvironment(KubernetesEnvironment k8sEnv) { + super(k8sEnv); + this.routes = new HashMap<>(); + } public static Builder builder() { return new Builder(); } - private OpenShiftEnvironment( + public OpenShiftEnvironment( InternalRecipe internalRecipe, Map machines, List warnings, Map pods, Map services, - Map routes, Map ingresses, - Map persistentVolumeClaims) { - super(internalRecipe, machines, warnings); - this.pods = pods; - this.services = services; + Map persistentVolumeClaims, + Map routes) { + super(internalRecipe, machines, warnings, pods, services, ingresses, persistentVolumeClaims); this.routes = routes; - this.ingresses = ingresses; - this.persistentVolumeClaims = persistentVolumeClaims; - } - - /** Returns pods that should be created when environment starts. */ - public Map getPods() { - return pods; - } - - /** Returns services that should be created when environment starts. */ - public Map getServices() { - return services; } /** Returns services that should be created when environment starts. */ @@ -75,25 +61,8 @@ public Map getRoutes() { return routes; } - /** Returns services that should be created when environment starts. */ - public Map getIngresses() { - return ingresses; - } - - /** Returns PVCs that should be created when environment starts. */ - public Map getPersistentVolumeClaims() { - return persistentVolumeClaims; - } - - public static class Builder { - private InternalRecipe internalRecipe; - private final Map machines = new HashMap<>(); - private final List warnings = new ArrayList<>(); - private final Map pods = new HashMap<>(); - private final Map services = new HashMap<>(); + public static class Builder extends KubernetesEnvironment.Builder { private final Map routes = new HashMap<>(); - private final Map ingresses = new HashMap<>(); - private final Map persistentVolumeClaims = new HashMap<>(); private Builder() {} @@ -122,11 +91,6 @@ public Builder setServices(Map services) { return this; } - public Builder setRoutes(Map route) { - this.routes.putAll(route); - return this; - } - public Builder setIngresses(Map ingresses) { this.ingresses.putAll(ingresses); return this; @@ -137,6 +101,11 @@ public Builder setPersistentVolumeClaims(Map pvcs return this; } + public Builder setRoutes(Map route) { + this.routes.putAll(route); + return this; + } + public OpenShiftEnvironment build() { return new OpenShiftEnvironment( internalRecipe, @@ -144,9 +113,9 @@ public OpenShiftEnvironment build() { warnings, pods, services, - routes, ingresses, - persistentVolumeClaims); + persistentVolumeClaims, + routes); } } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentFactory.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentFactory.java index f6079e436fd..f6efd811a90 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentFactory.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentFactory.java @@ -42,9 +42,10 @@ import org.eclipse.che.api.workspace.server.spi.environment.InternalRecipe; import org.eclipse.che.api.workspace.server.spi.environment.MachineConfigsValidator; import org.eclipse.che.api.workspace.server.spi.environment.RecipeRetriever; -import org.eclipse.che.workspace.infrastructure.openshift.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironmentValidator; +import org.eclipse.che.workspace.infrastructure.kubernetes.util.Containers; import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; -import org.eclipse.che.workspace.infrastructure.openshift.util.Containers; /** * Parses {@link InternalEnvironment} into {@link OpenShiftEnvironment}. @@ -63,7 +64,7 @@ public class OpenShiftEnvironmentFactory extends InternalEnvironmentFactory pods = new HashMap<>(); Map services = new HashMap<>(); - Map pvcs = new HashMap<>(); boolean isAnyRoutePresent = false; boolean isAnyPVCPresent = false; for (HasMetadata object : list.getItems()) { @@ -152,7 +152,8 @@ protected OpenShiftEnvironment doCreate( .setWarnings(warnings) .setPods(pods) .setServices(services) - .setPersistentVolumeClaims(pvcs) + .setPersistentVolumeClaims(new HashMap<>()) + .setRoutes(new HashMap<>()) .build(); envValidator.validate(osEnv); diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java index 097ba91b544..ad90a949c3f 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProject.java @@ -11,75 +11,52 @@ package org.eclipse.che.workspace.infrastructure.openshift.project; import com.google.common.annotations.VisibleForTesting; -import io.fabric8.kubernetes.api.model.Namespace; -import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.openshift.api.model.Project; import io.fabric8.openshift.api.model.Route; import io.fabric8.openshift.client.OpenShiftClient; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesIngresses; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPersistentVolumeClaims; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPods; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesServices; import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Defines an internal API for managing subset of objects inside {@link Project} instance. * * @author Sergii Leshchenko */ -public class OpenShiftProject { +public class OpenShiftProject extends KubernetesNamespace { - private static final Logger LOG = LoggerFactory.getLogger(OpenShiftProject.class); - - private final String workspaceId; - - private final OpenShiftPods pods; - private final OpenShiftServices services; private final OpenShiftRoutes routes; - private final OpenShiftPersistentVolumeClaims pvcs; - private final KubernetesIngress ingress; @VisibleForTesting OpenShiftProject( String workspaceId, - OpenShiftPods pods, - OpenShiftServices services, + KubernetesPods pods, + KubernetesServices services, OpenShiftRoutes routes, - OpenShiftPersistentVolumeClaims pvcs, - KubernetesIngress kubernetesIngress) { - this.workspaceId = workspaceId; - this.pods = pods; - this.services = services; + KubernetesPersistentVolumeClaims pvcs, + KubernetesIngresses ingresses) { + super(workspaceId, pods, services, pvcs, ingresses); this.routes = routes; - this.pvcs = pvcs; - this.ingress = kubernetesIngress; } public OpenShiftProject(OpenShiftClientFactory clientFactory, String name, String workspaceId) throws InfrastructureException { - this.workspaceId = workspaceId; - this.pods = new OpenShiftPods(name, workspaceId, clientFactory); - this.services = new OpenShiftServices(name, workspaceId, clientFactory); + super(clientFactory, name, workspaceId); this.routes = new OpenShiftRoutes(name, workspaceId, clientFactory); - this.pvcs = new OpenShiftPersistentVolumeClaims(name, clientFactory); - this.ingress = new KubernetesIngress(name, workspaceId, clientFactory); - final OpenShiftClient client = clientFactory.create(); - if (get(name, client) == null) { - create(name, client); - } - } - - /** Returns object for managing {@link Pod} instances inside project. */ - public OpenShiftPods pods() { - return pods; } - /** Returns object for managing {@link Service} instances inside project. */ - public OpenShiftServices services() { - return services; + @Override + protected void doPrepare(String name, KubernetesClient client) throws InfrastructureException { + OpenShiftClient osClient = client.adapt(OpenShiftClient.class); + if (get(name, osClient) == null) { + create(name, osClient); + } } /** Returns object for managing {@link Route} instances inside project. */ @@ -87,59 +64,28 @@ public OpenShiftRoutes routes() { return routes; } - /** Returns object for managing {@link PersistentVolumeClaim} instances inside project. */ - public OpenShiftPersistentVolumeClaims persistentVolumeClaims() { - return pvcs; - } - - /** Returns object for managing {@link PersistentVolumeClaim} instances inside project. */ - public KubernetesIngress kubernetesIngress() { - return ingress; - } - - /** Removes all object except persistent volume claim inside project. */ /** Removes all object except persistent volume claims inside project. */ public void cleanUp() throws InfrastructureException { - doRemove(pods::delete, services::delete, routes::delete); - } - - /** - * Performs all the specified operations and throw exception with composite message if errors - * occurred while any operation execution - */ - private void doRemove(RemoveOperation... operations) throws InfrastructureException { - StringBuilder errors = new StringBuilder(); - for (RemoveOperation operation : operations) { - try { - operation.perform(); - } catch (InternalInfrastructureException e) { - LOG.warn( - "Internal infra error occurred while cleaning project up for workspace with id " - + workspaceId, - e); - errors.append(" ").append(e.getMessage()); - } catch (InfrastructureException e) { - errors.append(" ").append(e.getMessage()); - } - } - - if (errors.length() > 0) { - throw new InfrastructureException( - "Error(s) occurs while cleaning project up." + errors.toString()); - } + doRemove(routes::delete, services()::delete, pods()::delete); } private void create(String projectName, OpenShiftClient client) throws InfrastructureException { try { - client.namespaces().createNew().withNewMetadata().withName(projectName).endMetadata().done(); + client + .projectrequests() + .createNew() + .withNewMetadata() + .withName(projectName) + .endMetadata() + .done(); } catch (KubernetesClientException e) { throw new InfrastructureException(e.getMessage(), e); } } - private Namespace get(String projectName, OpenShiftClient client) throws InfrastructureException { + private Project get(String projectName, OpenShiftClient client) throws InfrastructureException { try { - return client.namespaces().withName(projectName).get(); + return client.projects().withName(projectName).get(); } catch (KubernetesClientException e) { if (e.getCode() == 403) { // project is foreign or doesn't exist @@ -149,8 +95,4 @@ private Namespace get(String projectName, OpenShiftClient client) throws Infrast } } } - - interface RemoveOperation { - void perform() throws InfrastructureException; - } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectFactory.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectFactory.java index c0594698bc9..726ee25e3d8 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectFactory.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectFactory.java @@ -17,6 +17,7 @@ import javax.inject.Named; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.commons.annotation.Nullable; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; /** @@ -25,15 +26,16 @@ * @author Anton Korneta */ @Singleton -public class OpenShiftProjectFactory { +public class OpenShiftProjectFactory extends KubernetesNamespaceFactory { private final String projectName; private final OpenShiftClientFactory clientFactory; @Inject public OpenShiftProjectFactory( - @Nullable @Named("che.infra.openshift.project") String projectName, + @Nullable @Named("che.infra.kubernetes.namespace") String projectName, OpenShiftClientFactory clientFactory) { + super(projectName, clientFactory); this.projectName = projectName; this.clientFactory = clientFactory; } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftRoutes.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftRoutes.java index 1bc01488e36..1a3e2f5c8b8 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftRoutes.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftRoutes.java @@ -10,8 +10,8 @@ */ package org.eclipse.che.workspace.infrastructure.openshift.project; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_WORKSPACE_ID_LABEL; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.putLabel; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_WORKSPACE_ID_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putLabel; import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.openshift.api.model.Route; diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java index cb04905526e..2fbda010fba 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java @@ -39,7 +39,7 @@ public class RemoveProjectOnWorkspaceRemove implements EventSubscriberAdds OpenShift objects by calling {@link ServerExposer#expose(Map)} on each machine with - * servers. + *

Adds OpenShift objects by calling {@link OpenShiftServerExposer#expose(Map)} on each machine + * with servers. * * @author Alexander Garagatyi */ @Singleton -public class ServersConverter implements ConfigurationProvisioner { - - private final String host; - - @Inject - public ServersConverter(@Named("che.host") String host) { - this.host = host; - } +public class OpenShiftServersConverter implements ConfigurationProvisioner { @Override public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) @@ -55,9 +46,9 @@ public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) String machineName = Names.machineName(podConfig, containerConfig); InternalMachineConfig machineConfig = osEnv.getMachines().get(machineName); if (!machineConfig.getServers().isEmpty()) { - ServerExposer serverExposer = - new ServerExposer(machineName, podConfig, containerConfig, osEnv, host); - serverExposer.expose(machineConfig.getServers()); + OpenShiftServerExposer openShiftServerExposer = + new OpenShiftServerExposer(machineName, podConfig, containerConfig, osEnv); + openShiftServerExposer.expose(machineConfig.getServers()); } } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftUniqueNamesProvisioner.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftUniqueNamesProvisioner.java new file mode 100644 index 00000000000..125c52b367d --- /dev/null +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftUniqueNamesProvisioner.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.openshift.provision; + +import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putLabel; + +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.openshift.api.model.Route; +import java.util.HashSet; +import java.util.Set; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.workspace.infrastructure.kubernetes.Constants; +import org.eclipse.che.workspace.infrastructure.kubernetes.Names; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; +import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; + +/** @author Sergii Leshchenko */ +public class OpenShiftUniqueNamesProvisioner extends UniqueNamesProvisioner { + + @Override + public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) + throws InfrastructureException { + super.provision(osEnv, identity); + + final Set routes = new HashSet<>(osEnv.getRoutes().values()); + osEnv.getRoutes().clear(); + for (Route route : routes) { + final ObjectMeta ingressMeta = route.getMetadata(); + putLabel(route, Constants.CHE_ORIGINAL_NAME_LABEL, ingressMeta.getName()); + final String routeName = Names.generateName("route"); + ingressMeta.setName(routeName); + osEnv.getRoutes().put(routeName, route); + } + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/route/TlsRouteProvisioner.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/RouteTlsProvisioner.java similarity index 85% rename from infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/route/TlsRouteProvisioner.java rename to infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/RouteTlsProvisioner.java index 07d1b7612a3..9653dccc416 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/route/TlsRouteProvisioner.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/RouteTlsProvisioner.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.route; +package org.eclipse.che.workspace.infrastructure.openshift.provision; import io.fabric8.openshift.api.model.Route; import io.fabric8.openshift.api.model.RouteSpec; @@ -22,9 +22,9 @@ import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.Annotations; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ConfigurationProvisioner; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.provision.ConfigurationProvisioner; /** * Enables Transport Layer Security (TLS) for workspace routes and changes protocol to secure (wss / @@ -33,13 +33,13 @@ * @author Ilya Buziuk */ @Singleton -public class TlsRouteProvisioner implements ConfigurationProvisioner { - private static final String TERMINATION_EDGE = "edge"; - private static final String TERMINATION_POLICY_REDIRECT = "Redirect"; +public class RouteTlsProvisioner implements ConfigurationProvisioner { + static final String TERMINATION_EDGE = "edge"; + static final String TERMINATION_POLICY_REDIRECT = "Redirect"; private final boolean isTlsEnabled; @Inject - public TlsRouteProvisioner(@Named("che.infra.openshift.tls_enabled") boolean isTlsEnabled) { + public RouteTlsProvisioner(@Named("che.infra.openshift.tls_enabled") boolean isTlsEnabled) { this.isTlsEnabled = isTlsEnabled; } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/UniqueNamesProvisioner.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/UniqueNamesProvisioner.java deleted file mode 100644 index e7749c058ac..00000000000 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/UniqueNamesProvisioner.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che.workspace.infrastructure.openshift.provision; - -import static org.eclipse.che.workspace.infrastructure.openshift.Names.ROUTE_NAME_PREFIX; -import static org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftObjectUtil.putLabel; - -import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.openshift.api.model.Route; -import java.util.HashSet; -import java.util.Set; -import javax.inject.Singleton; -import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; -import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.workspace.infrastructure.openshift.Constants; -import org.eclipse.che.workspace.infrastructure.openshift.Names; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; - -/** - * Makes names of OpenShift pods and routes unique whole namespace by {@link Names}. - * - *

Original names will be stored in {@link Constants#CHE_ORIGINAL_NAME_LABEL} label of renamed - * object. - * - * @author Anton Korneta - * @see Names#uniquePodName(String, String) - * @see Names#generateName(String) - */ -@Singleton -public class UniqueNamesProvisioner implements ConfigurationProvisioner { - - @Override - public void provision(OpenShiftEnvironment osEnv, RuntimeIdentity identity) - throws InfrastructureException { - final String workspaceId = identity.getWorkspaceId(); - final Set pods = new HashSet<>(osEnv.getPods().values()); - osEnv.getPods().clear(); - for (Pod pod : pods) { - final ObjectMeta podMeta = pod.getMetadata(); - putLabel(pod, Constants.CHE_ORIGINAL_NAME_LABEL, podMeta.getName()); - final String podName = Names.uniquePodName(podMeta.getName(), workspaceId); - podMeta.setName(podName); - osEnv.getPods().put(podName, pod); - } - final Set routes = new HashSet<>(osEnv.getRoutes().values()); - osEnv.getRoutes().clear(); - for (Route route : routes) { - final ObjectMeta routeMeta = route.getMetadata(); - putLabel(route, Constants.CHE_ORIGINAL_NAME_LABEL, routeMeta.getName()); - final String routeName = Names.generateName(ROUTE_NAME_PREFIX); - routeMeta.setName(routeName); - osEnv.getRoutes().put(routeName, route); - } - } -} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java new file mode 100644 index 00000000000..3bccf041197 --- /dev/null +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.openshift.server; + +import static java.lang.Integer.parseInt; +import static java.util.stream.Collectors.toMap; + +import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServicePort; +import io.fabric8.openshift.api.model.Route; +import java.util.Map; +import org.eclipse.che.api.core.model.workspace.config.ServerConfig; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; +import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerExposer; +import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; + +/** + * Helps to modify {@link OpenShiftEnvironment} to make servers that are configured by {@link + * ServerConfig} publicly or workspace-wide accessible. + * + *

To make server accessible it is needed to make sure that container port is declared, create + * {@link Service}. To make it also publicly accessible it is needed to create corresponding {@link + * Route} for exposing this port. + * + *

Created services and routes will have serialized servers which are exposed by the + * corresponding object and machine name to which these servers belongs to. + * + *

Container, service and route are linked in the following way: + * + * 

+ * Pod
+ * metadata:
+ *   labels:
+ *     type: web-app
+ * spec:
+ *   containers:
+ *   ...
+ *   - ports:
+ *     - containerPort: 8080
+ *       name: web-app
+ *       protocol: TCP
+ *   ...
+ *
+ * + * Then services expose containers ports in the following way: + * + * 
+ * Service
+ * metadata:
+ *   name: service123
+ * spec:
+ *   selector:                        ---->> Pod.metadata.labels
+ *     type: web-app
+ *   ports:
+ *     - name: web-app
+ *       port: 8080
+ *       targetPort: [8080|web-app]   ---->> Pod.spec.ports[0].[containerPort|name]
+ *       protocol: TCP                ---->> Pod.spec.ports[0].protocol
+ *
+ * + * Then corresponding route expose one of the service's port: + * + * 
+ * Route
+ * ...
+ * spec:
+ *   to:
+ *     name: dev-machine              ---->> Service.metadata.name
+ *     targetPort: [8080|web-app]     ---->> Service.spec.ports[0].[port|name]
+ *
+ * + *

For accessing publicly accessible server user will use route host. For accessing + * workspace-wide accessible server user will use service name. Information about servers that are + * exposed by route and/or service are stored in annotations of a route or service. + * + * @author Sergii Leshchenko + * @author Alexander Garagatyi + * @see Annotations + */ +public class OpenShiftServerExposer extends KubernetesServerExposer { + + private final OpenShiftEnvironment openShiftEnvironment; + + public OpenShiftServerExposer( + String machineName, Pod pod, Container container, OpenShiftEnvironment openShiftEnvironment) { + super(machineName, pod, container, openShiftEnvironment); + this.openShiftEnvironment = openShiftEnvironment; + } + + @Override + protected void exposeExternalServers( + String serviceName, + Map portToServicePort, + Map externalServers) { + for (ServicePort servicePort : portToServicePort.values()) { + int port = servicePort.getTargetPort().getIntVal(); + Map routesServers = + externalServers + .entrySet() + .stream() + .filter(e -> parseInt(e.getValue().getPort().split("/")[0]) == port) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Route route = + new RouteBuilder() + .withName(serviceName + '-' + servicePort.getName()) + .withMachineName(machineName) + .withTargetPort(servicePort.getName()) + .withServers(routesServers) + .withTo(serviceName) + .build(); + openShiftEnvironment.getRoutes().put(route.getMetadata().getName(), route); + } + } + + private static class RouteBuilder { + + private String name; + private String serviceName; + private IntOrString targetPort; + private Map serversConfigs; + private String machineName; + + private RouteBuilder withName(String name) { + this.name = name; + return this; + } + + private RouteBuilder withTo(String serviceName) { + this.serviceName = serviceName; + return this; + } + + private RouteBuilder withTargetPort(String targetPortName) { + this.targetPort = new IntOrString(targetPortName); + return this; + } + + private RouteBuilder withServers(Map serversConfigs) { + this.serversConfigs = serversConfigs; + return this; + } + + public RouteBuilder withMachineName(String machineName) { + this.machineName = machineName; + return this; + } + + private Route build() { + io.fabric8.openshift.api.model.RouteBuilder builder = + new io.fabric8.openshift.api.model.RouteBuilder(); + + return builder + .withNewMetadata() + .withName(name.replace("/", "-")) + .withAnnotations( + Annotations.newSerializer() + .servers(serversConfigs) + .machineName(machineName) + .annotations()) + .endMetadata() + .withNewSpec() + .withNewTo() + .withName(serviceName) + .endTo() + .withNewPort() + .withTargetPort(targetPort) + .endPort() + .endSpec() + .build(); + } + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerResolver.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerResolver.java new file mode 100644 index 00000000000..d0604a6ffb5 --- /dev/null +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerResolver.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.openshift.server; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.openshift.api.model.Route; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.eclipse.che.api.workspace.server.model.impl.ServerImpl; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; +import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerResolver; + +/** + * Helps to resolve {@link ServerImpl servers} by machine name according to specified {@link Route + * routes} and {@link Service services}. + * + *

Objects annotations are used to check if {@link Service service} or {@link Route route} + * exposes the specified machine servers. + * + * @author Sergii Leshchenko + * @author Alexander Garagatyi + * @see OpenShiftServerExposer + * @see Annotations + */ +public class OpenShiftServerResolver extends KubernetesServerResolver { + + private final Multimap routes; + + public OpenShiftServerResolver(List services, List routes) { + super(services, Collections.emptyList()); + + this.routes = ArrayListMultimap.create(); + for (Route route : routes) { + String machineName = + Annotations.newDeserializer(route.getMetadata().getAnnotations()).machineName(); + this.routes.put(machineName, route); + } + } + + @Override + protected void fillExternalServers(String machineName, Map servers) { + routes.get(machineName).forEach(route -> fillRouteServers(route, servers)); + } + + private void fillRouteServers(Route route, Map servers) { + Annotations.newDeserializer(route.getMetadata().getAnnotations()) + .servers() + .forEach( + (name, config) -> + servers.put( + name, + newServer( + config.getProtocol(), + route.getSpec().getHost(), + null, + config.getPath(), + config.getAttributes()))); + } +} diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java index 719903f1dce..20ab71b96ea 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java @@ -14,16 +14,16 @@ import static org.mockito.Mockito.inOrder; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.InstallerServersPortProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.RamLimitProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.restartpolicy.RestartPolicyRewriter; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.project.pvc.WorkspaceVolumesStrategy; -import org.eclipse.che.workspace.infrastructure.openshift.provision.InstallerServersPortProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.LogsVolumeMachineProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.UniqueNamesProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.env.EnvVarsConverter; -import org.eclipse.che.workspace.infrastructure.openshift.provision.limits.ram.RamLimitProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.restartpolicy.RestartPolicyRewriter; -import org.eclipse.che.workspace.infrastructure.openshift.provision.route.TlsRouteProvisioner; -import org.eclipse.che.workspace.infrastructure.openshift.provision.server.ServersConverter; +import org.eclipse.che.workspace.infrastructure.openshift.provision.OpenShiftServersConverter; +import org.eclipse.che.workspace.infrastructure.openshift.provision.OpenShiftUniqueNamesProvisioner; +import org.eclipse.che.workspace.infrastructure.openshift.provision.RouteTlsProvisioner; import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; @@ -41,12 +41,12 @@ public class OpenShiftEnvironmentProvisionerTest { @Mock private WorkspaceVolumesStrategy volumesStrategy; @Mock private InstallerServersPortProvisioner installerServersPortProvisioner; - @Mock private UniqueNamesProvisioner uniqueNamesProvisioner; + @Mock private OpenShiftUniqueNamesProvisioner uniqueNamesProvisioner; @Mock private OpenShiftEnvironment osEnv; @Mock private RuntimeIdentity runtimeIdentity; - @Mock private TlsRouteProvisioner tlsRouteProvisioner; + @Mock private RouteTlsProvisioner tlsRouteProvisioner; @Mock private EnvVarsConverter envVarsProvisioner; - @Mock private ServersConverter serversProvisioner; + @Mock private OpenShiftServersConverter serversProvisioner; @Mock private RestartPolicyRewriter restartPolicyRewriter; @Mock private RamLimitProvisioner ramLimitProvisioner; @Mock private LogsVolumeMachineProvisioner logsVolumeMachineProvisioner; diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java index 84e814a9dc5..3d07c766919 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java @@ -11,23 +11,16 @@ package org.eclipse.che.workspace.infrastructure.openshift; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; -import static org.eclipse.che.api.core.model.workspace.runtime.MachineStatus.FAILED; -import static org.eclipse.che.api.core.model.workspace.runtime.MachineStatus.RUNNING; import static org.eclipse.che.api.core.model.workspace.runtime.MachineStatus.STARTING; import static org.eclipse.che.dto.server.DtoFactory.newDto; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_ORIGINAL_NAME_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_ORIGINAL_NAME_LABEL; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -66,24 +59,18 @@ import org.eclipse.che.api.workspace.server.hc.ServersChecker; import org.eclipse.che.api.workspace.server.hc.ServersCheckerFactory; import org.eclipse.che.api.workspace.server.hc.probe.ProbeScheduler; -import org.eclipse.che.api.workspace.server.hc.probe.WorkspaceProbes; import org.eclipse.che.api.workspace.server.hc.probe.WorkspaceProbesFactory; import org.eclipse.che.api.workspace.server.model.impl.RuntimeIdentityImpl; -import org.eclipse.che.api.workspace.server.spi.InfrastructureException; -import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; -import org.eclipse.che.api.workspace.shared.dto.event.MachineLogEvent; import org.eclipse.che.api.workspace.shared.dto.event.MachineStatusEvent; -import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftInternalRuntime.MachineLogsPublisher; -import org.eclipse.che.workspace.infrastructure.openshift.bootstrapper.OpenShiftBootstrapper; -import org.eclipse.che.workspace.infrastructure.openshift.bootstrapper.OpenShiftBootstrapperFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapper; +import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPods; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesServices; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftPods; import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftRoutes; -import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftServices; -import org.eclipse.che.workspace.infrastructure.openshift.project.event.ContainerEvent; -import org.eclipse.che.workspace.infrastructure.openshift.project.pvc.WorkspaceVolumesStrategy; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; @@ -97,7 +84,6 @@ * @author Anton Korneta */ public class OpenShiftInternalRuntimeTest { - private static final int EXPOSED_PORT_1 = 4401; private static final int EXPOSED_PORT_2 = 8081; private static final int INTERNAL_PORT = 4411; @@ -120,17 +106,16 @@ public class OpenShiftInternalRuntimeTest { @Mock private EventService eventService; @Mock private ServersCheckerFactory serverCheckerFactory; @Mock private ServersChecker serversChecker; - @Mock private OpenShiftBootstrapperFactory bootstrapperFactory; + @Mock private KubernetesBootstrapperFactory bootstrapperFactory; @Mock private OpenShiftEnvironment osEnv; @Mock private OpenShiftProject project; - @Mock private OpenShiftServices services; + @Mock private KubernetesServices services; @Mock private OpenShiftRoutes routes; - @Mock private OpenShiftPods pods; - @Mock private OpenShiftBootstrapper bootstrapper; + @Mock private KubernetesPods pods; + @Mock private KubernetesBootstrapper bootstrapper; @Mock private WorkspaceVolumesStrategy volumesStrategy; @Mock private WorkspaceProbesFactory workspaceProbesFactory; @Mock private ProbeScheduler probesScheduler; - @Mock private WorkspaceProbes workspaceProbes; @Captor private ArgumentCaptor machineStatusEventCaptor; @@ -185,184 +170,21 @@ public void setup() throws Exception { } @Test - public void startsOpenShiftEnvironment() throws Exception { + public void shouldStartMachines() throws Exception { final Container container1 = mockContainer(CONTAINER_NAME_1, EXPOSED_PORT_1); final Container container2 = mockContainer(CONTAINER_NAME_2, EXPOSED_PORT_2, INTERNAL_PORT); final ImmutableMap allPods = ImmutableMap.of(POD_NAME, mockPod(ImmutableList.of(container1, container2))); when(osEnv.getPods()).thenReturn(allPods); - internalRuntime.internalStart(emptyMap()); + internalRuntime.startMachines(); verify(pods).create(any()); - // verify(routes).create(any()); + verify(routes).create(any()); verify(services).create(any()); - verify(bootstrapper, times(2)).bootstrap(); - verify(eventService, times(4)).publish(any()); - verifyEventsOrder( - newEvent(M1_NAME, STARTING), - newEvent(M2_NAME, STARTING), - newEvent(M1_NAME, RUNNING), - newEvent(M2_NAME, RUNNING)); - verify(serverCheckerFactory).create(IDENTITY, M1_NAME, emptyMap()); - verify(serverCheckerFactory).create(IDENTITY, M2_NAME, emptyMap()); - verify(serversChecker, times(2)).startAsync(any()); - } - - @Test(expectedExceptions = InternalInfrastructureException.class) - public void throwsInternalInfrastructureExceptionWhenRuntimeErrorOccurs() throws Exception { - doNothing().when(project).cleanUp(); - when(osEnv.getServices()).thenThrow(new RuntimeException()); - - try { - internalRuntime.internalStart(emptyMap()); - } catch (Exception rethrow) { - verify(project).cleanUp(); - verify(project, never()).services(); - verify(project, never()).routes(); - verify(project, never()).pods(); - throw rethrow; - } - } - - @Test(expectedExceptions = InfrastructureException.class) - public void stopsWaitingAllMachineStartWhenOneMachineStartFailed() throws Exception { - final Container container1 = mockContainer(CONTAINER_NAME_1, EXPOSED_PORT_1); - final Container container2 = mockContainer(CONTAINER_NAME_2, EXPOSED_PORT_2, INTERNAL_PORT); - final ImmutableMap allPods = - ImmutableMap.of(POD_NAME, mockPod(ImmutableList.of(container1, container2))); - when(osEnv.getPods()).thenReturn(allPods); - doThrow(InfrastructureException.class).when(bootstrapper).bootstrap(); - - try { - internalRuntime.internalStart(emptyMap()); - } catch (Exception rethrow) { - verify(pods).create(any()); - // verify(routes).create(any()); - verify(services).create(any()); - verify(bootstrapper).bootstrap(); - verify(eventService, times(4)).publish(any()); - verifyEventsOrder( - newEvent(M1_NAME, STARTING), - newEvent(M2_NAME, STARTING), - newEvent(M1_NAME, RUNNING), - newEvent(M1_NAME, FAILED)); - throw rethrow; - } - } - - @Test(expectedExceptions = InfrastructureException.class) - public void throwsInfrastructureExceptionWhenErrorOccursAndCleanupFailed() throws Exception { - doNothing().doThrow(InfrastructureException.class).when(project).cleanUp(); - when(osEnv.getServices()).thenReturn(singletonMap("testService", mock(Service.class))); - when(services.create(any())).thenThrow(new InfrastructureException("service creation failed")); - doThrow(InfrastructureException.class).when(project).services(); - - try { - internalRuntime.internalStart(emptyMap()); - } catch (Exception rethrow) { - verify(project).cleanUp(); - verify(project).services(); - verify(project, never()).routes(); - verify(project, never()).pods(); - throw rethrow; - } - } - - @Test(expectedExceptions = InfrastructureException.class) - public void throwsInfrastructureExceptionWhenBootstrapInterrupted() throws Exception { - doThrow(InterruptedException.class).when(bootstrapper).bootstrap(); - try { - internalRuntime.internalStart(emptyMap()); - } catch (Exception rethrow) { - verify(project).cleanUp(); - verify(pods).create(any()); - // verify(routes).create(any()); - verify(services).create(any()); - verify(bootstrapper).bootstrap(); - verifyEventsOrder(newEvent(M1_NAME, STARTING), newEvent(M1_NAME, RUNNING)); - throw rethrow; - } - } - - @Test - public void stopsOpenShiftEnvironment() throws Exception { - doNothing().when(project).cleanUp(); - - internalRuntime.internalStop(emptyMap()); - - verify(project).cleanUp(); - } - - @Test(expectedExceptions = InfrastructureException.class) - public void throwsInfrastructureExceptionWhenOpenShiftProjectCleanupFailed() throws Exception { - doThrow(InfrastructureException.class).when(project).cleanUp(); - - internalRuntime.internalStop(emptyMap()); - } - - // @Test - // public void testRepublishContainerOutputAsMachineLogEvents() throws Exception { - // final MachineLogsPublisher logsPublisher = internalRuntime.new MachineLogsPublisher(); - // final ContainerEvent out1 = mockContainerEvent("pulling image", "07/07/2007 19:01:22"); - // final ContainerEvent out2 = mockContainerEvent("image pulled", "07/07/2007 19:08:53"); - // final ArgumentCaptor captor = - // ArgumentCaptor.forClass(MachineLogEvent.class); - // - // internalRuntime.createPods( - // newArrayList(allServices.values()), newArrayList(allRoutes.values())); - // logsPublisher.handle(out1); - // logsPublisher.handle(out2); - // - // verify(eventService, atLeastOnce()).publish(captor.capture()); - // final ImmutableList machineLogs = - // ImmutableList.of(asMachineLogEvent(out1), asMachineLogEvent(out2)); - // assertTrue(captor.getAllValues().containsAll(machineLogs)); - // } - - @Test - public void testDoNotPublishForeignMachineOutput() throws Exception { - final MachineLogsPublisher logsPublisher = internalRuntime.new MachineLogsPublisher(); - final ContainerEvent out1 = mockContainerEvent("folder created", "33/03/2033 19:01:06"); - - logsPublisher.handle(out1); - - verify(eventService, never()).publish(any()); - } - - @Test - public void cancelsWsProbesOnRuntimeStop() throws Exception { - doNothing().when(project).cleanUp(); - - internalRuntime.internalStop(emptyMap()); - - verify(probesScheduler).cancel(WORKSPACE_ID); - } - - @Test - public void cancelsWsProbesWhenErrorOnRuntimeStartOccurs() throws Exception { - doNothing().when(project).cleanUp(); - when(osEnv.getServices()).thenThrow(new RuntimeException()); - - try { - internalRuntime.internalStart(emptyMap()); - } catch (Exception e) { - verify(probesScheduler).cancel(WORKSPACE_ID); - return; - } - fail(); - } - - @Test - public void schedulesProbesOnRuntimeStart() throws Exception { - doNothing().when(project).cleanUp(); - when(workspaceProbesFactory.getProbes(eq(WORKSPACE_ID), anyString(), any())) - .thenReturn(workspaceProbes); - - internalRuntime.internalStart(emptyMap()); - - verify(probesScheduler).schedule(eq(workspaceProbes), any()); + verify(eventService, times(2)).publish(any()); + verifyEventsOrder(newEvent(M1_NAME, STARTING), newEvent(M2_NAME, STARTING)); } private static MachineStatusEvent newEvent(String machineName, MachineStatus status) { @@ -460,23 +282,6 @@ private static ObjectMeta mockName(String name, HasMetadata mock) { return metadata; } - private static ContainerEvent mockContainerEvent(String message, String time) { - final ContainerEvent event = mock(ContainerEvent.class); - when(event.getPodName()).thenReturn(POD_NAME); - when(event.getContainerName()).thenReturn(CONTAINER_NAME_1); - when(event.getMessage()).thenReturn(message); - when(event.getTime()).thenReturn(time); - return event; - } - - private static MachineLogEvent asMachineLogEvent(ContainerEvent event) { - return newDto(MachineLogEvent.class) - .withRuntimeId(DtoConverter.asDto(IDENTITY)) - .withText(event.getMessage()) - .withTime(event.getTime()) - .withMachineName(event.getPodName() + '/' + event.getContainerName()); - } - private static IntOrString intOrString(int port) { return new IntOrStringBuilder().withIntVal(port).withStrVal(String.valueOf(port)).build(); } diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposerTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftServerExposerTest.java similarity index 96% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposerTest.java rename to infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftServerExposerTest.java index 3ca9a9febe1..395a93a9466 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerExposerTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftServerExposerTest.java @@ -12,8 +12,8 @@ import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static org.eclipse.che.workspace.infrastructure.openshift.ServerExposer.SERVER_PREFIX; -import static org.eclipse.che.workspace.infrastructure.openshift.ServerExposer.SERVER_UNIQUE_PART_SIZE; +import static org.eclipse.che.workspace.infrastructure.openshift.server.OpenShiftServerExposer.SERVER_PREFIX; +import static org.eclipse.che.workspace.infrastructure.openshift.server.OpenShiftServerExposer.SERVER_UNIQUE_PART_SIZE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; @@ -35,16 +35,18 @@ import java.util.regex.Pattern; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; +import org.eclipse.che.workspace.infrastructure.openshift.server.OpenShiftServerExposer; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; /** - * Test for {@link ServerExposer}. + * Test for {@link OpenShiftServerExposer}. * * @author Sergii Leshchenko */ -public class ServerExposerTest { +public class OpenShiftServerExposerTest { private static final Map ATTRIBUTES_MAP = singletonMap("key", "value"); private static final Map INTERNAL_SERVER_ATTRIBUTE_MAP = @@ -54,7 +56,7 @@ public class ServerExposerTest { Pattern.compile('^' + SERVER_PREFIX + "[A-z0-9]{" + SERVER_UNIQUE_PART_SIZE + "}-pod-main$"); public static final String MACHINE_NAME = "pod/main"; - private ServerExposer serverExposer; + private OpenShiftServerExposer serverExposer; private OpenShiftEnvironment openShiftEnvironment; private Container container; @@ -73,7 +75,8 @@ public void setUp() throws Exception { openShiftEnvironment = OpenShiftEnvironment.builder().setPods(ImmutableMap.of("pod", pod)).build(); - this.serverExposer = new ServerExposer(MACHINE_NAME, pod, container, openShiftEnvironment, ""); + this.serverExposer = + new OpenShiftServerExposer(MACHINE_NAME, pod, container, openShiftEnvironment); } @Test diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftServerResolverTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftServerResolverTest.java new file mode 100644 index 00000000000..f713b3414f9 --- /dev/null +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftServerResolverTest.java @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.openshift; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.eclipse.che.api.core.model.workspace.runtime.ServerStatus.UNKNOWN; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import com.google.common.collect.ImmutableMap; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServiceBuilder; +import io.fabric8.kubernetes.api.model.ServicePortBuilder; +import io.fabric8.openshift.api.model.Route; +import io.fabric8.openshift.api.model.RouteBuilder; +import java.util.Map; +import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; +import org.eclipse.che.api.workspace.server.model.impl.ServerImpl; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations.Serializer; +import org.eclipse.che.workspace.infrastructure.openshift.server.OpenShiftServerResolver; +import org.testng.annotations.Test; + +/** + * Test for {@link OpenShiftServerResolver}. + * + * @author Sergii Leshchenko + */ +public class OpenShiftServerResolverTest { + + private static final Map ATTRIBUTES_MAP = singletonMap("key", "value"); + private static final int CONTAINER_PORT = 3054; + private static final String ROUTE_HOST = "localhost"; + + @Test + public void + testResolvingServersWhenThereIsNoTheCorrespondingServiceAndRouteForTheSpecifiedMachine() { + // given + Service nonMatchedByPodService = + createService("nonMatched", "foreignMachine", CONTAINER_PORT, null); + Route route = + createRoute( + "nonMatched", + "foreignMachine", + ImmutableMap.of( + "http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); + + OpenShiftServerResolver serverResolver = + new OpenShiftServerResolver(singletonList(nonMatchedByPodService), singletonList(route)); + + // when + Map resolved = serverResolver.resolve("machine"); + + // then + assertTrue(resolved.isEmpty()); + } + + @Test + public void testResolvingServersWhenThereIsMatchedRouteForTheSpecifiedMachine() { + Route route = + createRoute( + "matched", + "machine", + ImmutableMap.of( + "http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); + + OpenShiftServerResolver serverResolver = + new OpenShiftServerResolver(emptyList(), singletonList(route)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://localhost/api") + .withStatus(UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsNull() { + Route route = + createRoute( + "matched", + "machine", + singletonMap( + "http-server", new ServerConfigImpl("3054", "http", null, ATTRIBUTES_MAP))); + + OpenShiftServerResolver serverResolver = + new OpenShiftServerResolver(emptyList(), singletonList(route)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://localhost") + .withStatus(UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsEmpty() { + Route route = + createRoute( + "matched", + "machine", + singletonMap("http-server", new ServerConfigImpl("3054", "http", "", ATTRIBUTES_MAP))); + + OpenShiftServerResolver serverResolver = + new OpenShiftServerResolver(emptyList(), singletonList(route)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://localhost") + .withStatus(UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsRelative() { + Route route = + createRoute( + "matched", + "machine", + singletonMap( + "http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); + + OpenShiftServerResolver serverResolver = + new OpenShiftServerResolver(emptyList(), singletonList(route)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://localhost/api") + .withStatus(UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingInternalServers() { + Service service = + createService( + "service11", + "machine", + CONTAINER_PORT, + singletonMap( + "http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); + Route route = createRoute("matched", "machine", null); + + OpenShiftServerResolver serverResolver = + new OpenShiftServerResolver(singletonList(service), singletonList(route)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("http://service11:3054/api") + .withStatus(UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + @Test + public void testResolvingInternalServersWithPortWithTransportProtocol() { + Service service = + createService( + "service11", + "machine", + CONTAINER_PORT, + singletonMap( + "http-server", new ServerConfigImpl("3054/udp", "xxx", "api", ATTRIBUTES_MAP))); + Route route = createRoute("matched", "machine", null); + + OpenShiftServerResolver serverResolver = + new OpenShiftServerResolver(singletonList(service), singletonList(route)); + + Map resolved = serverResolver.resolve("machine"); + + assertEquals(resolved.size(), 1); + assertEquals( + resolved.get("http-server"), + new ServerImpl() + .withUrl("xxx://service11:3054/api") + .withStatus(UNKNOWN) + .withAttributes(ATTRIBUTES_MAP)); + } + + private Service createService( + String name, String machineName, Integer port, Map servers) { + Serializer serializer = Annotations.newSerializer(); + serializer.machineName(machineName); + if (servers != null) { + serializer.servers(servers); + } + + return new ServiceBuilder() + .withNewMetadata() + .withName(name) + .withAnnotations(serializer.annotations()) + .endMetadata() + .withNewSpec() + .withPorts( + new ServicePortBuilder() + .withPort(port) + .withNewTargetPort() + .withIntVal(port) + .endTargetPort() + .build()) + .endSpec() + .build(); + } + + private Route createRoute( + String name, String machineName, Map servers) { + Serializer serializer = Annotations.newSerializer(); + serializer.machineName(machineName); + if (servers != null) { + serializer.servers(servers); + } + return new RouteBuilder() + .withNewMetadata() + .withName(name) + .withAnnotations(serializer.annotations()) + .endMetadata() + .withNewSpec() + .withHost(ROUTE_HOST) + .withNewTo() + .withName(name) + .endTo() + .endSpec() + .build(); + } +} diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolverTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolverTest.java deleted file mode 100644 index a3193f6a8b4..00000000000 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/ServerResolverTest.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che.workspace.infrastructure.openshift; - -import static java.util.Collections.singletonMap; - -import java.util.Map; - -/** - * Test for {@link ServerResolver}. - * - * @author Sergii Leshchenko - */ -public class ServerResolverTest { - - private static final Map ATTRIBUTES_MAP = singletonMap("key", "value"); - private static final int CONTAINER_PORT = 3054; - private static final String ROUTE_HOST = "localhost"; - - // @Test - // public void - // testResolvingServersWhenThereIsNoTheCorrespondingServiceAndRouteForTheSpecifiedMachine() { - // // given - // Service nonMatchedByPodService = - // createService("nonMatched", "foreignMachine", CONTAINER_PORT, null); - // Route route = - // createRoute( - // "nonMatched", - // "foreignMachine", - // ImmutableMap.of( - // "http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); - // - // ServerResolver serverResolver = - // ServerResolver.of(singletonList(nonMatchedByPodService), singletonList(route)); - // - // // when - // Map resolved = serverResolver.resolve("machine"); - // - // // then - // assertTrue(resolved.isEmpty()); - // } - // - // @Test - // public void testResolvingServersWhenThereIsMatchedRouteForTheSpecifiedMachine() { - // Route route = - // createRoute( - // "matched", - // "machine", - // ImmutableMap.of( - // "http-server", new ServerConfigImpl("3054", "http", "/api", ATTRIBUTES_MAP))); - // - // ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); - // - // Map resolved = serverResolver.resolve("machine"); - // - // assertEquals(resolved.size(), 1); - // assertEquals( - // resolved.get("http-server"), - // new ServerImpl() - // .withUrl("http://localhost/api") - // .withStatus(UNKNOWN) - // .withAttributes(ATTRIBUTES_MAP)); - // } - // - // @Test - // public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsNull() { - // Route route = - // createRoute( - // "matched", - // "machine", - // singletonMap( - // "http-server", new ServerConfigImpl("3054", "http", null, ATTRIBUTES_MAP))); - // - // ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); - // - // Map resolved = serverResolver.resolve("machine"); - // - // assertEquals(resolved.size(), 1); - // assertEquals( - // resolved.get("http-server"), - // new ServerImpl() - // .withUrl("http://localhost") - // .withStatus(UNKNOWN) - // .withAttributes(ATTRIBUTES_MAP)); - // } - // - // @Test - // public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsEmpty() { - // Route route = - // createRoute( - // "matched", - // "machine", - // singletonMap("http-server", new ServerConfigImpl("3054", "http", "", - // ATTRIBUTES_MAP))); - // - // ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); - // - // Map resolved = serverResolver.resolve("machine"); - // - // assertEquals(resolved.size(), 1); - // assertEquals( - // resolved.get("http-server"), - // new ServerImpl() - // .withUrl("http://localhost") - // .withStatus(UNKNOWN) - // .withAttributes(ATTRIBUTES_MAP)); - // } - // - // @Test - // public void testResolvingServersWhenThereIsMatchedRouteForMachineAndServerPathIsRelative() { - // Route route = - // createRoute( - // "matched", - // "machine", - // singletonMap( - // "http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); - // - // ServerResolver serverResolver = ServerResolver.of(emptyList(), singletonList(route)); - // - // Map resolved = serverResolver.resolve("machine"); - // - // assertEquals(resolved.size(), 1); - // assertEquals( - // resolved.get("http-server"), - // new ServerImpl() - // .withUrl("http://localhost/api") - // .withStatus(UNKNOWN) - // .withAttributes(ATTRIBUTES_MAP)); - // } - // - // @Test - // public void testResolvingInternalServers() { - // Service service = - // createService( - // "service11", - // "machine", - // CONTAINER_PORT, - // singletonMap( - // "http-server", new ServerConfigImpl("3054", "http", "api", ATTRIBUTES_MAP))); - // Route route = createRoute("matched", "machine", null); - // - // ServerResolver serverResolver = ServerResolver.of(singletonList(service), - // singletonList(route)); - // - // Map resolved = serverResolver.resolve("machine"); - // - // assertEquals(resolved.size(), 1); - // assertEquals( - // resolved.get("http-server"), - // new ServerImpl() - // .withUrl("http://service11:3054/api") - // .withStatus(UNKNOWN) - // .withAttributes(ATTRIBUTES_MAP)); - // } - // - // @Test - // public void testResolvingInternalServersWithPortWithTransportProtocol() { - // Service service = - // createService( - // "service11", - // "machine", - // CONTAINER_PORT, - // singletonMap( - // "http-server", new ServerConfigImpl("3054/udp", "xxx", "api", ATTRIBUTES_MAP))); - // Route route = createRoute("matched", "machine", null); - // - // ServerResolver serverResolver = ServerResolver.of(singletonList(service), - // singletonList(route)); - // - // Map resolved = serverResolver.resolve("machine"); - // - // assertEquals(resolved.size(), 1); - // assertEquals( - // resolved.get("http-server"), - // new ServerImpl() - // .withUrl("xxx://service11:3054/api") - // .withStatus(UNKNOWN) - // .withAttributes(ATTRIBUTES_MAP)); - // } - // - // private Service createService( - // String name, String machineName, Integer port, Map servers) { - // Serializer serializer = Annotations.newSerializer(); - // serializer.machineName(machineName); - // if (servers != null) { - // serializer.servers(servers); - // } - // - // return new ServiceBuilder() - // .withNewMetadata() - // .withName(name) - // .withAnnotations(serializer.annotations()) - // .endMetadata() - // .withNewSpec() - // .withPorts( - // new ServicePortBuilder() - // .withPort(port) - // .withNewTargetPort() - // .withIntVal(port) - // .endTargetPort() - // .build()) - // .endSpec() - // .build(); - // } - // - // private Route createRoute( - // String name, String machineName, Map servers) { - // Serializer serializer = Annotations.newSerializer(); - // serializer.machineName(machineName); - // if (servers != null) { - // serializer.servers(servers); - // } - // return new RouteBuilder() - // .withNewMetadata() - // .withName(name) - // .withAnnotations(serializer.annotations()) - // .endMetadata() - // .withNewSpec() - // .withHost(ROUTE_HOST) - // .withNewTo() - // .withName(name) - // .endTo() - // .endSpec() - // .build(); - // } -} diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentFactoryTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentFactoryTest.java index b10d09e9ed5..7b4f572a5e9 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentFactoryTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/environment/OpenShiftEnvironmentFactoryTest.java @@ -17,7 +17,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.eclipse.che.api.core.model.workspace.config.MachineConfig.MEMORY_LIMIT_ATTRIBUTE; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.MACHINE_NAME_ANNOTATION_FMT; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.MACHINE_NAME_ANNOTATION_FMT; import static org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironmentFactory.PVC_IGNORED_WARNING_CODE; import static org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironmentFactory.PVC_IGNORED_WARNING_MESSAGE; import static org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironmentFactory.ROUTES_IGNORED_WARNING_MESSAGE; @@ -56,6 +56,7 @@ import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironment; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; import org.eclipse.che.api.workspace.server.spi.environment.InternalRecipe; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironmentValidator; import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; @@ -80,7 +81,7 @@ public class OpenShiftEnvironmentFactoryTest { private OpenShiftEnvironmentFactory osEnvironmentFactory; @Mock private OpenShiftClientFactory clientFactory; - @Mock private OpenShiftEnvironmentValidator osEnvValidator; + @Mock private KubernetesEnvironmentValidator k8sEnvValidator; @Mock private OpenShiftClient client; @Mock private InternalEnvironment internalEnvironment; @Mock private InternalRecipe internalRecipe; @@ -99,7 +100,7 @@ public class OpenShiftEnvironmentFactoryTest { public void setup() throws Exception { osEnvironmentFactory = new OpenShiftEnvironmentFactory( - null, null, null, clientFactory, osEnvValidator, DEFAULT_RAM_LIMIT_MB); + null, null, null, clientFactory, k8sEnvValidator, DEFAULT_RAM_LIMIT_MB); when(clientFactory.create()).thenReturn(client); when(client.lists()).thenReturn(listMixedOperation); when(listMixedOperation.load(any(InputStream.class))).thenReturn(serverGettable); diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectTest.java index 444ab5542f9..1eb8d3c4ff7 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/project/OpenShiftProjectTest.java @@ -28,6 +28,10 @@ import io.fabric8.openshift.client.OpenShiftClient; import io.fabric8.openshift.client.dsl.ProjectRequestOperation; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesIngresses; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPersistentVolumeClaims; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPods; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesServices; import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftClientFactory; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; @@ -46,22 +50,22 @@ public class OpenShiftProjectTest { public static final String PROJECT_NAME = "testProject"; public static final String WORKSPACE_ID = "workspace123"; - @Mock private OpenShiftPods pods; - @Mock private OpenShiftServices services; + @Mock private KubernetesPods pods; + @Mock private KubernetesServices services; @Mock private OpenShiftRoutes routes; - @Mock private OpenShiftPersistentVolumeClaims pvcs; + @Mock private KubernetesPersistentVolumeClaims pvcs; + @Mock private KubernetesIngresses ingresses; @Mock private OpenShiftClientFactory clientFactory; @Mock private OpenShiftClient openShiftClient; - @Mock private KubernetesIngress kubernetesIngress; private OpenShiftProject openShiftProject; @BeforeMethod public void setUp() throws Exception { when(clientFactory.create()).thenReturn(openShiftClient); + when(openShiftClient.adapt(OpenShiftClient.class)).thenReturn(openShiftClient); - openShiftProject = - new OpenShiftProject(WORKSPACE_ID, pods, services, routes, pvcs, kubernetesIngress); + openShiftProject = new OpenShiftProject(WORKSPACE_ID, pods, services, routes, pvcs, ingresses); } @Test @@ -94,15 +98,15 @@ public void testOpenShiftProjectCleaningUp() throws Exception { // when openShiftProject.cleanUp(); - verify(pods).delete(); - verify(services).delete(); verify(routes).delete(); + verify(services).delete(); + verify(pods).delete(); } @Test public void testOpenShiftProjectCleaningUpIfExceptionsOccurs() throws Exception { - doThrow(new InfrastructureException("err1.")).when(pods).delete(); - doThrow(new InfrastructureException("err2.")).when(services).delete(); + doThrow(new InfrastructureException("err1.")).when(services).delete(); + doThrow(new InfrastructureException("err2.")).when(pods).delete(); InfrastructureException error = null; // when @@ -116,7 +120,7 @@ public void testOpenShiftProjectCleaningUpIfExceptionsOccurs() throws Exception // then assertNotNull(error); String message = error.getMessage(); - assertEquals(message, "Error(s) occurs while cleaning project up. err1. err2."); + assertEquals(message, "Error(s) occurs while cleaning up the namespace. err1. err2."); verify(routes).delete(); } diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/LogsVolumeMachineProvisionerTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/LogsVolumeMachineProvisionerTest.java index 9e8cf45a819..c88661eaf07 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/LogsVolumeMachineProvisionerTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/LogsVolumeMachineProvisionerTest.java @@ -10,7 +10,7 @@ */ package org.eclipse.che.workspace.infrastructure.openshift.provision; -import static org.eclipse.che.workspace.infrastructure.openshift.provision.LogsVolumeMachineProvisioner.LOGS_VOLUME_NAME; +import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner.LOGS_VOLUME_NAME; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -19,6 +19,7 @@ import java.util.HashMap; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/UniqueNamesProvisionerTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftUniqueNamesProvisionerTest.java similarity index 90% rename from infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/UniqueNamesProvisionerTest.java rename to infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftUniqueNamesProvisionerTest.java index 7a936321dd0..3252cdc4d35 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/UniqueNamesProvisionerTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/OpenShiftUniqueNamesProvisionerTest.java @@ -10,7 +10,7 @@ */ package org.eclipse.che.workspace.infrastructure.openshift.provision; -import static org.eclipse.che.workspace.infrastructure.openshift.Constants.CHE_ORIGINAL_NAME_LABEL; +import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_ORIGINAL_NAME_LABEL; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; @@ -32,12 +32,12 @@ import org.testng.annotations.Test; /** - * Tests {@link UniqueNamesProvisioner}. + * Tests {@link OpenShiftUniqueNamesProvisioner}. * * @author Anton Korneta */ @Listeners(MockitoTestNGListener.class) -public class UniqueNamesProvisionerTest { +public class OpenShiftUniqueNamesProvisionerTest { private static final String WORKSPACE_ID = "workspace37"; private static final String POD_NAME = "testPod"; @@ -46,11 +46,11 @@ public class UniqueNamesProvisionerTest { @Mock private OpenShiftEnvironment osEnv; @Mock private RuntimeIdentity runtimeIdentity; - private UniqueNamesProvisioner uniqueNamesProvisioner; + private OpenShiftUniqueNamesProvisioner uniqueNamesProvisioner; @BeforeMethod public void setup() { - uniqueNamesProvisioner = new UniqueNamesProvisioner(); + uniqueNamesProvisioner = new OpenShiftUniqueNamesProvisioner(); } @Test diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/RouteTlsProvisionerTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/RouteTlsProvisionerTest.java new file mode 100644 index 00000000000..e620a7b33cf --- /dev/null +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/RouteTlsProvisionerTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.openshift.provision; + +import static java.util.Collections.emptyMap; +import static org.eclipse.che.workspace.infrastructure.openshift.provision.RouteTlsProvisioner.TERMINATION_EDGE; +import static org.eclipse.che.workspace.infrastructure.openshift.provision.RouteTlsProvisioner.TERMINATION_POLICY_REDIRECT; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +import com.google.common.collect.ImmutableMap; +import io.fabric8.openshift.api.model.Route; +import io.fabric8.openshift.api.model.RouteBuilder; +import java.util.HashMap; +import java.util.Map; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; +import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; +import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; +import org.mockito.Mock; +import org.mockito.testng.MockitoTestNGListener; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +/** + * Tests {@link RouteTlsProvisioner}. + * + * @author Ilya Buziuk + * @author Sergii Leshchenko + */ +@Listeners(MockitoTestNGListener.class) +public class RouteTlsProvisionerTest { + + @Mock private OpenShiftEnvironment osEnv; + @Mock private RuntimeIdentity runtimeIdentity; + + @Test + public void doNothingWhenTlsDisabled() throws Exception { + // given + RouteTlsProvisioner tlsProvisioner = new RouteTlsProvisioner(false); + + // when + tlsProvisioner.provision(osEnv, runtimeIdentity); + + // then + verify(osEnv, never()).getRoutes(); + } + + @Test + public void provisionTlsForRoutes() throws Exception { + // given + RouteTlsProvisioner tlsProvisioner = new RouteTlsProvisioner(true); + ServerConfigImpl httpServer = new ServerConfigImpl("8080/tpc", "http", "/api", emptyMap()); + ServerConfigImpl wsServer = new ServerConfigImpl("8080/tpc", "ws", "/ws", emptyMap()); + + final Map routes = new HashMap<>(); + Route route = + createRoute("route", ImmutableMap.of("http-server", httpServer, "ws-server", wsServer)); + routes.put("route", route); + when(osEnv.getRoutes()).thenReturn(routes); + + // when + tlsProvisioner.provision(osEnv, runtimeIdentity); + + // then + assertEquals(route.getSpec().getTls().getTermination(), TERMINATION_EDGE); + assertEquals( + route.getSpec().getTls().getInsecureEdgeTerminationPolicy(), TERMINATION_POLICY_REDIRECT); + + Map servers = + Annotations.newDeserializer(route.getMetadata().getAnnotations()).servers(); + assertEquals(servers.get("http-server").getProtocol(), "https"); + assertEquals(servers.get("ws-server").getProtocol(), "wss"); + } + + private Route createRoute(String name, Map servers) { + return new RouteBuilder() + .withNewMetadata() + .withName(name) + .withAnnotations(Annotations.newSerializer().servers(servers).annotations()) + .endMetadata() + .withNewSpec() + .endSpec() + .build(); + } +} diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/route/TlsRouteProvisionerTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/route/TlsRouteProvisionerTest.java deleted file mode 100644 index 3d36dd87e93..00000000000 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/provision/route/TlsRouteProvisionerTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che.workspace.infrastructure.openshift.provision.route; - -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import io.fabric8.openshift.api.model.Route; -import java.util.HashMap; -import java.util.Map; -import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; -import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; -import org.mockito.Mock; -import org.mockito.testng.MockitoTestNGListener; -import org.testng.annotations.Listeners; -import org.testng.annotations.Test; - -/** - * Tests {@link TlsRouteProvisioner}. - * - * @author Ilya Buziuk - */ -@Listeners(MockitoTestNGListener.class) -public class TlsRouteProvisionerTest { - - @Mock private OpenShiftEnvironment osEnv; - @Mock private RuntimeIdentity runtimeIdentity; - - @Test - public void doNothingWhenTlsDisabled() throws Exception { - TlsRouteProvisioner tlsProvisioner = new TlsRouteProvisioner(false); - tlsProvisioner.provision(osEnv, runtimeIdentity); - verify(osEnv, never()).getRoutes(); - } - - @Test - public void provisionTlsForRoutes() throws Exception { - TlsRouteProvisioner tlsProvisioner = new TlsRouteProvisioner(true); - final Map routes = new HashMap<>(); - when(osEnv.getRoutes()).thenReturn(routes); - tlsProvisioner.provision(osEnv, runtimeIdentity); - verify(osEnv, times(1)).getRoutes(); - } -} diff --git a/infrastructures/pom.xml b/infrastructures/pom.xml index 5c1932273f5..43ad16aea84 100644 --- a/infrastructures/pom.xml +++ b/infrastructures/pom.xml @@ -26,5 +26,6 @@ docker openshift + kubernetes diff --git a/pom.xml b/pom.xml index c066dcf1593..4c5534009e3 100644 --- a/pom.xml +++ b/pom.xml @@ -756,6 +756,11 @@ ${che.docs.version} war + + org.eclipse.che.infrastructure + infrastructure-kubernetes + ${che.version} + org.eclipse.che.infrastructure infrastructure-openshift diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/ServersChecker.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/ServersChecker.java index 1ef90e83d81..2da1b5b7941 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/ServersChecker.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/ServersChecker.java @@ -11,13 +11,14 @@ package org.eclipse.che.api.workspace.server.hc; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.inject.assistedinject.Assisted; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Timer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -40,12 +41,8 @@ */ public class ServersChecker { // Is used to define servers which will be checked by this server checker class. - // It is also a workaround to set correct paths for servers readiness checks. - private static final Map LIVENESS_CHECKS_PATHS = - ImmutableMap.of( - "wsagent/http", "/api/", - "exec-agent/http", "/process", - "terminal", "/"); + private static final Set LIVENESS_SERVERS = + ImmutableSet.of("wsagent/http", "exec-agent/http", "terminal"); private final RuntimeIdentity runtimeIdentity; private final String machineName; private final Map servers; @@ -163,7 +160,7 @@ private List getServerCheckers() throws InfrastructureException { for (Map.Entry serverEntry : servers.entrySet()) { // TODO replace with correct behaviour // workaround needed because we don't have server readiness check in the model - if (LIVENESS_CHECKS_PATHS.containsKey(serverEntry.getKey())) { + if (LIVENESS_SERVERS.contains(serverEntry.getKey())) { checkers.add(getChecker(serverEntry.getKey(), serverEntry.getValue())); } } @@ -173,35 +170,25 @@ private List getServerCheckers() throws InfrastructureException { private ServerChecker getChecker(String serverRef, Server server) throws InfrastructureException { // TODO replace with correct behaviour // workaround needed because we don't have server readiness check in the model - String livenessCheckPath = LIVENESS_CHECKS_PATHS.get(serverRef); // Create server readiness endpoint URL URL url; try { - // TODO: ws -> http is workaround used for terminal websocket server, - // should be removed after server checks added to model - // url = - // UriBuilder.fromUri(server.getUrl().replaceFirst("^ws", "http")) - // .replacePath(livenessCheckPath) - // .queryParam("token", - // machineTokenProvider.getToken(runtimeIdentity.getWorkspaceId())) - // .build() - // .toURL(); - String relativeLivenessCheckPath = UriBuilder.fromUri(server.getUrl()).build().getPath(); - if (relativeLivenessCheckPath.lastIndexOf("/") > 0) { - relativeLivenessCheckPath = - relativeLivenessCheckPath.substring(0, relativeLivenessCheckPath.lastIndexOf("/")); - relativeLivenessCheckPath = relativeLivenessCheckPath + livenessCheckPath; - } else { - relativeLivenessCheckPath = livenessCheckPath; + String serverUrl = server.getUrl(); + + if ("terminal".equals(serverRef)) { + serverUrl = serverUrl.replaceFirst("^ws", "http").replaceFirst("/pty$", "/"); + } + + if ("wsagent/http".equals(serverRef)) { + // add trailing slash + serverUrl = serverUrl + '/'; } url = - UriBuilder.fromUri(server.getUrl().replaceFirst("^ws", "http")) - .replacePath(relativeLivenessCheckPath) + UriBuilder.fromUri(serverUrl) .queryParam("token", machineTokenProvider.getToken(runtimeIdentity.getWorkspaceId())) .build() .toURL(); - System.out.println("checking liveness for: " + url); } catch (MalformedURLException e) { throw new InternalInfrastructureException( "Server " + serverRef + " URL is invalid. Error: " + e.getMessage(), e); diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/server/ExecServerLivenessProbeConfigFactory.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/server/ExecServerLivenessProbeConfigFactory.java index 692ed1be23b..b12d369e94b 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/server/ExecServerLivenessProbeConfigFactory.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/server/ExecServerLivenessProbeConfigFactory.java @@ -37,7 +37,7 @@ public HttpProbeConfig get(String workspaceId, Server server) url.getPort() == -1 ? url.getDefaultPort() : url.getPort(), url.getHost(), url.getProtocol(), - "/liveness", + url.getPath().replaceFirst("/process$", "/liveness"), null, successThreshold, 3, diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/server/TerminalServerLivenessProbeConfigFactory.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/server/TerminalServerLivenessProbeConfigFactory.java index 7ee495187a8..aa8e454f072 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/server/TerminalServerLivenessProbeConfigFactory.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/server/TerminalServerLivenessProbeConfigFactory.java @@ -55,7 +55,9 @@ public HttpProbeConfig get(String workspaceId, Server server) port = uri.getPort(); } + String path = uri.getPath().replaceFirst("/pty$", "/liveness"); + return new HttpProbeConfig( - port, uri.getHost(), protocol, "/liveness", null, successThreshold, 3, 120, 10, 10); + port, uri.getHost(), protocol, path, null, successThreshold, 3, 120, 10, 10); } } diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/hc/ServersCheckerTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/hc/ServersCheckerTest.java index 298d185b9ae..8f506c944f1 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/hc/ServersCheckerTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/hc/ServersCheckerTest.java @@ -65,9 +65,9 @@ public void setUp() throws Exception { servers = new HashMap<>(); servers.putAll( ImmutableMap.of( - "wsagent/http", new ServerImpl().withUrl("http://localhost"), - "exec-agent/http", new ServerImpl().withUrl("http://localhost"), - "terminal", new ServerImpl().withUrl("http://localhost"))); + "wsagent/http", new ServerImpl().withUrl("http://localhost/api"), + "exec-agent/http", new ServerImpl().withUrl("http://localhost/exec-agent/process"), + "terminal", new ServerImpl().withUrl("http://localhost/terminal/pty"))); compFuture = new CompletableFuture<>(); diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/hc/probe/WorkspaceProbesFactoryTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/hc/probe/WorkspaceProbesFactoryTest.java index bde247e584e..f7455a518a9 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/hc/probe/WorkspaceProbesFactoryTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/hc/probe/WorkspaceProbesFactoryTest.java @@ -90,7 +90,7 @@ public void returnsProbesForAMachineForTerminal() throws Exception { WORKSPACE_ID, MACHINE_NAME, singletonMap( - SERVER_TERMINAL_REFERENCE, new ServerImpl().withUrl("wss://localhost:4040"))); + SERVER_TERMINAL_REFERENCE, new ServerImpl().withUrl("wss://localhost:4040/pty"))); verifyHttpProbeConfig( wsProbes, @@ -111,7 +111,11 @@ public void returnsProbesForAMachineForTerminal() throws Exception { public void returnsProbesForAMachineForExec() throws Exception { WorkspaceProbes wsProbes = probesFactory.getProbes( - WORKSPACE_ID, MACHINE_NAME, singletonMap(SERVER_EXEC_AGENT_HTTP_REFERENCE, SERVER)); + WORKSPACE_ID, + MACHINE_NAME, + singletonMap( + SERVER_EXEC_AGENT_HTTP_REFERENCE, + new ServerImpl().withUrl("https://localhost:4040/process"))); verifyHttpProbeConfig( wsProbes, From 9f350af06fa38825293ee2ddb476a1ab9afb619d Mon Sep 17 00:00:00 2001 From: Oleksandr Garagatyi Date: Thu, 1 Feb 2018 16:58:17 +0200 Subject: [PATCH 04/11] CHE-5908: add config to deploy Che on k8s Signed-off-by: Oleksandr Garagatyi --- .../init/modules/kubernetes/Deploy Che.md | 18 ++ .../kubernetes/files/che-kubernetes.yaml | 219 +++++++----------- 2 files changed, 99 insertions(+), 138 deletions(-) create mode 100644 dockerfiles/init/modules/kubernetes/Deploy Che.md diff --git a/dockerfiles/init/modules/kubernetes/Deploy Che.md b/dockerfiles/init/modules/kubernetes/Deploy Che.md new file mode 100644 index 00000000000..6b3b7a0fdad --- /dev/null +++ b/dockerfiles/init/modules/kubernetes/Deploy Che.md @@ -0,0 +1,18 @@ +# Deploy single user Che to k8s +Tested on minikube with vm provider Virtualbox. Note that Che with workspaces requires quite a lot +of RAM. Initial tests were done with 10GB, but it is definitely more than it is needed to start Che +and couple of workspaces. +IP of VM is supposed to be `192.168.99.100`. `nip.io` is also used for handling hosts resolution. +If you have another IP or DNS replace these values in k8s.yml file. + +###Prerequisites: +- Ingress controller is running. Note: you can start it on minikube with `minikube addons enable ingress`. +- Currently Che workspaces work with NginX ingress controller only. Note: it is default ingress controller on minikube. +- DNS discovery should be enabled. Note: enabled by default in minikube. +### Deployment process: +Note: despite the fact that it is not necessary to use a separate namespace for Che +we use it to simplify development operations such as cleaning of spoiled environment +and clean redeploy of Che. +- Create namespace `che`: `kubectl create namespace che` +- Deploy Che: `kubectl --namespace=che apply -f che-kubernetes.yaml` +- Check Che pod status until it become `Running`: `kubectl get --namespace=che pods` diff --git a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml index c8bd976df3f..b3b0b3b15dd 100644 --- a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml +++ b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml @@ -34,18 +34,6 @@ items: resources: requests: storage: 1Gi -- apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - labels: - app: che - name: claim-che-workspace - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - apiVersion: v1 kind: ConfigMap metadata: @@ -53,39 +41,45 @@ items: app: che name: che data: - CHE_MULTIUSER: "false" - CHE_INFRASTRUCTURE_ACTIVE: openshift - CHE_INFRA_OPENSHIFT_MASTER__URL: "" - CHE_INFRA_OPENSHIFT_USERNAME: "" - CHE_INFRA_OPENSHIFT_PASSWORD: "" - CHE_INFRA_OPENSHIFT_OAUTH__TOKEN: "" - CHE_INFRA_OPENSHIFT_TRUST__CERTS: "false" - CHE_INFRA_OPENSHIFT_TLS__ENABLED: "false" - CHE_INFRA_OPENSHIFT_PVC_STRATEGY: "common" - CHE_INFRA_OPENSHIFT_PROJECT: "" - CHE_INFRA_OPENSHIFT_MACHINE__START__TIMEOUT__MIN: "5" - CHE_INFRA_OPENSHIFT_BOOTSTRAPPER_BINARY__URL: http://192.168.99.100/agent-binaries/linux_amd64/bootstrapper/bootstrapper - CHE_API: http://192.168.99.100/api - CHE_HOST: 192.168.99.100 - CHE_WEBSOCKET_ENDPOINT: ws://192.168.99.100/api/websocket + CHE_HOST: "192.168.99.100.nip.io" CHE_PORT: "8080" + CHE_API: "http://192.168.99.100.nip.io/api" + CHE_WEBSOCKET_ENDPOINT: ws://192.168.99.100.nip.io/api/websocket + CHE_DEBUG_SERVER: "true" + CHE_INFRASTRUCTURE_ACTIVE: kubernetes + CHE_INFRA_KUBERNETES_BOOTSTRAPPER_BINARY__URL: http://192.168.99.100.nip.io/agent-binaries/linux_amd64/bootstrapper/bootstrapper + CHE_INFRA_KUBERNETES_MACHINE__START__TIMEOUT__MIN: "5" + CHE_INFRA_KUBERNETES_MASTER__URL: "" + CHE_INFRA_KUBERNETES_OAUTH__TOKEN: "" + CHE_INFRA_KUBERNETES_PASSWORD: "" + CHE_INFRA_KUBERNETES_PVC_STRATEGY: "common" + CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS: "true" + CHE_INFRA_KUBERNETES_TLS__ENABLED: "false" + CHE_INFRA_KUBERNETES_TRUST__CERTS: "false" + CHE_INFRA_KUBERNETES_USERNAME: "" CHE_LOCAL_CONF_DIR: /etc/conf CHE_LOGS_DIR: /data/logs - CHE_PREDEFINED_STACKS_RELOAD__ON__START: "true" - CHE_LOG_LEVEL: INFO - CHE_PORT: "8080" - CHE_DEBUG_SERVER: "false" - CHE_OAUTH_GITHUB_FORCEACTIVATION: "true" - CHE_WORKSPACE_AUTO_START: "false" - CHE_KEYCLOAK_OSO_ENDPOINT: "" - CHE_KEYCLOAK_GITHUB_ENDPOINT: "" - CHE_SERVER_JAVA_OPTS: -XX:+UseG1GC -XX:+UseStringDeduplication -XX:MinHeapFreeRatio=20 -XX:MaxHeapFreeRatio=40 -XX:MaxRAM=700m -Xms256m - CHE_WORKSPACE_JAVA_OPTIONS: -XX:+UseG1GC -XX:+UseStringDeduplication -XX:MinHeapFreeRatio=20 -XX:MaxHeapFreeRatio=40 -XX:MaxRAM=1300m -Xms256m - CHE_KEYCLOAK_AUTH__SERVER__URL: not-in-use - CHE_KEYCLOAK_REALM: "" - CHE_KEYCLOAK_CLIENT__ID: "" + CHE_LOG_LEVEL: "INFO" + CHE_MULTIUSER: "false" CHE_OAUTH_GITHUB_CLIENTID: "" CHE_OAUTH_GITHUB_CLIENTSECRET: "" + CHE_PREDEFINED_STACKS_RELOAD__ON__START: "false" + JAVA_OPTS: "-XX:MaxRAMFraction=2 -XX:+UseParallelGC -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -Dsun.zip.disableMemoryMapping=true -Xms20m " + CHE_WORKSPACE_AUTO_START: "false" +- apiVersion: extensions/v1beta1 + kind: Ingress + metadata: + name: che-ingress + annotations: + ingress.kubernetes.io/rewrite-target: / + spec: + rules: + - host: 192.168.99.100.nip.io + http: + paths: + - backend: + serviceName: che-host + servicePort: 8080 - apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -98,189 +92,138 @@ items: selector: matchLabels: app: che + strategy: + type: Recreate template: metadata: + annotations: + pod.alpha.kubernetes.io/init-containers: '[{"image":"busybox","imagePullPolicy":"IfNotPresent","name":"fmp-volume-permission","command":["chmod","777","/data"],"volumeMounts":[{"mountPath":"/data","name":"che-data-volume"}]}]' labels: app: che spec: containers: - env: - - name: CHE_OAUTH_GITHUB_CLIENTID - valueFrom: - configMapKeyRef: - key: CHE_OAUTH_GITHUB_CLIENTID - name: che - - name: CHE_OAUTH_GITHUB_CLIENTSECRET - valueFrom: - configMapKeyRef: - key: CHE_OAUTH_GITHUB_CLIENTSECRET - name: che - - name: CHE_MULTIUSER - valueFrom: - configMapKeyRef: - key: CHE_MULTIUSER - name: che - - name: CHE_KEYCLOAK_AUTH__SERVER__URL + - name: CHE_HOST valueFrom: configMapKeyRef: - key: CHE_KEYCLOAK_AUTH__SERVER__URL + key: CHE_HOST name: che - - name: CHE_KEYCLOAK_REALM + - name: CHE_PORT valueFrom: configMapKeyRef: - key: CHE_KEYCLOAK_REALM + key: CHE_PORT name: che - - name: CHE_KEYCLOAK_CLIENT__ID + - name: CHE_API valueFrom: configMapKeyRef: - key: CHE_KEYCLOAK_CLIENT__ID + key: CHE_API name: che - - name: CHE_INFRA_OPENSHIFT_MASTER__URL + - name: CHE_WEBSOCKET_ENDPOINT valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_MASTER__URL + key: CHE_WEBSOCKET_ENDPOINT name: che - - name: CHE_INFRA_OPENSHIFT_USERNAME + - name: CHE_DEBUG_SERVER valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_USERNAME + key: CHE_DEBUG_SERVER name: che - name: CHE_INFRASTRUCTURE_ACTIVE valueFrom: configMapKeyRef: key: CHE_INFRASTRUCTURE_ACTIVE name: che - - name: CHE_INFRA_OPENSHIFT_PASSWORD + - name: CHE_INFRA_KUBERNETES_BOOTSTRAPPER_BINARY__URL valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_PASSWORD + key: CHE_INFRA_KUBERNETES_BOOTSTRAPPER_BINARY__URL name: che - - name: CHE_INFRA_OPENSHIFT_OAUTH__TOKEN + - name: CHE_INFRA_KUBERNETES_MACHINE__START__TIMEOUT__MIN valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_OAUTH__TOKEN + key: CHE_INFRA_KUBERNETES_MACHINE__START__TIMEOUT__MIN name: che - - name: CHE_INFRA_OPENSHIFT_TRUST__CERTS + - name: CHE_INFRA_KUBERNETES_MASTER__URL valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_TRUST__CERTS + key: CHE_INFRA_KUBERNETES_MASTER__URL name: che - - name: CHE_INFRA_OPENSHIFT_PVC_STRATEGY + - name: CHE_INFRA_KUBERNETES_OAUTH__TOKEN valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_PVC_STRATEGY + key: CHE_INFRA_KUBERNETES_OAUTH__TOKEN name: che - - name: CHE_INFRA_OPENSHIFT_TLS__ENABLED + - name: CHE_INFRA_KUBERNETES_PVC_STRATEGY valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_TLS__ENABLED + key: CHE_INFRA_KUBERNETES_PVC_STRATEGY name: che - - name: CHE_INFRA_OPENSHIFT_PROJECT + - name: CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_PROJECT + key: CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS name: che - - name: CHE_INFRA_OPENSHIFT_MACHINE__START__TIMEOUT__MIN + - name: JAVA_OPTS valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_MACHINE__START__TIMEOUT__MIN + key: JAVA_OPTS name: che - - name: CHE_INFRA_OPENSHIFT_BOOTSTRAPPER_BINARY__URL + - name: CHE_INFRA_KUBERNETES_PASSWORD valueFrom: configMapKeyRef: - key: CHE_INFRA_OPENSHIFT_BOOTSTRAPPER_BINARY__URL + key: CHE_INFRA_KUBERNETES_PASSWORD name: che - - name: CHE_WEBSOCKET_ENDPOINT + - name: CHE_INFRA_KUBERNETES_TRUST__CERTS valueFrom: configMapKeyRef: - key: CHE_WEBSOCKET_ENDPOINT + key: CHE_INFRA_KUBERNETES_TRUST__CERTS name: che - - name: CHE_LOGS_DIR + - name: CHE_INFRA_KUBERNETES_USERNAME valueFrom: configMapKeyRef: - key: CHE_LOGS_DIR + key: CHE_INFRA_KUBERNETES_USERNAME name: che - name: CHE_LOCAL_CONF_DIR valueFrom: configMapKeyRef: key: CHE_LOCAL_CONF_DIR name: che - - name: CHE_PREDEFINED_STACKS_RELOAD__ON__START + - name: CHE_LOGS_DIR valueFrom: configMapKeyRef: - key: CHE_PREDEFINED_STACKS_RELOAD__ON__START + key: CHE_LOGS_DIR name: che - name: CHE_LOG_LEVEL valueFrom: configMapKeyRef: key: CHE_LOG_LEVEL name: che - - name: CHE_PORT - valueFrom: - configMapKeyRef: - key: CHE_PORT - name: che - - name: CHE_DEBUG_SERVER + - name: CHE_MULTIUSER valueFrom: configMapKeyRef: - key: CHE_DEBUG_SERVER + key: CHE_MULTIUSER name: che - - name: CHE_HOST + - name: CHE_OAUTH_GITHUB_CLIENTID valueFrom: configMapKeyRef: - key: CHE_HOST + key: CHE_OAUTH_GITHUB_CLIENTID name: che - - name: CHE_API + - name: CHE_OAUTH_GITHUB_CLIENTSECRET valueFrom: configMapKeyRef: - key: CHE_API + key: CHE_OAUTH_GITHUB_CLIENTSECRET name: che - - name: CHE_OAUTH_GITHUB_FORCEACTIVATION + - name: CHE_PREDEFINED_STACKS_RELOAD__ON__START valueFrom: configMapKeyRef: - key: CHE_OAUTH_GITHUB_FORCEACTIVATION + key: CHE_PREDEFINED_STACKS_RELOAD__ON__START name: che - - name: CHE_WORKSPACE_AUTO__START + - name: CHE_WORKSPACE_AUTO_START valueFrom: configMapKeyRef: key: CHE_WORKSPACE_AUTO_START name: che - - name: JAVA_OPTS - valueFrom: - configMapKeyRef: - key: CHE_SERVER_JAVA_OPTS - name: che - - name: CHE_WORKSPACE_JAVA_OPTIONS - valueFrom: - configMapKeyRef: - key: CHE_WORKSPACE_JAVA_OPTIONS - name: che - - name: CHE_KEYCLOAK_OSO_ENDPOINT - valueFrom: - configMapKeyRef: - key: CHE_KEYCLOAK_OSO_ENDPOINT - name: che - - name: CHE_KEYCLOAK_GITHUB_ENDPOINT - valueFrom: - configMapKeyRef: - key: CHE_KEYCLOAK_GITHUB_ENDPOINT - name: che - - name: CHE_KEYCLOAK_AUTH__SERVER__URL - valueFrom: - configMapKeyRef: - key: CHE_KEYCLOAK_AUTH__SERVER__URL - name: che - - name: CHE_KEYCLOAK_REALM - valueFrom: - configMapKeyRef: - key: CHE_KEYCLOAK_REALM - name: che - - name: CHE_KEYCLOAK_CLIENT__ID - valueFrom: - configMapKeyRef: - key: CHE_KEYCLOAK_CLIENT__ID - name: che - image: + image: eclipse/che-server:nightly imagePullPolicy: Always livenessProbe: httpGet: @@ -314,4 +257,4 @@ items: volumes: - name: che-data-volume persistentVolumeClaim: - claimName: che-data-volume \ No newline at end of file + claimName: che-data-volume From ac1c5160b89a45209dc2d3cb76bbb465040faafb Mon Sep 17 00:00:00 2001 From: Sergii Leshchenko Date: Fri, 2 Feb 2018 15:47:58 +0200 Subject: [PATCH 05/11] CHE-5908 Add an ability to configure security context for pods --- .../webapp/WEB-INF/classes/che/che.properties | 5 ++ .../kubernetes/files/che-kubernetes.yaml | 17 +++- .../KubernetesEnvironmentProvisioner.java | 7 +- .../bootstrapper/KubernetesBootstrapper.java | 1 + .../namespace/pvc/PVCSubPathHelper.java | 9 ++- .../provision/SecurityContextProvisioner.java | 49 ++++++++++++ .../KubernetesEnvironmentProvisionerTest.java | 9 ++- .../namespace/pvc/PVCSubPathHelperTest.java | 6 +- .../SecurityContextProvisionerTest.java | 79 +++++++++++++++++++ 9 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/SecurityContextProvisioner.java create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/SecurityContextProvisionerTest.java diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties index 9c3efcbbb97..7b2a9146291 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties @@ -395,6 +395,11 @@ che.infra.kubernetes.pvc.access_mode=ReadWriteOnce che.infra.kubernetes.installer_server_min_port=10000 che.infra.kubernetes.installer_server_max_port=20000 +# Defines security context for pods that will be created by Kubernetes Infra +# +# This is ignored by OpenShift infra +che.infra.kubernetes.pod.security_context.run_as_user=NULL +che.infra.kubernetes.pod.security_context.fs_group=NULL ### OpenShift Infra parameters # diff --git a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml index b3b0b3b15dd..e5035c53da6 100644 --- a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml +++ b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml @@ -52,11 +52,12 @@ items: CHE_INFRA_KUBERNETES_MASTER__URL: "" CHE_INFRA_KUBERNETES_OAUTH__TOKEN: "" CHE_INFRA_KUBERNETES_PASSWORD: "" + CHE_INFRA_KUBERNETES_USERNAME: "" + CHE_INFRA_KUBERNETES_TRUST__CERTS: "false" CHE_INFRA_KUBERNETES_PVC_STRATEGY: "common" CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS: "true" - CHE_INFRA_KUBERNETES_TLS__ENABLED: "false" - CHE_INFRA_KUBERNETES_TRUST__CERTS: "false" - CHE_INFRA_KUBERNETES_USERNAME: "" + CHE_INFRA_KUBERNETES_POD_SECURITY__CONTEXT_RUN__AS__USER: "0" + CHE_INFRA_KUBERNETES_POD_SECURITY__CONTEXT_FS__GROUP: "0" CHE_LOCAL_CONF_DIR: /etc/conf CHE_LOGS_DIR: /data/logs CHE_LOG_LEVEL: "INFO" @@ -163,6 +164,16 @@ items: configMapKeyRef: key: CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS name: che + - name: CHE_INFRA_KUBERNETES_POD_SECURITY__CONTEXT_RUN__AS__USER + valueFrom: + configMapKeyRef: + key: CHE_INFRA_KUBERNETES_POD_SECURITY__CONTEXT_RUN__AS__USER + name: che + - name: CHE_INFRA_KUBERNETES_POD_SECURITY__CONTEXT_FS__GROUP + valueFrom: + configMapKeyRef: + key: CHE_INFRA_KUBERNETES_POD_SECURITY__CONTEXT_FS__GROUP + name: che - name: JAVA_OPTS valueFrom: configMapKeyRef: diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java index b12d250d8ca..66d42c48d63 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java @@ -19,6 +19,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.InstallerServersPortProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SecurityContextProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.RamLimitProvisioner; @@ -44,6 +45,7 @@ public class KubernetesEnvironmentProvisioner { private final RamLimitProvisioner ramLimitProvisioner; private final InstallerServersPortProvisioner installerServersPortProvisioner; private final LogsVolumeMachineProvisioner logsVolumeMachineProvisioner; + private final SecurityContextProvisioner securityContextProvisioner; @Inject public KubernetesEnvironmentProvisioner( @@ -55,7 +57,8 @@ public KubernetesEnvironmentProvisioner( WorkspaceVolumesStrategy volumesStrategy, RamLimitProvisioner ramLimitProvisioner, InstallerServersPortProvisioner installerServersPortProvisioner, - LogsVolumeMachineProvisioner logsVolumeMachineProvisioner) { + LogsVolumeMachineProvisioner logsVolumeMachineProvisioner, + SecurityContextProvisioner securityContextProvisioner) { this.pvcEnabled = pvcEnabled; this.volumesStrategy = volumesStrategy; this.uniqueNamesProvisioner = uniqueNamesProvisioner; @@ -65,6 +68,7 @@ public KubernetesEnvironmentProvisioner( this.ramLimitProvisioner = ramLimitProvisioner; this.installerServersPortProvisioner = installerServersPortProvisioner; this.logsVolumeMachineProvisioner = logsVolumeMachineProvisioner; + this.securityContextProvisioner = securityContextProvisioner; } public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) @@ -86,5 +90,6 @@ public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) restartPolicyRewriter.provision(k8sEnv, identity); uniqueNamesProvisioner.provision(k8sEnv, identity); ramLimitProvisioner.provision(k8sEnv, identity); + securityContextProvisioner.provision(k8sEnv, identity); } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java index 54d4f567110..bc8fe23965d 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/bootstrapper/KubernetesBootstrapper.java @@ -118,6 +118,7 @@ protected void doBootstrapAsync(String installerWebsocketEndpoint, String output private void injectBootstrapper() throws InfrastructureException { String machineName = kubernetesMachine.getName(); + LOG.debug( "Bootstrapping {}:{}. Creating folder for bootstrapper", runtimeIdentity, machineName); kubernetesMachine.exec("mkdir", "-p", BOOTSTRAPPER_DIR, bootstrapperLogsFolder); diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java index 90997a6b39a..ec5ce450f2d 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java @@ -37,6 +37,7 @@ import org.eclipse.che.commons.lang.concurrent.ThreadLocalPropagateContext; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPods; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SecurityContextProvisioner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,16 +77,20 @@ public class PVCSubPathHelper { private final KubernetesNamespaceFactory factory; private final ExecutorService executor; + private final SecurityContextProvisioner securityContextProvisioner; + @Inject PVCSubPathHelper( @Named("che.infra.kubernetes.pvc.name") String pvcName, @Named("che.infra.kubernetes.pvc.jobs.memorylimit") String jobMemoryLimit, @Named("che.infra.kubernetes.pvc.jobs.image") String jobImage, - KubernetesNamespaceFactory factory) { + KubernetesNamespaceFactory factory, + SecurityContextProvisioner securityContextProvisioner) { this.pvcName = pvcName; this.jobMemoryLimit = jobMemoryLimit; this.jobImage = jobImage; this.factory = factory; + this.securityContextProvisioner = securityContextProvisioner; this.executor = Executors.newFixedThreadPool( COUNT_THREADS, @@ -130,6 +135,8 @@ void execute(String workspaceId, String[] commandBase, String... arguments) { final String podName = jobName + '-' + workspaceId; final String[] command = buildCommand(commandBase, arguments); final Pod pod = newPod(podName, command); + securityContextProvisioner.provision(pod); + KubernetesPods pods = null; try { pods = factory.create(workspaceId).pods(); diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/SecurityContextProvisioner.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/SecurityContextProvisioner.java new file mode 100644 index 00000000000..2c74c17b82c --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/SecurityContextProvisioner.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; + +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodSecurityContextBuilder; +import javax.inject.Inject; +import javax.inject.Named; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.commons.annotation.Nullable; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; + +/** @author Sergii Leshchenko */ +public class SecurityContextProvisioner implements ConfigurationProvisioner { + + private final Long runAsUser; + private final Long fsGroup; + + @Inject + public SecurityContextProvisioner( + @Nullable @Named("che.infra.kubernetes.pod.security_context.run_as_user") String runAsUser, + @Nullable @Named("che.infra.kubernetes.pod.security_context.fs_group") String fsGroup) { + this.runAsUser = runAsUser == null ? null : Long.parseLong(runAsUser); + this.fsGroup = fsGroup == null ? null : Long.parseLong(fsGroup); + } + + @Override + public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) + throws InfrastructureException { + if (runAsUser != null) { + k8sEnv.getPods().values().forEach(this::provision); + } + } + + public void provision(Pod pod) { + pod.getSpec() + .setSecurityContext( + new PodSecurityContextBuilder().withRunAsUser(runAsUser).withFsGroup(fsGroup).build()); + } +} diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java index 0fe7217f4d1..2e91492862f 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java @@ -18,6 +18,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.InstallerServersPortProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SecurityContextProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.EnvVarsConverter; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.limits.ram.RamLimitProvisioner; @@ -48,6 +49,7 @@ public class KubernetesEnvironmentProvisionerTest { @Mock private RestartPolicyRewriter restartPolicyRewriter; @Mock private RamLimitProvisioner ramLimitProvisioner; @Mock private LogsVolumeMachineProvisioner logsVolumeMachineProvisioner; + @Mock private SecurityContextProvisioner securityContextProvisioner; private KubernetesEnvironmentProvisioner osInfraProvisioner; @@ -65,7 +67,8 @@ public void setUp() { volumesStrategy, ramLimitProvisioner, installerServersPortProvisioner, - logsVolumeMachineProvisioner); + logsVolumeMachineProvisioner, + securityContextProvisioner); provisionOrder = inOrder( installerServersPortProvisioner, @@ -75,7 +78,8 @@ public void setUp() { serversProvisioner, envVarsProvisioner, restartPolicyRewriter, - ramLimitProvisioner); + ramLimitProvisioner, + securityContextProvisioner); } @Test @@ -92,6 +96,7 @@ public void performsOrderedProvisioning() throws Exception { provisionOrder.verify(restartPolicyRewriter).provision(eq(k8sEnv), eq(runtimeIdentity)); provisionOrder.verify(uniqueNamesProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); provisionOrder.verify(ramLimitProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); + provisionOrder.verify(securityContextProvisioner).provision(eq(k8sEnv), eq(runtimeIdentity)); provisionOrder.verifyNoMoreInteractions(); } } diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java index 491f2247780..a207811045e 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java @@ -34,6 +34,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPods; +import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SecurityContextProvisioner; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; @@ -57,6 +58,7 @@ public class PVCSubPathHelperTest { private static final String PROJECTS_PATH = "/projects"; private static final String M2_PATH = "/.m2"; + @Mock private SecurityContextProvisioner securityContextProvisioner; @Mock private KubernetesNamespaceFactory k8sNamespaceFactory; @Mock private KubernetesNamespace k8sNamespace; @Mock private KubernetesPods osPods; @@ -70,7 +72,8 @@ public class PVCSubPathHelperTest { @BeforeMethod public void setup() throws Exception { pvcSubPathHelper = - new PVCSubPathHelper(PVC_NAME, jobMemoryLimit, jobImage, k8sNamespaceFactory); + new PVCSubPathHelper( + PVC_NAME, jobMemoryLimit, jobImage, k8sNamespaceFactory, securityContextProvisioner); when(k8sNamespaceFactory.create(anyString())).thenReturn(k8sNamespace); when(k8sNamespace.pods()).thenReturn(osPods); when(pod.getStatus()).thenReturn(podStatus); @@ -109,6 +112,7 @@ public void testSuccessfullyCreatesWorkspaceDirs() throws Exception { verify(osPods).wait(anyString(), anyInt(), any()); verify(podStatus).getPhase(); verify(osPods).delete(anyString()); + verify(securityContextProvisioner).provision(any()); } @Test diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/SecurityContextProvisionerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/SecurityContextProvisionerTest.java new file mode 100644 index 00000000000..95a661bef48 --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/SecurityContextProvisionerTest.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.provision; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import com.google.common.collect.ImmutableMap; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodBuilder; +import io.fabric8.kubernetes.api.model.PodSecurityContext; +import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; +import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.mockito.Mock; +import org.mockito.testng.MockitoTestNGListener; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +/** + * Test {@link SecurityContextProvisioner}. + * + * @author Sergii Leshchenko + */ +@Listeners(MockitoTestNGListener.class) +public class SecurityContextProvisionerTest { + + @Mock private RuntimeIdentity runtimeIdentity; + + private KubernetesEnvironment kubernetesEnvironment; + private Pod pod; + + private SecurityContextProvisioner securityContextProvisioner; + + @BeforeMethod + public void setUp() throws Exception { + pod = new PodBuilder().withNewSpec().endSpec().build(); + + kubernetesEnvironment = + KubernetesEnvironment.builder().setPods(ImmutableMap.of("pod", pod)).build(); + } + + @Test + public void shouldNotProvisionSecurityContextIfItIsNotConfigured() throws Exception { + // given + securityContextProvisioner = new SecurityContextProvisioner(null, null); + + // when + securityContextProvisioner.provision(kubernetesEnvironment, runtimeIdentity); + + // then + assertNull(pod.getSpec().getSecurityContext()); + } + + @Test + public void shouldProvisionSecurityContextIfItIsConfigured() throws Exception { + // given + securityContextProvisioner = new SecurityContextProvisioner("1", "2"); + + // when + securityContextProvisioner.provision(kubernetesEnvironment, runtimeIdentity); + + // then + PodSecurityContext securityContext = pod.getSpec().getSecurityContext(); + assertNotNull(securityContext); + + assertEquals(securityContext.getRunAsUser(), new Long(1)); + assertEquals(securityContext.getFsGroup(), new Long(2)); + } +} From cb43481e5432db75f4245fa3cd9caeba546a8dac Mon Sep 17 00:00:00 2001 From: Oleksandr Garagatyi Date: Mon, 5 Feb 2018 15:45:03 +0200 Subject: [PATCH 06/11] CHE-5908 Allow to customize ingress controller specific annotations for ingresses Signed-off-by: Oleksandr Garagatyi --- .../webapp/WEB-INF/classes/che/che.properties | 1 + .../init/modules/kubernetes/Deploy Che.md | 20 ++++++++ .../kubernetes/files/che-kubernetes.yaml | 8 +++ .../kubernetes/KubernetesInfraModule.java | 7 +++ .../provision/server/ServersConverter.java | 13 ++++- .../server/IngressAnnotationsProvider.java | 49 +++++++++++++++++++ .../server/KubernetesServerExposer.java | 21 ++++++-- .../server/KubernetesServerExposerTest.java | 7 ++- .../server/OpenShiftServerExposer.java | 3 +- 9 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/IngressAnnotationsProvider.java diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties index 7b2a9146291..ec672be7584 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties @@ -394,6 +394,7 @@ che.infra.kubernetes.pvc.access_mode=ReadWriteOnce # then OpenShift infrastructure will reconfigure installer to use first available from this range che.infra.kubernetes.installer_server_min_port=10000 che.infra.kubernetes.installer_server_max_port=20000 +che.infra.kubernetes.ingress.annotations_json=NULL # Defines security context for pods that will be created by Kubernetes Infra # diff --git a/dockerfiles/init/modules/kubernetes/Deploy Che.md b/dockerfiles/init/modules/kubernetes/Deploy Che.md index 6b3b7a0fdad..0e0c297b1db 100644 --- a/dockerfiles/init/modules/kubernetes/Deploy Che.md +++ b/dockerfiles/init/modules/kubernetes/Deploy Che.md @@ -2,9 +2,29 @@ Tested on minikube with vm provider Virtualbox. Note that Che with workspaces requires quite a lot of RAM. Initial tests were done with 10GB, but it is definitely more than it is needed to start Che and couple of workspaces. + IP of VM is supposed to be `192.168.99.100`. `nip.io` is also used for handling hosts resolution. If you have another IP or DNS replace these values in k8s.yml file. +Services are exposed using ingress controller approach. +We added ingress annotations to customize ingress controller behavior - +not to break websocket connections. +In particular testing environment was setup with NginX ingress controller 0.9.0-beta.17. +So we added annotations specific to this implementation and version: +- ingress.kubernetes.io/rewrite-target: / +- ingress.kubernetes.io/proxy-read-timeout: "3600" +- ingress.kubernetes.io/proxy-connect-timeout: "3600" + +If you use another ingress controller implementation or version you need to customize +Che master ingress and value of environment variable `CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON` stored in ConfigMap. +Value of the map should be expressed as a stringified JSON. For example most recent NginX controller uses other annotations: +- nginx.ingress.kubernetes.io/rewrite-target +- nginx.ingress.kubernetes.io/proxy-read-timeout +- nginx.ingress.kubernetes.io/proxy-connect-timeout +- nginx.ingress.kubernetes.io/ssl-redirect + +And environment variable would be: `'{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}'` + ###Prerequisites: - Ingress controller is running. Note: you can start it on minikube with `minikube addons enable ingress`. - Currently Che workspaces work with NginX ingress controller only. Note: it is default ingress controller on minikube. diff --git a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml index e5035c53da6..c99b9b1c810 100644 --- a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml +++ b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml @@ -67,12 +67,15 @@ items: CHE_PREDEFINED_STACKS_RELOAD__ON__START: "false" JAVA_OPTS: "-XX:MaxRAMFraction=2 -XX:+UseParallelGC -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -Dsun.zip.disableMemoryMapping=true -Xms20m " CHE_WORKSPACE_AUTO_START: "false" + CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}' - apiVersion: extensions/v1beta1 kind: Ingress metadata: name: che-ingress annotations: ingress.kubernetes.io/rewrite-target: / + ingress.kubernetes.io/proxy-read-timeout: "3600" + ingress.kubernetes.io/proxy-connect-timeout: "3600" spec: rules: - host: 192.168.99.100.nip.io @@ -234,6 +237,11 @@ items: configMapKeyRef: key: CHE_WORKSPACE_AUTO_START name: che + - name: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON + valueFrom: + configMapKeyRef: + key: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON + name: che image: eclipse/che-server:nightly imagePullPolicy: Always livenessProbe: diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java index 7a422e1d7b6..596077ada51 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java @@ -14,9 +14,11 @@ import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy.UNIQUE_STRATEGY; import com.google.inject.AbstractModule; +import com.google.inject.TypeLiteral; import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.multibindings.MapBinder; import com.google.inject.multibindings.Multibinder; +import java.util.Map; import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure; import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironmentFactory; import org.eclipse.che.api.workspace.server.spi.provision.env.CheApiEnvVarProvider; @@ -34,6 +36,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiEnvVarProvider; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.LogsRootEnvVariableProvider; +import org.eclipse.che.workspace.infrastructure.kubernetes.server.IngressAnnotationsProvider; /** @author Sergii Leshchenko */ public class KubernetesInfraModule extends AbstractModule { @@ -65,5 +68,9 @@ protected void configure() { Multibinder envVarProviders = Multibinder.newSetBinder(binder(), EnvVarProvider.class); envVarProviders.addBinding().to(LogsRootEnvVariableProvider.class); + + bind(new TypeLiteral>() {}) + .annotatedWith(com.google.inject.name.Names.named("infra.kubernetes.ingress.annotations")) + .toProvider(IngressAnnotationsProvider.class); } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java index 59e43e95e0f..511dccf4686 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java @@ -14,6 +14,8 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodSpec; import java.util.Map; +import javax.inject.Inject; +import javax.inject.Named; import javax.inject.Singleton; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; @@ -36,6 +38,14 @@ @Singleton public class ServersConverter implements ConfigurationProvisioner { + private final Map ingressAnnotations; + + @Inject + public ServersConverter( + @Named("infra.kubernetes.ingress.annotations") Map ingressAnnotations) { + this.ingressAnnotations = ingressAnnotations; + } + @Override public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) throws InfrastructureException { @@ -47,7 +57,8 @@ public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity) InternalMachineConfig machineConfig = k8sEnv.getMachines().get(machineName); if (!machineConfig.getServers().isEmpty()) { KubernetesServerExposer kubernetesServerExposer = - new KubernetesServerExposer<>(machineName, podConfig, containerConfig, k8sEnv); + new KubernetesServerExposer<>( + ingressAnnotations, machineName, podConfig, containerConfig, k8sEnv); kubernetesServerExposer.expose(machineConfig.getServers()); } } diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/IngressAnnotationsProvider.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/IngressAnnotationsProvider.java new file mode 100644 index 00000000000..c2a0d556696 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/IngressAnnotationsProvider.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.server; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.Map; +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Provider; +import javax.inject.Singleton; +import org.eclipse.che.commons.annotation.Nullable; + +/** @author Alexander Garagatyi */ +@Singleton +public class IngressAnnotationsProvider implements Provider> { + + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + private static final Type type = new TypeToken>() {}.getType(); + + private Map annotations; + + @Inject + public IngressAnnotationsProvider( + @Nullable @Named("che.infra.kubernetes.ingress.annotations_json") String annotationsString) { + + if (annotationsString != null) { + annotations = GSON.fromJson(annotationsString, type); + } else { + annotations = Collections.emptyMap(); + } + } + + @Override + public Map get() { + return annotations; + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java index 1a7bf0387f0..bc0c7f43212 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java @@ -45,6 +45,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import javax.inject.Named; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; @@ -125,13 +126,19 @@ public class KubernetesServerExposer { public static final int SERVER_UNIQUE_PART_SIZE = 8; public static final String SERVER_PREFIX = "server"; + private final Map ingressAnnotations; protected final String machineName; protected final Container container; protected final Pod pod; protected final T kubernetesEnvironment; public KubernetesServerExposer( - String machineName, Pod pod, Container container, T kubernetesEnvironment) { + @Named("infra.kubernetes.ingress.annotations") Map ingressAnnotations, + String machineName, + Pod pod, + Container container, + T kubernetesEnvironment) { + this.ingressAnnotations = ingressAnnotations; this.machineName = machineName; this.pod = pod; this.container = container; @@ -196,6 +203,7 @@ protected void exposeExternalServers( .withName(serviceName + '-' + servicePort.getName()) .withMachineName(machineName) .withServiceName(serviceName) + .withAnnotations(ingressAnnotations) .withServicePort(servicePort.getName()) .withServers(ingressesServers) .build(); @@ -299,6 +307,7 @@ private static class IngressBuilder { private IntOrString servicePort; private Map serversConfigs; private String machineName; + private Map annotations; private IngressBuilder withName(String name) { this.name = name; @@ -310,6 +319,11 @@ private IngressBuilder withServiceName(String serviceName) { return this; } + private IngressBuilder withAnnotations(Map annotations) { + this.annotations = annotations; + return this; + } + private IngressBuilder withServicePort(String targetPortName) { this.servicePort = new IntOrString(targetPortName); return this; @@ -342,10 +356,7 @@ private Ingress build() { IngressRule ingressRule = new IngressRuleBuilder().withHttp(httpIngressRuleValue).build(); IngressSpec ingressSpec = new IngressSpecBuilder().withRules(ingressRule).build(); - Map ingressAnnotations = new HashMap<>(); - ingressAnnotations.put("ingress.kubernetes.io/rewrite-target", "/"); - ingressAnnotations.put("ingress.kubernetes.io/ssl-redirect", "false"); - ingressAnnotations.put("kubernetes.io/ingress.class", "nginx"); + Map ingressAnnotations = new HashMap<>(annotations); ingressAnnotations.putAll( Annotations.newSerializer() .servers(serversConfigs) diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java index fdd847f773e..3912962f5d3 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposerTest.java @@ -10,6 +10,7 @@ */ package org.eclipse.che.workspace.infrastructure.kubernetes.server; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerExposer.SERVER_PREFIX; @@ -38,7 +39,9 @@ import org.eclipse.che.api.workspace.server.model.impl.ServerConfigImpl; import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; +import org.mockito.testng.MockitoTestNGListener; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; import org.testng.annotations.Test; /** @@ -46,6 +49,7 @@ * * @author Sergii Leshchenko */ +@Listeners(MockitoTestNGListener.class) public class KubernetesServerExposerTest { private static final Map ATTRIBUTES_MAP = singletonMap("key", "value"); @@ -76,7 +80,8 @@ public void setUp() throws Exception { kubernetesEnvironment = KubernetesEnvironment.builder().setPods(ImmutableMap.of("pod", pod)).build(); this.serverExposer = - new KubernetesServerExposer<>(MACHINE_NAME, pod, container, kubernetesEnvironment); + new KubernetesServerExposer<>( + emptyMap(), MACHINE_NAME, pod, container, kubernetesEnvironment); } @Test diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java index 3bccf041197..da017de0e40 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/server/OpenShiftServerExposer.java @@ -19,6 +19,7 @@ import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.ServicePort; import io.fabric8.openshift.api.model.Route; +import java.util.Collections; import java.util.Map; import org.eclipse.che.api.core.model.workspace.config.ServerConfig; import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations; @@ -94,7 +95,7 @@ public class OpenShiftServerExposer extends KubernetesServerExposer Date: Tue, 6 Feb 2018 11:51:01 +0200 Subject: [PATCH 07/11] CHE-8611 Add warning suppressing when namespace/project doesn't exist on removing --- .../namespace/RemoveNamespaceOnWorkspaceRemove.java | 10 +++++++++- .../project/RemoveProjectOnWorkspaceRemove.java | 11 ++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java index c02c2c5dec7..c854e4445df 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/RemoveNamespaceOnWorkspaceRemove.java @@ -15,6 +15,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; import com.google.inject.Singleton; +import io.fabric8.kubernetes.client.KubernetesClientException; import javax.inject.Named; import org.eclipse.che.api.core.notification.EventService; import org.eclipse.che.api.core.notification.EventSubscriber; @@ -66,6 +67,13 @@ public void onEvent(WorkspaceRemovedEvent event) { @VisibleForTesting void doRemoveNamespace(String namespaceName) throws InfrastructureException { - clientFactory.create().namespaces().withName(namespaceName).delete(); + try { + clientFactory.create().namespaces().withName(namespaceName).delete(); + } catch (KubernetesClientException e) { + if (!(e.getCode() == 403)) { + throw new InfrastructureException(e.getMessage(), e); + } + // namespace doesn't exist + } } } diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java index 2fbda010fba..b271664cd2b 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/project/RemoveProjectOnWorkspaceRemove.java @@ -15,6 +15,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; import com.google.inject.Singleton; +import io.fabric8.kubernetes.client.KubernetesClientException; import javax.inject.Named; import org.eclipse.che.api.core.notification.EventService; import org.eclipse.che.api.core.notification.EventSubscriber; @@ -32,6 +33,7 @@ */ @Singleton public class RemoveProjectOnWorkspaceRemove implements EventSubscriber { + private static final Logger LOG = LoggerFactory.getLogger(RemoveProjectOnWorkspaceRemove.class); private final OpenShiftClientFactory clientFactory; @@ -66,6 +68,13 @@ public void onEvent(WorkspaceRemovedEvent event) { @VisibleForTesting void doRemoveProject(String projectName) throws InfrastructureException { - clientFactory.create().projects().withName(projectName).delete(); + try { + clientFactory.create().projects().withName(projectName).delete(); + } catch (KubernetesClientException e) { + if (!(e.getCode() == 403)) { + throw new InfrastructureException(e.getMessage(), e); + } + // project doesn't exist + } } } From 4ee894ec45314534fb99e3386443f66676632f0e Mon Sep 17 00:00:00 2001 From: Sergii Leshchenko Date: Tue, 6 Feb 2018 18:16:18 +0200 Subject: [PATCH 08/11] CHE-5908 Add CHE_INFRA_KUBERNETES_NAMESPACE into yml file --- .../init/modules/kubernetes/files/che-kubernetes.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml index c99b9b1c810..4e8390cdbb4 100644 --- a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml +++ b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml @@ -53,6 +53,7 @@ items: CHE_INFRA_KUBERNETES_OAUTH__TOKEN: "" CHE_INFRA_KUBERNETES_PASSWORD: "" CHE_INFRA_KUBERNETES_USERNAME: "" + CHE_INFRA_KUBERNETES_NAMESPACE: "" CHE_INFRA_KUBERNETES_TRUST__CERTS: "false" CHE_INFRA_KUBERNETES_PVC_STRATEGY: "common" CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS: "true" @@ -197,6 +198,11 @@ items: configMapKeyRef: key: CHE_INFRA_KUBERNETES_USERNAME name: che + - name: CHE_INFRA_KUBERNETES_NAMESPACE + valueFrom: + configMapKeyRef: + key: CHE_INFRA_KUBERNETES_NAMESPACE + name: che - name: CHE_LOCAL_CONF_DIR valueFrom: configMapKeyRef: From 0256ff10eea5c5f405df93845294f9ca2060a536 Mon Sep 17 00:00:00 2001 From: Sergii Leshchenko Date: Wed, 7 Feb 2018 09:05:33 +0200 Subject: [PATCH 09/11] CHE-5908 Add aliases for kubernetes configuration properties --- .../WEB-INF/classes/che_aliases.properties | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che_aliases.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che_aliases.properties index a2e3bc33837..80c88933229 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che_aliases.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che_aliases.properties @@ -10,3 +10,24 @@ # # File with a map new_property_name=old_property_name1,old_property_name2 + +che.infra.kubernetes.master_url=che.infra.openshift.master_url +che.infra.kubernetes.username=che.infra.openshift.username +che.infra.kubernetes.password=che.infra.openshift.password +che.infra.kubernetes.oauth_token=che.infra.openshift.oauth_token +che.infra.kubernetes.trust_certs=che.infra.openshift.trust_certs +che.infra.kubernetes.machine_start_timeout_min=che.infra.openshift.machine_start_timeout_min +che.infra.kubernetes.bootstrapper.binary_url=che.infra.openshift.bootstrapper.binary_url +che.infra.kubernetes.bootstrapper.timeout_min=che.infra.openshift.bootstrapper.timeout_min +che.infra.kubernetes.bootstrapper.installer_timeout_sec=che.infra.openshift.bootstrapper.installer_timeout_sec +che.infra.kubernetes.bootstrapper.server_check_period_sec=che.infra.openshift.bootstrapper.server_check_period_sec +che.infra.kubernetes.pvc.enabled=che.infra.openshift.pvc.enabled +che.infra.kubernetes.pvc.strategy=che.infra.openshift.pvc.strategy +che.infra.kubernetes.pvc.precreate_subpaths=che.infra.openshift.pvc.precreate_subpaths +che.infra.kubernetes.pvc.name=che.infra.openshift.pvc.name +che.infra.kubernetes.pvc.quantity=che.infra.openshift.pvc.quantity +che.infra.kubernetes.pvc.jobs.image=che.infra.openshift.pvc.jobs.image +che.infra.kubernetes.pvc.jobs.memorylimit=che.infra.openshift.pvc.jobs.memorylimit +che.infra.kubernetes.pvc.access_mode=che.infra.openshift.pvc.access_mode +che.infra.kubernetes.installer_server_min_port=che.infra.openshift.installer_server_min_port +che.infra.kubernetes.installer_server_max_port=che.infra.openshift.installer_server_max_port From 9c88f336e55f85a05d019dd399308b1be2123332 Mon Sep 17 00:00:00 2001 From: Sergii Leshchenko Date: Wed, 7 Feb 2018 09:07:13 +0200 Subject: [PATCH 10/11] CHE-5908 Add warning when ingress annotations are absent in configuration --- .../src/main/webapp/WEB-INF/classes/che/che.properties | 8 ++++++++ dockerfiles/init/manifests/che.env | 8 ++++++++ dockerfiles/init/modules/kubernetes/Deploy Che.md | 1 + .../kubernetes/server/KubernetesServerExposer.java | 9 +++++++++ 4 files changed, 26 insertions(+) diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties index ec672be7584..ba86b29afa4 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties @@ -394,6 +394,13 @@ che.infra.kubernetes.pvc.access_mode=ReadWriteOnce # then OpenShift infrastructure will reconfigure installer to use first available from this range che.infra.kubernetes.installer_server_min_port=10000 che.infra.kubernetes.installer_server_max_port=20000 + +# Defines annotations for ingresses which are used for servers exposing. Value depends on ingress controller. +# For example for nginx ingress controller 0.9.0-beta17 the following value is recommended: +# {"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false",\ +# "ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"} +# +# OpenShift infrastructure ignores this property because it uses Routes instead of ingresses. che.infra.kubernetes.ingress.annotations_json=NULL # Defines security context for pods that will be created by Kubernetes Infra @@ -402,6 +409,7 @@ che.infra.kubernetes.ingress.annotations_json=NULL che.infra.kubernetes.pod.security_context.run_as_user=NULL che.infra.kubernetes.pod.security_context.fs_group=NULL + ### OpenShift Infra parameters # # Since OpenShift infrastructure reuse Kubernetes infrastructure components diff --git a/dockerfiles/init/manifests/che.env b/dockerfiles/init/manifests/che.env index 6968edbdac7..8994f3aa5c9 100644 --- a/dockerfiles/init/manifests/che.env +++ b/dockerfiles/init/manifests/che.env @@ -497,6 +497,14 @@ CHE_SINGLE_PORT=false # https://docs.kubernetes.com/container-platform/latest/architecture/additional_concepts/storage.html#pv-access-modes #CHE_INFRA_KUBERNETES_PVC_ACCESS__MODE=ReadWriteOnce +# Defines annotations for ingresses which are used for servers exposing. Value depends on ingress controller. +# For example for nginx ingress controller 0.9.0-beta17 the following value is recommended: +# {"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false",\ +# "ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"} +# +# OpenShift infrastructure ignores this property because it uses Routes instead of ingresses. +#CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON=NULL + ######################################################################################## ##### ##### ##### Openshift Infrastructure ##### diff --git a/dockerfiles/init/modules/kubernetes/Deploy Che.md b/dockerfiles/init/modules/kubernetes/Deploy Che.md index 0e0c297b1db..b28991da873 100644 --- a/dockerfiles/init/modules/kubernetes/Deploy Che.md +++ b/dockerfiles/init/modules/kubernetes/Deploy Che.md @@ -12,6 +12,7 @@ not to break websocket connections. In particular testing environment was setup with NginX ingress controller 0.9.0-beta.17. So we added annotations specific to this implementation and version: - ingress.kubernetes.io/rewrite-target: / +- ingress.kubernetes.io/ssl-redirect": "false" - ingress.kubernetes.io/proxy-read-timeout: "3600" - ingress.kubernetes.io/proxy-connect-timeout: "3600" diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java index bc0c7f43212..3db5001ef4a 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java @@ -52,6 +52,8 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.Constants; import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Helps to modify {@link KubernetesEnvironment} to make servers that are configured by {@link @@ -123,6 +125,8 @@ */ public class KubernetesServerExposer { + private static final Logger LOG = LoggerFactory.getLogger(KubernetesServerExposer.class); + public static final int SERVER_UNIQUE_PART_SIZE = 8; public static final String SERVER_PREFIX = "server"; @@ -138,6 +142,11 @@ public KubernetesServerExposer( Pod pod, Container container, T kubernetesEnvironment) { + if (ingressAnnotations == null) { + LOG.warn( + "Ingresses annotations are absent. Make sure that workspace ingresses don't need " + + "to be configured according to ingress controller."); + } this.ingressAnnotations = ingressAnnotations; this.machineName = machineName; this.pod = pod; From 10ac41f7f884f1af200d7dce85ee0952d63f21ff Mon Sep 17 00:00:00 2001 From: Sergii Leshchenko Date: Wed, 7 Feb 2018 10:11:01 +0200 Subject: [PATCH 11/11] CHE-5908 Update ingress annotations for Nginx 0.9.0 --- .../init/modules/kubernetes/Deploy Che.md | 32 ++++++++----------- .../kubernetes/files/che-kubernetes.yaml | 8 ++--- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/dockerfiles/init/modules/kubernetes/Deploy Che.md b/dockerfiles/init/modules/kubernetes/Deploy Che.md index b28991da873..6bd72822f46 100644 --- a/dockerfiles/init/modules/kubernetes/Deploy Che.md +++ b/dockerfiles/init/modules/kubernetes/Deploy Che.md @@ -1,38 +1,34 @@ # Deploy single user Che to k8s -Tested on minikube with vm provider Virtualbox. Note that Che with workspaces requires quite a lot +Tested on minikube with vm providers Virtualbox and kvm2. Note that Che with workspaces requires quite a lot of RAM. Initial tests were done with 10GB, but it is definitely more than it is needed to start Che and couple of workspaces. IP of VM is supposed to be `192.168.99.100`. `nip.io` is also used for handling hosts resolution. If you have another IP or DNS replace these values in k8s.yml file. -Services are exposed using ingress controller approach. -We added ingress annotations to customize ingress controller behavior - +Services are exposed using ingress controller approach. +We added ingress annotations to customize ingress controller behavior - not to break websocket connections. -In particular testing environment was setup with NginX ingress controller 0.9.0-beta.17. +In particular testing environment was setup with NginX ingress controller 0.9.0. So we added annotations specific to this implementation and version: -- ingress.kubernetes.io/rewrite-target: / -- ingress.kubernetes.io/ssl-redirect": "false" -- ingress.kubernetes.io/proxy-read-timeout: "3600" -- ingress.kubernetes.io/proxy-connect-timeout: "3600" +- nginx.ingress.kubernetes.io/rewrite-target: / +- nginx.ingress.kubernetes.io/ssl-redirect": "false" +- nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" +- nginx.ingress.kubernetes.io/proxy-connect-timeout: "3600" -If you use another ingress controller implementation or version you need to customize -Che master ingress and value of environment variable `CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON` stored in ConfigMap. -Value of the map should be expressed as a stringified JSON. For example most recent NginX controller uses other annotations: -- nginx.ingress.kubernetes.io/rewrite-target -- nginx.ingress.kubernetes.io/proxy-read-timeout -- nginx.ingress.kubernetes.io/proxy-connect-timeout -- nginx.ingress.kubernetes.io/ssl-redirect +If you use another ingress controller implementation or version you need to customize +Che master ingress and value of environment variable `CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON` stored in ConfigMap. +Value of the map should be expressed as a stringified JSON. -And environment variable would be: `'{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}'` +And environment variable would be: `'{"nginx.ingress.kubernetes.io/rewrite-target": "/","nginx.ingress.kubernetes.io/ssl-redirect": "false","nginx.ingress.kubernetes.io/proxy-connect-timeout": "3600","nginx.ingress.kubernetes.io/proxy-read-timeout": "3600"}'` ###Prerequisites: - Ingress controller is running. Note: you can start it on minikube with `minikube addons enable ingress`. - Currently Che workspaces work with NginX ingress controller only. Note: it is default ingress controller on minikube. - DNS discovery should be enabled. Note: enabled by default in minikube. ### Deployment process: -Note: despite the fact that it is not necessary to use a separate namespace for Che -we use it to simplify development operations such as cleaning of spoiled environment +Note: despite the fact that it is not necessary to use a separate namespace for Che +we use it to simplify development operations such as cleaning of spoiled environment and clean redeploy of Che. - Create namespace `che`: `kubectl create namespace che` - Deploy Che: `kubectl --namespace=che apply -f che-kubernetes.yaml` diff --git a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml index 4e8390cdbb4..a813bc32e8b 100644 --- a/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml +++ b/dockerfiles/init/modules/kubernetes/files/che-kubernetes.yaml @@ -68,15 +68,15 @@ items: CHE_PREDEFINED_STACKS_RELOAD__ON__START: "false" JAVA_OPTS: "-XX:MaxRAMFraction=2 -XX:+UseParallelGC -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -Dsun.zip.disableMemoryMapping=true -Xms20m " CHE_WORKSPACE_AUTO_START: "false" - CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false","ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}' + CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"nginx.ingress.kubernetes.io/rewrite-target": "/","nginx.ingress.kubernetes.io/ssl-redirect": "false","nginx.ingress.kubernetes.io/proxy-connect-timeout": "3600","nginx.ingress.kubernetes.io/proxy-read-timeout": "3600"}' - apiVersion: extensions/v1beta1 kind: Ingress metadata: name: che-ingress annotations: - ingress.kubernetes.io/rewrite-target: / - ingress.kubernetes.io/proxy-read-timeout: "3600" - ingress.kubernetes.io/proxy-connect-timeout: "3600" + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "3600" spec: rules: - host: 192.168.99.100.nip.io