Skip to content

Commit

Permalink
fixes #17060 - gateway single host kube (#17557)
Browse files Browse the repository at this point in the history
Signed-off-by: Michal Vala <mvala@redhat.com>
  • Loading branch information
sparkoo committed Aug 31, 2020
1 parent 2aa0b4f commit 7c4d0a8
Show file tree
Hide file tree
Showing 30 changed files with 1,846 additions and 152 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,20 @@ public interface ServerConfig {
*/
String SERVER_NAME_ATTRIBUTE = "serverName";

/**
* This attribute is used to remember name of the service for single-host gateway configuration.
* It's used internally only, so the attribute is removed from {@link ServerConfig}'s attributes
* before going outside.
*/
String SERVICE_NAME_ATTRIBUTE = "serviceName";

/**
* This attribute is used to remember port of the service for single-host gateway configuration.
* It's used internally only, so the attribute is removed from {@link ServerConfig}'s attributes
* before going outside.
*/
String SERVICE_PORT_ATTRIBUTE = "servicePort";

/**
* Port used by server.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@ public class Annotations {

public static final String MACHINE_NAME_ANNOTATION = ANNOTATION_PREFIX + "machine.name";

/**
* Object annotated with this set to `true` should be created in Che installation namespace. It's
* used only internally so it may be removed before actually creating k8s object, so it's not
* exposed.
*/
public static final String CREATE_IN_CHE_INSTALLATION_NAMESPACE =
ANNOTATION_PREFIX + "installation.namespace";

/** Pattern that matches server annotations e.g. "org.eclipse.che.server.exec-agent.port". */
private static final Pattern SERVER_ANNOTATION_PATTERN =
Pattern.compile("org\\.eclipse\\.che\\.server\\.(?<ref>[\\w-/]+)\\..+");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes;

import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.KubernetesClient;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import okhttp3.EventListener;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.commons.annotation.Nullable;

/**
* This {@link KubernetesClientFactory} is used to access Che installation namespace. It always
* provides client with default {@link Config}.
*/
@Singleton
public class CheServerKubernetesClientFactory extends KubernetesClientFactory {

@Inject
public CheServerKubernetesClientFactory(
@Nullable @Named("che.infra.kubernetes.master_url") String masterUrl,
@Nullable @Named("che.infra.kubernetes.trust_certs") Boolean doTrustCerts,
@Named("che.infra.kubernetes.client.http.async_requests.max") int maxConcurrentRequests,
@Named("che.infra.kubernetes.client.http.async_requests.max_per_host")
int maxConcurrentRequestsPerHost,
@Named("che.infra.kubernetes.client.http.connection_pool.max_idle") int maxIdleConnections,
@Named("che.infra.kubernetes.client.http.connection_pool.keep_alive_min")
int connectionPoolKeepAlive,
EventListener eventListener) {
super(
masterUrl,
doTrustCerts,
maxConcurrentRequests,
maxConcurrentRequestsPerHost,
maxIdleConnections,
connectionPoolKeepAlive,
eventListener);
}

/** @param workspaceId ignored */
@Override
public KubernetesClient create(String workspaceId) throws InfrastructureException {
return create();
}

/**
* creates an instance of {@link KubernetesClient} that is meant to be used on Che installation
* namespace
*/
@Override
public KubernetesClient create() throws InfrastructureException {
return super.create();
}

@Override
protected Config buildConfig(Config config, String workspaceId) {
return config;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStoragePodInterceptor;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.CertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GatewayRouterProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GitConfigProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ImagePullSecretProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner;
Expand Down Expand Up @@ -84,6 +85,7 @@ class KubernetesEnvironmentProvisionerImpl
private final GitConfigProvisioner gitConfigProvisioner;
private final PreviewUrlExposer<KubernetesEnvironment> previewUrlExposer;
private final VcsSslCertificateProvisioner vcsSslCertificateProvisioner;
private final GatewayRouterProvisioner gatewayRouterProvisioner;

@Inject
public KubernetesEnvironmentProvisionerImpl(
Expand All @@ -108,7 +110,8 @@ public KubernetesEnvironmentProvisionerImpl(
SshKeysProvisioner sshKeysProvisioner,
GitConfigProvisioner gitConfigProvisioner,
PreviewUrlExposer<KubernetesEnvironment> previewUrlExposer,
VcsSslCertificateProvisioner vcsSslCertificateProvisioner) {
VcsSslCertificateProvisioner vcsSslCertificateProvisioner,
GatewayRouterProvisioner gatewayRouterProvisioner) {
this.pvcEnabled = pvcEnabled;
this.volumesStrategy = volumesStrategy;
this.uniqueNamesProvisioner = uniqueNamesProvisioner;
Expand All @@ -131,6 +134,7 @@ public KubernetesEnvironmentProvisionerImpl(
this.vcsSslCertificateProvisioner = vcsSslCertificateProvisioner;
this.gitConfigProvisioner = gitConfigProvisioner;
this.previewUrlExposer = previewUrlExposer;
this.gatewayRouterProvisioner = gatewayRouterProvisioner;
}

@Traced
Expand Down Expand Up @@ -173,6 +177,7 @@ public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
sshKeysProvisioner.provision(k8sEnv, identity);
vcsSslCertificateProvisioner.provision(k8sEnv, identity);
gitConfigProvisioner.provision(k8sEnv, identity);
gatewayRouterProvisioner.provision(k8sEnv, identity);
LOG.debug("Provisioning Kubernetes environment done for workspace '{}'", workspaceId);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import static java.lang.String.format;
import static java.util.Collections.emptyMap;
import static java.util.stream.Collectors.toMap;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.shouldCreateInCheNamespace;
import static org.eclipse.che.workspace.infrastructure.kubernetes.util.TracingSpanConstants.CHECK_SERVERS;
import static org.eclipse.che.workspace.infrastructure.kubernetes.util.TracingSpanConstants.WAIT_MACHINES_START;
import static org.eclipse.che.workspace.infrastructure.kubernetes.util.TracingSpanConstants.WAIT_RUNNING_ASYNC;
Expand Down Expand Up @@ -85,6 +86,7 @@
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.PodMerger;
import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesMachineImpl;
import org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesRuntimeState;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.CheNamespace;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEvent;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.log.LogWatchTimeouts;
Expand Down Expand Up @@ -131,6 +133,7 @@ public class KubernetesInternalRuntime<E extends KubernetesEnvironment>
private final PreviewUrlCommandProvisioner previewUrlCommandProvisioner;
private final SecretAsContainerResourceProvisioner secretAsContainerResourceProvisioner;
private final KubernetesServerResolverFactory serverResolverFactory;
protected final CheNamespace cheNamespace;
protected final Tracer tracer;

@Inject
Expand All @@ -155,6 +158,7 @@ public KubernetesInternalRuntime(
PreviewUrlCommandProvisioner previewUrlCommandProvisioner,
SecretAsContainerResourceProvisioner secretAsContainerResourceProvisioner,
KubernetesServerResolverFactory kubernetesServerResolverFactory,
CheNamespace cheNamespace,
Tracer tracer,
@Assisted KubernetesRuntimeContext<E> context,
@Assisted KubernetesNamespace namespace) {
Expand All @@ -167,6 +171,7 @@ public KubernetesInternalRuntime(
this.probeScheduler = probeScheduler;
this.probesFactory = probesFactory;
this.namespace = namespace;
this.cheNamespace = cheNamespace;
this.eventPublisher = eventPublisher;
this.executor = sharedPool.getExecutor();
this.runtimeStates = runtimeStates;
Expand All @@ -190,7 +195,7 @@ protected void internalStart(Map<String, String> startOptions) throws Infrastruc
startSynchronizer.setStartThread();
startSynchronizer.start();

namespace.cleanUp();
cleanUp(workspaceId);
provisionWorkspace(startOptions, context, workspaceId);

volumesStrategy.prepare(
Expand Down Expand Up @@ -255,7 +260,7 @@ protected void internalStart(Map<String, String> startOptions) throws Infrastruc
// stop watching before namespace cleaning up
namespace.deployments().stopWatch(true);
try {
namespace.cleanUp();
cleanUp(workspaceId);
} catch (InfrastructureException cleanUppingEx) {
LOG.warn(
"Failed to clean up namespace after workspace '{}' start failing. Cause: {}",
Expand All @@ -272,6 +277,11 @@ protected void internalStart(Map<String, String> startOptions) throws Infrastruc
}
}

private void cleanUp(String workspaceId) throws InfrastructureException {
namespace.cleanUp();
cheNamespace.cleanUp(workspaceId);
}

protected void provisionWorkspace(
Map<String, String> startOptions, KubernetesRuntimeContext<E> context, String workspaceId)
throws InfrastructureException {
Expand Down Expand Up @@ -584,7 +594,7 @@ protected void internalStop(Map<String, String> stopOptions) throws Infrastructu
// Che Server that is crashed so start is hung up in STOPPING phase.
// Need to clean up runtime resources
probeScheduler.cancel(identity.getWorkspaceId());
namespace.cleanUp();
cleanUp(identity.getWorkspaceId());
}
} catch (InterruptedException e) {
throw new InfrastructureException(
Expand All @@ -594,7 +604,7 @@ protected void internalStop(Map<String, String> stopOptions) throws Infrastructu
// runtime is RUNNING. Clean up used resources
// Cancels workspace servers probes if any
probeScheduler.cancel(identity.getWorkspaceId());
namespace.cleanUp();
cleanUp(identity.getWorkspaceId());
}
}

Expand All @@ -614,7 +624,7 @@ protected void startMachines() throws InfrastructureException {
String workspaceId = getContext().getIdentity().getWorkspaceId();

createSecrets(k8sEnv, workspaceId);
List<ConfigMap> createdConfigMaps = createConfigMaps(k8sEnv, workspaceId);
List<ConfigMap> createdConfigMaps = createConfigMaps(k8sEnv, getContext().getIdentity());
List<Service> createdServices = createServices(k8sEnv, workspaceId);

// needed for resolution later on, even though n routes are actually created by ingress
Expand Down Expand Up @@ -684,15 +694,25 @@ void createSecrets(KubernetesEnvironment env, String workspaceId) throws Infrast
}

@Traced
@SuppressWarnings("WeakerAccess") // package-private so that interception is possible
List<ConfigMap> createConfigMaps(KubernetesEnvironment env, String workspaceId)
protected List<ConfigMap> createConfigMaps(KubernetesEnvironment env, RuntimeIdentity identity)
throws InfrastructureException {
TracingTags.WORKSPACE_ID.set(workspaceId);
TracingTags.WORKSPACE_ID.set(identity.getWorkspaceId());

List<ConfigMap> createdConfigMaps = new ArrayList<>();

List<ConfigMap> cheNamespaceConfigMaps = new ArrayList<>();
for (ConfigMap configMap : env.getConfigMaps().values()) {
createdConfigMaps.add(namespace.configMaps().create(configMap));
if (shouldCreateInCheNamespace(configMap)) {
// we collect the che namespace configmaps into separate list
cheNamespaceConfigMaps.add(configMap);
} else {
createdConfigMaps.add(namespace.configMaps().create(configMap));
}
}

// create che namespace configmaps in one batch, because we're doing some extra checks inside
createdConfigMaps.addAll(cheNamespace.createConfigMaps(cheNamespaceConfigMaps, identity));

return createdConfigMaps;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.environment;

import com.google.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;

/**
* This class checks the KUBERNETES_NAMESPACE and POD_NAMESPACE environment variables to determine
* what namespace Che is installed in. Users should use this class to retrieve the installation
* namespace name.
*
* @author Tom George
*/
@Singleton
public class CheInstallationLocation {
@Inject(optional = true)
@Named("env.KUBERNETES_NAMESPACE")
String kubernetesNamespace;

@Inject(optional = true)
@Named("env.POD_NAMESPACE")
String podNamespace;

/**
* @return The name of the namespace where Che is installed or null if both {@code
* KUBERNETES_NAMESPACE} and {@code POD_NAMESPACE} environment variables are not set
* @throws InfrastructureException when both {@code KUBERNETES_NAMESPACE} and {@code
* POD_NAMESPACE} are null
*/
public String getInstallationLocationNamespace() throws InfrastructureException {
if (kubernetesNamespace == null && podNamespace == null) {
throw new InfrastructureException(
"Neither KUBERNETES_NAMESPACE nor POD_NAMESPACE is defined. Unable to determine Che installation location");
}
return kubernetesNamespace == null ? podNamespace : kubernetesNamespace;
}
}
Loading

0 comments on commit 7c4d0a8

Please sign in to comment.