Skip to content

Commit

Permalink
Extract unrecoverable events listener from KubernetesInternalRuntime
Browse files Browse the repository at this point in the history
  • Loading branch information
sleshchenko committed Sep 27, 2018
1 parent d9c210b commit 21798bf
Show file tree
Hide file tree
Showing 7 changed files with 390 additions and 176 deletions.
Expand Up @@ -15,9 +15,7 @@
import static java.util.Collections.emptyMap;
import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.POD_STATUS_PHASE_FAILED;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.inject.assistedinject.Assisted;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.Container;
Expand Down Expand Up @@ -80,6 +78,7 @@
import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerResolver;
import org.eclipse.che.workspace.infrastructure.kubernetes.util.KubernetesSharedPool;
import org.eclipse.che.workspace.infrastructure.kubernetes.util.RuntimeEventsPublisher;
import org.eclipse.che.workspace.infrastructure.kubernetes.util.UnrecoverablePodEventListenerFactory;
import org.eclipse.che.workspace.infrastructure.kubernetes.wsplugins.SidecarToolingProvisioner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -95,7 +94,7 @@ public class KubernetesInternalRuntime<E extends KubernetesEnvironment>

private final int workspaceStartTimeout;
private final int ingressStartTimeout;
private final Set<String> unrecoverableEvents;
private final UnrecoverablePodEventListenerFactory unrecoverableEventListenerFactory;
private final ServersCheckerFactory serverCheckerFactory;
private final KubernetesBootstrapperFactory bootstrapperFactory;
private final ProbeScheduler probeScheduler;
Expand All @@ -115,8 +114,8 @@ public class KubernetesInternalRuntime<E extends KubernetesEnvironment>
public KubernetesInternalRuntime(
@Named("che.infra.kubernetes.workspace_start_timeout_min") int workspaceStartTimeout,
@Named("che.infra.kubernetes.ingress_start_timeout_min") int ingressStartTimeout,
@Named("che.infra.kubernetes.workspace_unrecoverable_events") String[] unrecoverableEvents,
NoOpURLRewriter urlRewriter,
UnrecoverablePodEventListenerFactory unrecoverableEventListenerFactory,
KubernetesBootstrapperFactory bootstrapperFactory,
ServersCheckerFactory serverCheckerFactory,
WorkspaceVolumesStrategy volumesStrategy,
Expand All @@ -134,12 +133,12 @@ public KubernetesInternalRuntime(
@Assisted KubernetesNamespace namespace,
@Assisted List<Warning> warnings) {
super(context, urlRewriter, warnings);
this.unrecoverableEventListenerFactory = unrecoverableEventListenerFactory;
this.bootstrapperFactory = bootstrapperFactory;
this.serverCheckerFactory = serverCheckerFactory;
this.volumesStrategy = volumesStrategy;
this.workspaceStartTimeout = workspaceStartTimeout;
this.ingressStartTimeout = ingressStartTimeout;
this.unrecoverableEvents = ImmutableSet.copyOf(unrecoverableEvents);
this.probeScheduler = probeScheduler;
this.probesFactory = probesFactory;
this.namespace = namespace;
Expand Down Expand Up @@ -516,9 +515,13 @@ protected void startMachines() throws InfrastructureException {
// TODO https://github.com/eclipse/che/issues/7653
// namespace.pods().watch(new AbnormalStopHandler());
namespace.deployments().watchEvents(new MachineLogsPublisher());
if (!unrecoverableEvents.isEmpty()) {
if (unrecoverableEventListenerFactory.isConfigured()) {
Map<String, Pod> pods = getContext().getEnvironment().getPods();
namespace.deployments().watchEvents(new UnrecoverablePodEventHandler(pods));
namespace
.deployments()
.watchEvents(
unrecoverableEventListenerFactory.create(
pods.keySet(), this::handleUnrecoverableEvent));
}

final KubernetesServerResolver serverResolver =
Expand Down Expand Up @@ -677,6 +680,23 @@ public void scheduleServersCheckers() throws InfrastructureException {
}
}

protected void handleUnrecoverableEvent(PodEvent podEvent) {
String reason = podEvent.getReason();
String message = podEvent.getMessage();
LOG.error(
"Unrecoverable event occurred during workspace '{}' startup: {}, {}, {}",
getContext().getIdentity().getWorkspaceId(),
reason,
message,
podEvent.getPodName());

startSynchronizer.completeExceptionally(
new InfrastructureException(
format(
"Unrecoverable event occurred: '%s', '%s', '%s'",
reason, message, podEvent.getPodName())));
}

private class ServerReadinessHandler implements Consumer<String> {

private String machineName;
Expand Down Expand Up @@ -706,7 +726,6 @@ public void accept(String serverRef) {
}

private class ServerLivenessHandler implements Consumer<ProbeResult> {

@Override
public void accept(ProbeResult probeResult) {
String machineName = probeResult.getMachineName();
Expand Down Expand Up @@ -742,81 +761,6 @@ public void accept(ProbeResult probeResult) {
}
}

/** Listens Pod events and terminates workspace if unrecoverable event occurs. */
public class UnrecoverablePodEventHandler implements PodEventHandler {
private Map<String, Pod> workspacePods;

public UnrecoverablePodEventHandler(Map<String, Pod> workspacePods) {
this.workspacePods = workspacePods;
}

/*
* Event is considered to be unrecoverable if it belongs to one of the workspace pods
* and 'lastTimestamp' of the event is *after* the time of handler initialization
*/
@Override
public void handle(PodEvent event) {
if (isWorkspaceEvent(event) && isUnrecoverable(event)) {
String reason = event.getReason();
String message = event.getMessage();
String workspaceId = getContext().getIdentity().getWorkspaceId();
LOG.error(
"Unrecoverable event occurred during workspace '{}' startup: {}, {}, {}",
workspaceId,
reason,
message,
event.getPodName());

startSynchronizer.completeExceptionally(
new InfrastructureException(
format(
"Unrecoverable event occurred: '%s', '%s', '%s'",
reason, message, event.getPodName())));
}
}

/** Returns true if event belongs to one of the workspace pods, false otherwise */
private boolean isWorkspaceEvent(PodEvent event) {
String podName = event.getPodName();
if (Strings.isNullOrEmpty(podName)) {
return false;
}
// Note it is necessary to compare via startsWith rather than equals here, as pods managed by
// deployments have their name set as [deploymentName]-[hash]. `workspacePodName` is used to
// define the deployment name, so pods that are created aren't an exact match.
return workspacePods
.keySet()
.stream()
.anyMatch(workspacePodName -> podName.startsWith(workspacePodName));
}

/**
* Returns true if event reason or message matches one of the comma separated values defined in
* 'che.infra.kubernetes.workspace_unrecoverable_events',false otherwise
*
* @param event event to check
*/
private boolean isUnrecoverable(PodEvent event) {
boolean isUnrecoverable = false;
String reason = event.getReason();
String message = event.getMessage();
// Consider unrecoverable if event reason 'equals' one of the property values e.g. "Failed
// Mount"
if (unrecoverableEvents.contains(reason)) {
isUnrecoverable = true;
} else {
for (String e : unrecoverableEvents) {
// Consider unrecoverable if event message 'startsWith' one of the property values e.g.
// "Failed to pull image"
if (message != null && message.startsWith(e)) {
isUnrecoverable = true;
}
}
}
return isUnrecoverable;
}
}

/** Listens pod events and publish them as machine logs. */
public class MachineLogsPublisher implements PodEventHandler {

Expand Down
@@ -0,0 +1,85 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.util;

import com.google.common.base.Strings;
import java.util.Set;
import java.util.function.Consumer;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEvent;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEventHandler;

/**
* Listens Pod events and propagates unrecoverable events via the specified handler.
*
* @author Sergii Leshchenko
* @author Ilya Buziuk
*/
public class UnrecoverablePodEventListener implements PodEventHandler {

private final Set<String> pods;
private final Consumer<PodEvent> unrecoverableEventHandler;
private final Set<String> unrecoverableEvents;

public UnrecoverablePodEventListener(
Set<String> unrecoverableEvents,
Set<String> pods,
Consumer<PodEvent> unrecoverableEventHandler) {
this.unrecoverableEvents = unrecoverableEvents;
this.pods = pods;
this.unrecoverableEventHandler = unrecoverableEventHandler;
}

@Override
public void handle(PodEvent event) {
if (isWorkspaceEvent(event) && isUnrecoverable(event)) {
unrecoverableEventHandler.accept(event);
}
}

/** Returns true if event belongs to one of the workspace pods, false otherwise */
private boolean isWorkspaceEvent(PodEvent event) {
String podName = event.getPodName();
if (Strings.isNullOrEmpty(podName)) {
return false;
}
// Note it is necessary to compare via startsWith rather than equals here, as pods managed by
// deployments have their name set as [deploymentName]-[hash]. `workspacePodName` is used to
// define the deployment name, so pods that are created aren't an exact match.
return pods.stream().anyMatch(podName::startsWith);
}

/**
* Returns true if event reason or message matches one of the comma separated values defined in
* 'che.infra.kubernetes.workspace_unrecoverable_events',false otherwise
*
* @param event event to check
*/
private boolean isUnrecoverable(PodEvent event) {
boolean isUnrecoverable = false;
String reason = event.getReason();
String message = event.getMessage();
// Consider unrecoverable if event reason 'equals' one of the property values e.g. "Failed
// Mount"
if (unrecoverableEvents.contains(reason)) {
isUnrecoverable = true;
} else {
for (String e : unrecoverableEvents) {
// Consider unrecoverable if event message 'startsWith' one of the property values e.g.
// "Failed to pull image"
if (message != null && message.startsWith(e)) {
isUnrecoverable = true;
}
}
}
return isUnrecoverable;
}
}
@@ -0,0 +1,63 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.util;

import com.google.common.collect.ImmutableSet;
import java.util.Set;
import java.util.function.Consumer;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEvent;

/**
* Helps to create {@link UnrecoverablePodEventListener} instaces.
*
* @author Sergii Leshchenko
*/
@Singleton
public class UnrecoverablePodEventListenerFactory {

private final Set<String> unrecoverableEvents;

@Inject
public UnrecoverablePodEventListenerFactory(
@Named("che.infra.kubernetes.workspace_unrecoverable_events") String[] unrecoverableEvents) {
this.unrecoverableEvents = ImmutableSet.copyOf(unrecoverableEvents);
}

/**
* Creates unrecoverable events listener.
*
* @param pods pods which unrecoverable events should be propagated
* @param unrecoverableEventHandler handler which is invoked when unrecoverable event occurs
* @return created unrecoverable events listener
* @throws IllegalStateException is unrecoverable events are not configured.
* @see #isConfigured()
*/
public UnrecoverablePodEventListener create(
Set<String> pods, Consumer<PodEvent> unrecoverableEventHandler) {
if (!isConfigured()) {
throw new IllegalStateException("Unrecoverable events are not configured");
}

return new UnrecoverablePodEventListener(unrecoverableEvents, pods, unrecoverableEventHandler);
}

/**
* Returns true if unrecoverable events are configured and it's possible to create unrecoverable
* events listener, false otherwise
*/
public boolean isConfigured() {
return !unrecoverableEvents.isEmpty();
}
}

0 comments on commit 21798bf

Please sign in to comment.