From 21798bf95c223c4f56a4faecfbf68476c6948806 Mon Sep 17 00:00:00 2001 From: Sergii Leshchenko Date: Fri, 14 Sep 2018 16:51:59 +0300 Subject: [PATCH] Extract unrecoverable events listener from KubernetesInternalRuntime --- .../kubernetes/KubernetesInternalRuntime.java | 110 +++--------- .../util/UnrecoverablePodEventListener.java | 85 +++++++++ .../UnrecoverablePodEventListenerFactory.java | 63 +++++++ .../KubernetesInternalRuntimeTest.java | 108 +++-------- .../UnrecoverablePodEventListenerTest.java | 167 ++++++++++++++++++ .../openshift/OpenShiftInternalRuntime.java | 20 ++- .../OpenShiftInternalRuntimeTest.java | 13 +- 7 files changed, 390 insertions(+), 176 deletions(-) create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListener.java create mode 100644 infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListenerFactory.java create mode 100644 infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListenerTest.java diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java index 74e0bb830f6..7d4a1be48e3 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java @@ -15,9 +15,7 @@ import static java.util.Collections.emptyMap; import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.POD_STATUS_PHASE_FAILED; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.inject.assistedinject.Assisted; import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.Container; @@ -80,6 +78,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerResolver; import org.eclipse.che.workspace.infrastructure.kubernetes.util.KubernetesSharedPool; import org.eclipse.che.workspace.infrastructure.kubernetes.util.RuntimeEventsPublisher; +import org.eclipse.che.workspace.infrastructure.kubernetes.util.UnrecoverablePodEventListenerFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.wsplugins.SidecarToolingProvisioner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,7 +94,7 @@ public class KubernetesInternalRuntime private final int workspaceStartTimeout; private final int ingressStartTimeout; - private final Set unrecoverableEvents; + private final UnrecoverablePodEventListenerFactory unrecoverableEventListenerFactory; private final ServersCheckerFactory serverCheckerFactory; private final KubernetesBootstrapperFactory bootstrapperFactory; private final ProbeScheduler probeScheduler; @@ -115,8 +114,8 @@ public class KubernetesInternalRuntime public KubernetesInternalRuntime( @Named("che.infra.kubernetes.workspace_start_timeout_min") int workspaceStartTimeout, @Named("che.infra.kubernetes.ingress_start_timeout_min") int ingressStartTimeout, - @Named("che.infra.kubernetes.workspace_unrecoverable_events") String[] unrecoverableEvents, NoOpURLRewriter urlRewriter, + UnrecoverablePodEventListenerFactory unrecoverableEventListenerFactory, KubernetesBootstrapperFactory bootstrapperFactory, ServersCheckerFactory serverCheckerFactory, WorkspaceVolumesStrategy volumesStrategy, @@ -134,12 +133,12 @@ public KubernetesInternalRuntime( @Assisted KubernetesNamespace namespace, @Assisted List warnings) { super(context, urlRewriter, warnings); + this.unrecoverableEventListenerFactory = unrecoverableEventListenerFactory; this.bootstrapperFactory = bootstrapperFactory; this.serverCheckerFactory = serverCheckerFactory; this.volumesStrategy = volumesStrategy; this.workspaceStartTimeout = workspaceStartTimeout; this.ingressStartTimeout = ingressStartTimeout; - this.unrecoverableEvents = ImmutableSet.copyOf(unrecoverableEvents); this.probeScheduler = probeScheduler; this.probesFactory = probesFactory; this.namespace = namespace; @@ -516,9 +515,13 @@ protected void startMachines() throws InfrastructureException { // TODO https://github.com/eclipse/che/issues/7653 // namespace.pods().watch(new AbnormalStopHandler()); namespace.deployments().watchEvents(new MachineLogsPublisher()); - if (!unrecoverableEvents.isEmpty()) { + if (unrecoverableEventListenerFactory.isConfigured()) { Map pods = getContext().getEnvironment().getPods(); - namespace.deployments().watchEvents(new UnrecoverablePodEventHandler(pods)); + namespace + .deployments() + .watchEvents( + unrecoverableEventListenerFactory.create( + pods.keySet(), this::handleUnrecoverableEvent)); } final KubernetesServerResolver serverResolver = @@ -677,6 +680,23 @@ public void scheduleServersCheckers() throws InfrastructureException { } } + protected void handleUnrecoverableEvent(PodEvent podEvent) { + String reason = podEvent.getReason(); + String message = podEvent.getMessage(); + LOG.error( + "Unrecoverable event occurred during workspace '{}' startup: {}, {}, {}", + getContext().getIdentity().getWorkspaceId(), + reason, + message, + podEvent.getPodName()); + + startSynchronizer.completeExceptionally( + new InfrastructureException( + format( + "Unrecoverable event occurred: '%s', '%s', '%s'", + reason, message, podEvent.getPodName()))); + } + private class ServerReadinessHandler implements Consumer { private String machineName; @@ -706,7 +726,6 @@ public void accept(String serverRef) { } private class ServerLivenessHandler implements Consumer { - @Override public void accept(ProbeResult probeResult) { String machineName = probeResult.getMachineName(); @@ -742,81 +761,6 @@ public void accept(ProbeResult probeResult) { } } - /** Listens Pod events and terminates workspace if unrecoverable event occurs. */ - public class UnrecoverablePodEventHandler implements PodEventHandler { - private Map workspacePods; - - public UnrecoverablePodEventHandler(Map workspacePods) { - this.workspacePods = workspacePods; - } - - /* - * Event is considered to be unrecoverable if it belongs to one of the workspace pods - * and 'lastTimestamp' of the event is *after* the time of handler initialization - */ - @Override - public void handle(PodEvent event) { - if (isWorkspaceEvent(event) && isUnrecoverable(event)) { - String reason = event.getReason(); - String message = event.getMessage(); - String workspaceId = getContext().getIdentity().getWorkspaceId(); - LOG.error( - "Unrecoverable event occurred during workspace '{}' startup: {}, {}, {}", - workspaceId, - reason, - message, - event.getPodName()); - - startSynchronizer.completeExceptionally( - new InfrastructureException( - format( - "Unrecoverable event occurred: '%s', '%s', '%s'", - reason, message, event.getPodName()))); - } - } - - /** Returns true if event belongs to one of the workspace pods, false otherwise */ - private boolean isWorkspaceEvent(PodEvent event) { - String podName = event.getPodName(); - if (Strings.isNullOrEmpty(podName)) { - return false; - } - // Note it is necessary to compare via startsWith rather than equals here, as pods managed by - // deployments have their name set as [deploymentName]-[hash]. `workspacePodName` is used to - // define the deployment name, so pods that are created aren't an exact match. - return workspacePods - .keySet() - .stream() - .anyMatch(workspacePodName -> podName.startsWith(workspacePodName)); - } - - /** - * Returns true if event reason or message matches one of the comma separated values defined in - * 'che.infra.kubernetes.workspace_unrecoverable_events',false otherwise - * - * @param event event to check - */ - private boolean isUnrecoverable(PodEvent event) { - boolean isUnrecoverable = false; - String reason = event.getReason(); - String message = event.getMessage(); - // Consider unrecoverable if event reason 'equals' one of the property values e.g. "Failed - // Mount" - if (unrecoverableEvents.contains(reason)) { - isUnrecoverable = true; - } else { - for (String e : unrecoverableEvents) { - // Consider unrecoverable if event message 'startsWith' one of the property values e.g. - // "Failed to pull image" - if (message != null && message.startsWith(e)) { - isUnrecoverable = true; - } - } - } - return isUnrecoverable; - } - } - /** Listens pod events and publish them as machine logs. */ public class MachineLogsPublisher implements PodEventHandler { diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListener.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListener.java new file mode 100644 index 00000000000..d1e77597057 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListener.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.util; + +import com.google.common.base.Strings; +import java.util.Set; +import java.util.function.Consumer; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEvent; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEventHandler; + +/** + * Listens Pod events and propagates unrecoverable events via the specified handler. + * + * @author Sergii Leshchenko + * @author Ilya Buziuk + */ +public class UnrecoverablePodEventListener implements PodEventHandler { + + private final Set pods; + private final Consumer unrecoverableEventHandler; + private final Set unrecoverableEvents; + + public UnrecoverablePodEventListener( + Set unrecoverableEvents, + Set pods, + Consumer unrecoverableEventHandler) { + this.unrecoverableEvents = unrecoverableEvents; + this.pods = pods; + this.unrecoverableEventHandler = unrecoverableEventHandler; + } + + @Override + public void handle(PodEvent event) { + if (isWorkspaceEvent(event) && isUnrecoverable(event)) { + unrecoverableEventHandler.accept(event); + } + } + + /** Returns true if event belongs to one of the workspace pods, false otherwise */ + private boolean isWorkspaceEvent(PodEvent event) { + String podName = event.getPodName(); + if (Strings.isNullOrEmpty(podName)) { + return false; + } + // Note it is necessary to compare via startsWith rather than equals here, as pods managed by + // deployments have their name set as [deploymentName]-[hash]. `workspacePodName` is used to + // define the deployment name, so pods that are created aren't an exact match. + return pods.stream().anyMatch(podName::startsWith); + } + + /** + * Returns true if event reason or message matches one of the comma separated values defined in + * 'che.infra.kubernetes.workspace_unrecoverable_events',false otherwise + * + * @param event event to check + */ + private boolean isUnrecoverable(PodEvent event) { + boolean isUnrecoverable = false; + String reason = event.getReason(); + String message = event.getMessage(); + // Consider unrecoverable if event reason 'equals' one of the property values e.g. "Failed + // Mount" + if (unrecoverableEvents.contains(reason)) { + isUnrecoverable = true; + } else { + for (String e : unrecoverableEvents) { + // Consider unrecoverable if event message 'startsWith' one of the property values e.g. + // "Failed to pull image" + if (message != null && message.startsWith(e)) { + isUnrecoverable = true; + } + } + } + return isUnrecoverable; + } +} diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListenerFactory.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListenerFactory.java new file mode 100644 index 00000000000..f4df0d5bef4 --- /dev/null +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListenerFactory.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.util; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import java.util.function.Consumer; +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEvent; + +/** + * Helps to create {@link UnrecoverablePodEventListener} instaces. + * + * @author Sergii Leshchenko + */ +@Singleton +public class UnrecoverablePodEventListenerFactory { + + private final Set unrecoverableEvents; + + @Inject + public UnrecoverablePodEventListenerFactory( + @Named("che.infra.kubernetes.workspace_unrecoverable_events") String[] unrecoverableEvents) { + this.unrecoverableEvents = ImmutableSet.copyOf(unrecoverableEvents); + } + + /** + * Creates unrecoverable events listener. + * + * @param pods pods which unrecoverable events should be propagated + * @param unrecoverableEventHandler handler which is invoked when unrecoverable event occurs + * @return created unrecoverable events listener + * @throws IllegalStateException is unrecoverable events are not configured. + * @see #isConfigured() + */ + public UnrecoverablePodEventListener create( + Set pods, Consumer unrecoverableEventHandler) { + if (!isConfigured()) { + throw new IllegalStateException("Unrecoverable events are not configured"); + } + + return new UnrecoverablePodEventListener(unrecoverableEvents, pods, unrecoverableEventHandler); + } + + /** + * Returns true if unrecoverable events are configured and it's possible to create unrecoverable + * events listener, false otherwise + */ + public boolean isConfigured() { + return !unrecoverableEvents.isEmpty(); + } +} diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java index 79c65bb50c2..3b57d76708b 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java @@ -102,7 +102,6 @@ import org.eclipse.che.api.workspace.shared.dto.event.MachineLogEvent; import org.eclipse.che.api.workspace.shared.dto.event.MachineStatusEvent; import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInternalRuntime.MachineLogsPublisher; -import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInternalRuntime.UnrecoverablePodEventHandler; import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapper; import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesMachineCache; @@ -125,6 +124,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.util.KubernetesSharedPool; import org.eclipse.che.workspace.infrastructure.kubernetes.util.PodEvents; import org.eclipse.che.workspace.infrastructure.kubernetes.util.RuntimeEventsPublisher; +import org.eclipse.che.workspace.infrastructure.kubernetes.util.UnrecoverablePodEventListenerFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.wsplugins.SidecarToolingProvisioner; import org.mockito.ArgumentCaptor; import org.mockito.Captor; @@ -153,7 +153,7 @@ public class KubernetesInternalRuntimeTest { private static final String CONTAINER_NAME_1 = "test1"; private static final String CONTAINER_NAME_2 = "test2"; private static final String EVENT_CREATION_TIMESTAMP = "2018-05-15T16:17:54Z"; - private static final String EVENT_LAST_TIMESTAMP_IN_PAST = "2018-05-15T16:18:54Z"; + /* Pods created by a deployment are created with a random suffix, so Pod names won't match exactly. */ private static final String POD_NAME_RANDOM_SUFFIX = "-12345"; @@ -163,10 +163,6 @@ public class KubernetesInternalRuntimeTest { private static final String M1_NAME = WORKSPACE_POD_NAME + '/' + CONTAINER_NAME_1; private static final String M2_NAME = WORKSPACE_POD_NAME + '/' + CONTAINER_NAME_2; - private static final String[] EMPTY_UNRECOVERABLE_EVENTS = new String[0]; - private static final String[] UNRECOVERABLE_EVENTS = - new String[] {"Failed Mount", "Failed Scheduling", "Failed to pull image"}; - private static final RuntimeIdentity IDENTITY = new RuntimeIdentityImpl(WORKSPACE_ID, "env1", "id1"); @@ -176,6 +172,7 @@ public class KubernetesInternalRuntimeTest { @Mock private KubernetesRuntimeContext context; @Mock private ServersCheckerFactory serverCheckerFactory; @Mock private ServersChecker serversChecker; + @Mock private UnrecoverablePodEventListenerFactory unrecoverablePodEventListenerFactory; @Mock private KubernetesBootstrapperFactory bootstrapperFactory; @Mock private KubernetesEnvironment k8sEnv; @Mock private KubernetesNamespace namespace; @@ -195,7 +192,7 @@ public class KubernetesInternalRuntimeTest { @Mock private KubernetesEnvironmentProvisioner kubernetesEnvironmentProvisioner; - @Mock private SidecarToolingProvisioner toolingProvisioner; + @Mock private SidecarToolingProvisioner toolingProvisioner; private KubernetesRuntimeStateCache runtimeStatesCache; private KubernetesMachineCache machinesCache; @@ -228,8 +225,8 @@ public void setup() throws Exception { new KubernetesInternalRuntime<>( 13, 5, - UNRECOVERABLE_EVENTS, new URLRewriter.NoOpURLRewriter(), + unrecoverablePodEventListenerFactory, bootstrapperFactory, serverCheckerFactory, volumesStrategy, @@ -251,8 +248,8 @@ public void setup() throws Exception { new KubernetesInternalRuntime<>( 13, 5, - EMPTY_UNRECOVERABLE_EVENTS, new URLRewriter.NoOpURLRewriter(), + unrecoverablePodEventListenerFactory, bootstrapperFactory, serverCheckerFactory, volumesStrategy, @@ -320,6 +317,27 @@ public void startsKubernetesEnvironment() throws Exception { verify(services).create(any()); verify(secrets).create(any()); verify(configMaps).create(any()); + verify(namespace.deployments(), times(1)).watchEvents(any()); + verify(bootstrapper, times(2)).bootstrapAsync(); + verify(eventService, times(4)).publish(any()); + verifyOrderedEventsChains( + new MachineStatusEvent[] {newEvent(M1_NAME, STARTING), newEvent(M1_NAME, RUNNING)}, + new MachineStatusEvent[] {newEvent(M2_NAME, STARTING), newEvent(M2_NAME, RUNNING)}); + verify(serverCheckerFactory).create(IDENTITY, M1_NAME, emptyMap()); + verify(serverCheckerFactory).create(IDENTITY, M2_NAME, emptyMap()); + verify(serversChecker, times(2)).startAsync(any()); + verify(namespace.deployments(), times(1)).stopWatch(); + } + + @Test + public void startsKubernetesEnvironmentWithUnrecoverableHandler() throws Exception { + when(unrecoverablePodEventListenerFactory.isConfigured()).thenReturn(true); + + internalRuntimeWithoutUnrecoverableHandler.internalStart(emptyMap()); + + verify(deployments).deploy(any()); + verify(ingresses).create(any()); + verify(services).create(any()); verify(namespace.deployments(), times(2)).watchEvents(any()); verify(bootstrapper, times(2)).bootstrapAsync(); verify(eventService, times(4)).publish(any()); @@ -334,6 +352,8 @@ public void startsKubernetesEnvironment() throws Exception { @Test public void startsKubernetesEnvironmentWithoutUnrecoverableHandler() throws Exception { + when(unrecoverablePodEventListenerFactory.isConfigured()).thenReturn(false); + internalRuntimeWithoutUnrecoverableHandler.internalStart(emptyMap()); verify(deployments).deploy(any()); @@ -471,76 +491,6 @@ public void testRepublishContainerOutputAsMachineLogEvents() throws Exception { assertTrue(captor.getAllValues().containsAll(machineLogs)); } - @Test - public void testHandleUnrecoverableEventByReason() throws Exception { - final String unrecoverableEventReason = "Failed Mount"; - final UnrecoverablePodEventHandler unrecoverableEventHandler = - internalRuntime.new UnrecoverablePodEventHandler(k8sEnv.getPods()); - final PodEvent unrecoverableEvent = - mockContainerEvent( - WORKSPACE_POD_NAME, - unrecoverableEventReason, - "Failed to mount volume 'claim-che-workspace'", - EVENT_CREATION_TIMESTAMP, - getCurrentTimestampWithOneHourShiftAhead()); - unrecoverableEventHandler.handle(unrecoverableEvent); - - verify(startSynchronizer).completeExceptionally(any(InfrastructureException.class)); - } - - @Test - public void testHandleUnrecoverableEventByMessage() throws Exception { - final String unrecoverableEventMessage = - "Failed to pull image eclipse/che-server:nightly-centos"; - final UnrecoverablePodEventHandler unrecoverableEventHandler = - internalRuntime.new UnrecoverablePodEventHandler(k8sEnv.getPods()); - final PodEvent unrecoverableEvent = - mockContainerEvent( - WORKSPACE_POD_NAME, - "Pulling", - unrecoverableEventMessage, - EVENT_CREATION_TIMESTAMP, - getCurrentTimestampWithOneHourShiftAhead()); - unrecoverableEventHandler.handle(unrecoverableEvent); - - verify(startSynchronizer).completeExceptionally(any(InfrastructureException.class)); - } - - @Test - public void testDoNotHandleUnrecoverableEventFromNonWorkspacePod() throws Exception { - final String unrecoverableEventMessage = - "Failed to pull image eclipse/che-server:nightly-centos"; - final UnrecoverablePodEventHandler unrecoverableEventHandler = - internalRuntime.new UnrecoverablePodEventHandler(k8sEnv.getPods()); - final PodEvent unrecoverableEvent = - mockContainerEvent( - "NonWorkspacePod", - "Pulling", - unrecoverableEventMessage, - EVENT_CREATION_TIMESTAMP, - getCurrentTimestampWithOneHourShiftAhead()); - unrecoverableEventHandler.handle(unrecoverableEvent); - // 'internalStop' is NOT expected to be called since event does not belong to the workspace - // pod.Cleanup will not be triggered - verify(namespace, never()).cleanUp(); - } - - @Test - public void testHandleRegularEvent() throws Exception { - final UnrecoverablePodEventHandler unrecoverableEventHandler = - internalRuntime.new UnrecoverablePodEventHandler(k8sEnv.getPods()); - final PodEvent regularEvent = - mockContainerEvent( - WORKSPACE_POD_NAME, - "Pulling", - "pulling image", - EVENT_CREATION_TIMESTAMP, - getCurrentTimestampWithOneHourShiftAhead()); - unrecoverableEventHandler.handle(regularEvent); - // 'internalStop' is NOT expected to be called and namespace cleanup will not be triggered - verify(namespace, never()).cleanUp(); - } - @Test public void testDoNotPublishForeignMachineOutput() throws ParseException { final MachineLogsPublisher logsPublisher = internalRuntime.new MachineLogsPublisher(); diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListenerTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListenerTest.java new file mode 100644 index 00000000000..ffe9b79f9db --- /dev/null +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/UnrecoverablePodEventListenerTest.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.workspace.infrastructure.kubernetes.util; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableSet; +import java.util.Date; +import java.util.Set; +import java.util.function.Consumer; +import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEvent; +import org.mockito.Mock; +import org.mockito.testng.MockitoTestNGListener; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +/** + * Tests {@link UnrecoverablePodEventListener}. + * + * @author Sergii Leshchenko + * @author Ilya Buziuk + */ +@Listeners(MockitoTestNGListener.class) +public class UnrecoverablePodEventListenerTest { + + private static final String WORKSPACE_POD_NAME = "app"; + private static final String CONTAINER_NAME_1 = "test1"; + private static final String EVENT_CREATION_TIMESTAMP = "2018-05-15T16:17:54Z"; + + /* Pods created by a deployment are created with a random suffix, so Pod names won't match + exactly. */ + private static final String POD_NAME_RANDOM_SUFFIX = "-12345"; + + private static final Set UNRECOVERABLE_EVENTS = + ImmutableSet.of("Failed Mount", "Failed Scheduling", "Failed to pull image"); + + @Mock private Consumer unrecoverableEventConsumer; + + private UnrecoverablePodEventListener unrecoverableEventListener; + + @BeforeMethod + public void setUp() { + unrecoverableEventListener = + new UnrecoverablePodEventListener( + UNRECOVERABLE_EVENTS, ImmutableSet.of(WORKSPACE_POD_NAME), unrecoverableEventConsumer); + } + + @Test + public void testHandleUnrecoverableEventByReason() throws Exception { + // given + String unrecoverableEventReason = "Failed Mount"; + PodEvent unrecoverableEvent = + mockContainerEvent( + WORKSPACE_POD_NAME, + unrecoverableEventReason, + "Failed to mount volume 'claim-che-workspace'", + EVENT_CREATION_TIMESTAMP, + getCurrentTimestampWithOneHourShiftAhead()); + + // when + unrecoverableEventListener.handle(unrecoverableEvent); + + // then + verify(unrecoverableEventConsumer).accept(unrecoverableEvent); + } + + @Test + public void testHandleUnrecoverableEventByMessage() throws Exception { + // given + String unrecoverableEventMessage = "Failed to pull image eclipse/che-server:nightly-centos"; + PodEvent unrecoverableEvent = + mockContainerEvent( + WORKSPACE_POD_NAME, + "Pulling", + unrecoverableEventMessage, + EVENT_CREATION_TIMESTAMP, + getCurrentTimestampWithOneHourShiftAhead()); + + // when + unrecoverableEventListener.handle(unrecoverableEvent); + + // then + verify(unrecoverableEventConsumer).accept(unrecoverableEvent); + } + + @Test + public void testDoNotHandleUnrecoverableEventFromNonWorkspacePod() throws Exception { + // given + final String unrecoverableEventMessage = + "Failed to pull image eclipse/che-server:nightly-centos"; + final PodEvent unrecoverableEvent = + mockContainerEvent( + "NonWorkspacePod", + "Pulling", + unrecoverableEventMessage, + EVENT_CREATION_TIMESTAMP, + getCurrentTimestampWithOneHourShiftAhead()); + + // when + unrecoverableEventListener.handle(unrecoverableEvent); + + // then + verify(unrecoverableEventConsumer, never()).accept(any()); + } + + @Test + public void testHandleRegularEvent() throws Exception { + // given + final PodEvent regularEvent = + mockContainerEvent( + WORKSPACE_POD_NAME, + "Pulling", + "pulling image", + EVENT_CREATION_TIMESTAMP, + getCurrentTimestampWithOneHourShiftAhead()); + + // when + unrecoverableEventListener.handle(regularEvent); + + // then + verify(unrecoverableEventConsumer, never()).accept(any()); + } + + /** + * Mock a container event, as though it was triggered by the OpenShift API. As workspace Pods are + * created indirectly through deployments, they are given generated names with the provided name + * as a root.
+ * Use this method in a test to ensure that tested code manages this fact correctly. For example, + * code such as unrecoverable events handling cannot directly look at an event's pod name and + * compare it to the internal representation, and so must confirm the event is relevant in some + * other way. + */ + private static PodEvent mockContainerEvent( + String podName, + String reason, + String message, + String creationTimestamp, + String lastTimestamp) { + final PodEvent event = mock(PodEvent.class); + when(event.getPodName()).thenReturn(podName + POD_NAME_RANDOM_SUFFIX); + when(event.getContainerName()).thenReturn(CONTAINER_NAME_1); + when(event.getReason()).thenReturn(reason); + when(event.getMessage()).thenReturn(message); + when(event.getCreationTimeStamp()).thenReturn(creationTimestamp); + when(event.getLastTimestamp()).thenReturn(lastTimestamp); + return event; + } + + private String getCurrentTimestampWithOneHourShiftAhead() { + Date currentTimestampWithOneHourShiftAhead = new Date(new Date().getTime() + 3600 * 1000); + return PodEvents.convertDateToEventTimestamp(currentTimestampWithOneHourShiftAhead); + } +} diff --git a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java index 8d9002287c1..1f347c70581 100644 --- a/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java +++ b/infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntime.java @@ -11,7 +11,6 @@ */ package org.eclipse.che.workspace.infrastructure.openshift; -import com.google.common.collect.ImmutableSet; import com.google.inject.assistedinject.Assisted; import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.Pod; @@ -39,6 +38,8 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; import org.eclipse.che.workspace.infrastructure.kubernetes.util.KubernetesSharedPool; import org.eclipse.che.workspace.infrastructure.kubernetes.util.RuntimeEventsPublisher; +import org.eclipse.che.workspace.infrastructure.kubernetes.util.UnrecoverablePodEventListener; +import org.eclipse.che.workspace.infrastructure.kubernetes.util.UnrecoverablePodEventListenerFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.wsplugins.SidecarToolingProvisioner; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; @@ -51,14 +52,14 @@ public class OpenShiftInternalRuntime extends KubernetesInternalRuntime { private final OpenShiftProject project; - private final Set unrecoverableEvents; + private final UnrecoverablePodEventListenerFactory unrecoverablePodEventListenerFactory; @Inject public OpenShiftInternalRuntime( @Named("che.infra.kubernetes.workspace_start_timeout_min") int workspaceStartTimeout, @Named("che.infra.kubernetes.ingress_start_timeout_min") int ingressStartTimeout, - @Named("che.infra.kubernetes.workspace_unrecoverable_events") String[] unrecoverableEvents, NoOpURLRewriter urlRewriter, + UnrecoverablePodEventListenerFactory unrecoverablePodEventListenerFactory, KubernetesBootstrapperFactory bootstrapperFactory, ServersCheckerFactory serverCheckerFactory, WorkspaceVolumesStrategy volumesStrategy, @@ -71,15 +72,15 @@ public OpenShiftInternalRuntime( StartSynchronizerFactory startSynchronizerFactory, Set internalEnvironmentProvisioners, OpenShiftEnvironmentProvisioner kubernetesEnvironmentProvisioner, - SidecarToolingProvisioner toolingProvisioner, + SidecarToolingProvisioner toolingProvisioner, @Assisted OpenShiftRuntimeContext context, @Assisted OpenShiftProject project, @Assisted List warnings) { super( workspaceStartTimeout, ingressStartTimeout, - unrecoverableEvents, urlRewriter, + unrecoverablePodEventListenerFactory, bootstrapperFactory, serverCheckerFactory, volumesStrategy, @@ -97,7 +98,7 @@ public OpenShiftInternalRuntime( project, warnings); this.project = project; - this.unrecoverableEvents = ImmutableSet.copyOf(unrecoverableEvents); + this.unrecoverablePodEventListenerFactory = unrecoverablePodEventListenerFactory; } @Override @@ -125,9 +126,12 @@ protected void startMachines() throws InfrastructureException { // project.pods().watch(new AbnormalStopHandler()); project.deployments().watchEvents(new MachineLogsPublisher()); - if (!unrecoverableEvents.isEmpty()) { + if (unrecoverablePodEventListenerFactory.isConfigured()) { Map pods = getContext().getEnvironment().getPods(); - project.deployments().watchEvents(new UnrecoverablePodEventHandler(pods)); + UnrecoverablePodEventListener handler = + unrecoverablePodEventListenerFactory.create( + pods.keySet(), this::handleUnrecoverableEvent); + project.deployments().watchEvents(handler); } doStartMachine(new OpenShiftServerResolver(createdServices, createdRoutes)); diff --git a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java index 59a77488a7b..d1b769d61cd 100644 --- a/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java +++ b/infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInternalRuntimeTest.java @@ -81,6 +81,7 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy; import org.eclipse.che.workspace.infrastructure.kubernetes.util.KubernetesSharedPool; import org.eclipse.che.workspace.infrastructure.kubernetes.util.RuntimeEventsPublisher; +import org.eclipse.che.workspace.infrastructure.kubernetes.util.UnrecoverablePodEventListenerFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.wsplugins.SidecarToolingProvisioner; import org.eclipse.che.workspace.infrastructure.openshift.environment.OpenShiftEnvironment; import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject; @@ -112,9 +113,6 @@ public class OpenShiftInternalRuntimeTest { private static final String ROUTE_HOST = "localhost"; private static final String M1_NAME = POD_NAME + '/' + CONTAINER_NAME_1; private static final String M2_NAME = POD_NAME + '/' + CONTAINER_NAME_2; - private static final String[] EMPTY_UNRECOVERABLE_EVENTS = new String[0]; - private static final String[] UNRECOVERABLE_EVENTS = - new String[] {"Failed Mount", "Failed Scheduling", "Failed to pull image"}; private static final RuntimeIdentity IDENTITY = new RuntimeIdentityImpl(WORKSPACE_ID, "env1", "id1"); @@ -142,7 +140,8 @@ public class OpenShiftInternalRuntimeTest { @Mock private KubernetesMachineCache machinesCache; @Mock private InternalEnvironmentProvisioner internalEnvironmentProvisioner; @Mock private OpenShiftEnvironmentProvisioner kubernetesEnvironmentProvisioner; - @Mock private SidecarToolingProvisioner toolingProvisioner; + @Mock private SidecarToolingProvisioner toolingProvisioner; + @Mock private UnrecoverablePodEventListenerFactory unrecoverablePodEventListenerFactory; @Captor private ArgumentCaptor machineStatusEventCaptor; @@ -162,8 +161,8 @@ public void setup() throws Exception { new OpenShiftInternalRuntime( 13, 5, - UNRECOVERABLE_EVENTS, new URLRewriter.NoOpURLRewriter(), + unrecoverablePodEventListenerFactory, bootstrapperFactory, serverCheckerFactory, volumesStrategy, @@ -185,8 +184,8 @@ public void setup() throws Exception { new OpenShiftInternalRuntime( 13, 5, - EMPTY_UNRECOVERABLE_EVENTS, new URLRewriter.NoOpURLRewriter(), + unrecoverablePodEventListenerFactory, bootstrapperFactory, serverCheckerFactory, volumesStrategy, @@ -245,6 +244,7 @@ public void shouldStartMachines() throws Exception { final ImmutableMap allPods = ImmutableMap.of(POD_NAME, mockPod(ImmutableList.of(container1, container2))); when(osEnv.getPods()).thenReturn(allPods); + when(unrecoverablePodEventListenerFactory.isConfigured()).thenReturn(true); internalRuntime.startMachines(); @@ -261,6 +261,7 @@ public void shouldStartMachines() throws Exception { @Test public void shouldStartMachinesWithoutUnrecoverableEventHandler() throws Exception { + when(unrecoverablePodEventListenerFactory.isConfigured()).thenReturn(false); final Container container1 = mockContainer(CONTAINER_NAME_1, EXPOSED_PORT_1); final Container container2 = mockContainer(CONTAINER_NAME_2, EXPOSED_PORT_2, INTERNAL_PORT); final ImmutableMap allPods =