MAPREDUCE-7407. Avoid stopContainer() on dead node

apache · Aug 21, 2022 · 3abd3f6 · 3abd3f6
1 parent 7f176d0
commit 3abd3f6
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 2 deletions.
diff --git a/...-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java b/...-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
@@ -385,6 +385,20 @@ public void run() {
       // TODO: Do it only once per NodeManager.
       ContainerId containerID = event.getContainerID();
 
+      // If the container failed to launch earlier (due to dead node for example),
+      // it has been marked as FAILED and removed from containers during
+      // CONTAINER_REMOTE_LAUNCH event handling.
+      // Skip kill() such container during CONTAINER_REMOTE_CLEANUP as
+      // it is not necessary and could cost 15 minutes delay if the node is dead.
+      if (event.getType() == EventType.CONTAINER_REMOTE_CLEANUP &&
+          !containers.containsKey(containerID)) {
+        LOG.info("Skip cleanup of already-removed container " + containerID);
+        // send killed event to task attempt regardless like in kill().
+        context.getEventHandler().handle(new TaskAttemptEvent(event.getTaskAttemptID(),
+            TaskAttemptEventType.TA_CONTAINER_CLEANED));
+        return;
+      }
+
       Container c = getContainer(event);
       switch(event.getType()) {
 

diff --git a/.../src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/.../src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -283,8 +283,24 @@ public void testOutOfOrder() throws Exception {
       ut.handle(mockLaunchEvent);
 
       ut.waitForPoolToIdle();
-
-      verify(mockCM, never()).startContainers(any(StartContainersRequest.class));
+
+      verify(mockCM).startContainers(any(StartContainersRequest.class));
+
+      LOG.info("inserting cleanup event");
+      ContainerLauncherEvent mockCleanupEvent2 =
+          mock(ContainerLauncherEvent.class);
+      when(mockCleanupEvent2.getType())
+          .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
+      when(mockCleanupEvent2.getContainerID())
+          .thenReturn(contId);
+      when(mockCleanupEvent2.getTaskAttemptID()).thenReturn(taskAttemptId);
+      when(mockCleanupEvent2.getContainerMgrAddress()).thenReturn(cmAddress);
+      ut.handle(mockCleanupEvent2);
+
+      ut.waitForPoolToIdle();
+
+      // Verifies stopContainers is called on existing container
+      verify(mockCM).stopContainers(any(StopContainersRequest.class));
     } finally {
       ut.stop();
     }