get container exit code from event if we can't get it from inspect.

In case when we received a container die event but was not able to inspect the container (e.g. due to timeout), we will use the exit code from the event, so that the exit code of the container is still reported and available for customer to see from describing task.

agent/dockerclient/dockerapi/docker_client.go

@@ -64,6 +64,10 @@ const (
 	maxHealthCheckOutputLength = 1024
 	// VolumeDriverType is one of the plugin capabilities see https://docs.docker.com/engine/reference/commandline/plugin_ls/#filtering
 	VolumeDriverType = "volumedriver"
+	// dockerContainerDieEvent is the name of the event generated by Docker when a container died.
+	dockerContainerDieEvent = "die"
+	// dockerContainerEventExitCodeAttribute is the attribute name to get exit code from Docker event attribute.
+	dockerContainerEventExitCodeAttribute = "exitCode"
 )
 
 // Timelimits for docker operations enforced above docker
@@ -1015,6 +1019,10 @@ func (dg *dockerGoClient) handleContainerEvents(ctx context.Context,
 		}
 
 		metadata := dg.containerMetadata(ctx, containerID)
+		// In case when we received a container die event but was not able to inspect the container (e.g. due to timeout),
+		// we will use the exit code from the event, so that the exit code of the container is still reported and
+		// available for customer to see from describing task.
+		setExitCodeFromEvent(event, &metadata)
 
 		changedContainers <- DockerContainerChangeEvent{
 			Status:                  status,
@@ -1024,6 +1032,26 @@ func (dg *dockerGoClient) handleContainerEvents(ctx context.Context,
 	}
 }
 
+// setExitCodeFromEvent tries to get exit code from event and stores it in metadata, if metadata doesn't
+// contain the exit code already.
+func setExitCodeFromEvent(event *events.Message, metadata *DockerContainerMetadata) {
+	// exit code is only available from die event.
+	if metadata.ExitCode != nil || event.Status != dockerContainerDieEvent {
+		return
+	}
+	exitCode, ok := event.Actor.Attributes[dockerContainerEventExitCodeAttribute]
+	if !ok {
+		return
+	}
+	code, err := strconv.Atoi(exitCode)
+	if err != nil {
+		seelog.Errorf("Received invalid exit code from docker container event. exit code: %s, parse err: %v",
+			exitCode, err)
+		return
+	}
+	metadata.ExitCode = &code
+}
+
 // ListContainers returns a slice of container IDs.
 func (dg *dockerGoClient) ListContainers(ctx context.Context, all bool, timeout time.Duration) ListContainersResponse {
 	ctx, cancel := context.WithTimeout(ctx, timeout)

agent/dockerclient/dockerapi/docker_client_test.go

@@ -995,6 +995,74 @@ func TestContainerEventsError(t *testing.T) {
 	}
 }
 
+func TestSetExitCodeFromEvent(t *testing.T) {
+	var (
+		exitCodeInt    = 42
+		exitCodeStr    = "42"
+		altExitCodeInt = 1
+	)
+
+	defaultEvent := &events.Message{
+		Status: dockerContainerDieEvent,
+		Actor: events.Actor{
+			Attributes: map[string]string{
+				dockerContainerEventExitCodeAttribute: exitCodeStr,
+			},
+		},
+	}
+
+	testCases := []struct {
+		name             string
+		event            *events.Message
+		metadata         DockerContainerMetadata
+		expectedExitCode *int
+	}{
+		{
+			name:             "exit code set from event",
+			event:            defaultEvent,
+			metadata:         DockerContainerMetadata{},
+			expectedExitCode: &exitCodeInt,
+		},
+		{
+			name:  "exit code not set from event when metadata already has it",
+			event: defaultEvent,
+			metadata: DockerContainerMetadata{
+				ExitCode: &altExitCodeInt,
+			},
+			expectedExitCode: &altExitCodeInt,
+		},
+		{
+			name: "exit code not set from event when event does not has it",
+			event: &events.Message{
+				Status: dockerContainerDieEvent,
+				Actor:  events.Actor{},
+			},
+			metadata:         DockerContainerMetadata{},
+			expectedExitCode: nil,
+		},
+		{
+			name: "exit code not set from event when event has invalid exit code",
+			event: &events.Message{
+				Status: dockerContainerDieEvent,
+				Actor: events.Actor{
+					Attributes: map[string]string{
+						dockerContainerEventExitCodeAttribute: "invalid",
+					},
+				},
+			},
+			metadata:         DockerContainerMetadata{},
+			expectedExitCode: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			setExitCodeFromEvent(tc.event, &tc.metadata)
+			assert.Equal(t, tc.expectedExitCode, tc.metadata.ExitCode)
+		})
+	}
+}
+
 func TestDockerVersion(t *testing.T) {
 	mockDockerSDK, client, _, _, _, done := dockerClientSetup(t)
 	defer done()