Merge branch 'main' into DEVPROD-7219

agent/agent.go

@@ -586,11 +586,7 @@ func (a *Agent) startLogging(ctx context.Context, tc *taskContext) error {
 	}
 	taskLogDir := filepath.Join(a.opts.WorkingDirectory, taskLogDirectory)
 	grip.Error(errors.Wrapf(os.RemoveAll(taskLogDir), "removing task log directory '%s'", taskLogDir))
-	if tc.taskConfig.Project.Loggers != nil {
-		tc.logger, err = a.makeLoggerProducer(ctx, tc, tc.taskConfig.Project.Loggers, "")
-	} else {
-		tc.logger, err = a.makeLoggerProducer(ctx, tc, &model.LoggerConfig{}, "")
-	}
+	tc.logger, err = a.makeLoggerProducer(ctx, tc, "")
 	if err != nil {
 		return errors.Wrap(err, "making the logger producer")
 	}
@@ -960,21 +956,29 @@ func (a *Agent) finishTask(ctx context.Context, tc *taskContext, status string,
 	switch detail.Status {
 	case evergreen.TaskSucceeded:
 		a.handleTimeoutAndOOM(ctx, tc, detail, status)
+
 		tc.logger.Task().Info("Task completed - SUCCESS.")
 		if err := a.runPostOrTeardownTaskCommands(ctx, tc); err != nil {
 			tc.logger.Task().Info("Post task completed - FAILURE. Overall task status changed to FAILED.")
 			setEndTaskFailureDetails(tc, detail, evergreen.TaskFailed, "", "", nil)
 		}
+
 		detail.PostErrored = tc.getPostErrored()
+		detail.OtherFailingCommands = tc.getOtherFailingCommands()
+
 		a.runEndTaskSync(ctx, tc, detail)
 	case evergreen.TaskFailed:
 		a.handleTimeoutAndOOM(ctx, tc, detail, status)
+
 		tc.logger.Task().Info("Task completed - FAILURE.")
 		// If the post commands error, ignore the error. runCommandsInBlock
 		// already logged the error, and the post commands cannot cause the
 		// task to fail since the task already failed.
 		_ = a.runPostOrTeardownTaskCommands(ctx, tc)
+
 		detail.PostErrored = tc.getPostErrored()
+		detail.OtherFailingCommands = tc.getOtherFailingCommands()
+
 		a.runEndTaskSync(ctx, tc, detail)
 	case evergreen.TaskSystemFailed:
 		// This is a special status indicating that the agent failed for reasons
@@ -1141,30 +1145,34 @@ func (a *Agent) endTaskResponse(ctx context.Context, tc *taskContext, status str
 
 func setEndTaskFailureDetails(tc *taskContext, detail *apimodels.TaskEndDetail, status, description, failureType string, failureMetadataTagsToAdd []string) {
 	var isDefaultDescription bool
-	if tc.getCurrentCommand() != nil {
+	currCmd := tc.getCurrentCommand()
+	if currCmd != nil {
 		// If there is no explicit user-defined description or failure type,
 		// infer that information from the last command that ran.
 		if description == "" {
-			description = tc.getCurrentCommand().FullDisplayName()
+			description = currCmd.FullDisplayName()
 			isDefaultDescription = true
 		}
 		if failureType == "" {
-			failureType = tc.getCurrentCommand().Type()
+			failureType = currCmd.Type()
 		}
 	}
 
 	detail.Status = status
 	if status != evergreen.TaskSucceeded {
 		detail.Type = failureType
 		detail.Description = description
-		detail.FailureMetadataTags = utility.UniqueStrings(append(tc.getCurrentCommand().FailureMetadataTags(), failureMetadataTagsToAdd...))
+		detail.FailureMetadataTags = utility.UniqueStrings(append(currCmd.FailureMetadataTags(), failureMetadataTagsToAdd...))
+		tc.setFailingCommand(currCmd)
 	}
 	if !isDefaultDescription {
 		// If there's an explicit user-defined description, always set that
 		// description.
 		detail.Description = description
 	}
 
+	detail.OtherFailingCommands = tc.getOtherFailingCommands()
+
 	if !detail.TimedOut {
 		// Only set timeout details if a prior command in the task hasn't
 		// already recorded a timeout. For example, if a command times out in
@@ -1174,7 +1182,6 @@ func setEndTaskFailureDetails(tc *taskContext, detail *apimodels.TaskEndDetail,
 		detail.TimeoutType = string(tc.getTimeoutType())
 		detail.TimeoutDuration = tc.getTimeoutDuration()
 	}
-
 }
 
 func (a *Agent) killProcs(ctx context.Context, tc *taskContext, ignoreTaskGroupCheck bool, reason string) {

agent/agent_test.go

@@ -762,7 +762,7 @@ func (s *AgentSuite) setupRunTask(projYml string) {
 	s.mockCommunicator.GetProjectResponse = p
 }
 
-func (s *AgentSuite) TestFailingPostWithPostErrorFailsTaskSetsEndTaskResults() {
+func (s *AgentSuite) TestFailingPostWithPostErrorFailsTaskSetsFailedEndTaskResults() {
 	projYml := `
 buildvariants:
   - name: mock_build_variant
@@ -796,7 +796,8 @@ post:
 	s.True(s.mockCommunicator.EndTaskResult.Detail.TimedOut)
 	s.EqualValues(globals.PostTimeout, s.mockCommunicator.EndTaskResult.Detail.TimeoutType)
 	s.Equal(time.Second, s.mockCommunicator.EndTaskResult.Detail.TimeoutDuration)
-	s.ElementsMatch([]string{"failure_tag1"}, s.mockCommunicator.EndTaskResult.Detail.FailureMetadataTags, "failure tags should be set for failing post command")
+	s.ElementsMatch([]string{"failure_tag1"}, s.mockCommunicator.EndTaskResult.Detail.FailureMetadataTags, "failure tags should be set for post command that fails task")
+	s.Empty(s.mockCommunicator.EndTaskResult.Detail.OtherFailingCommands, "should not set other failing commands when main task command succeeds and post command fails task")
 
 	s.NoError(s.tc.logger.Close())
 	checkMockLogs(s.T(), s.mockCommunicator, s.tc.taskConfig.Task.Id, []string{
@@ -818,20 +819,22 @@ post:
 	})
 }
 
-func (s *AgentSuite) TestFailingPostDoesNotChangeEndTaskResults() {
+func (s *AgentSuite) TestFailingPostSetsSuccessfulEndTaskResults() {
 	projYml := `
 buildvariants:
   - name: mock_build_variant
 
 tasks:
   - name: this_is_a_task_name
+    failure_metadata_tags: ["failure_tag0"]
     commands:
       - command: shell.exec
         params:
           script: exit 0
 
 post:
   - command: shell.exec
+    failure_metadata_tags: ["failure_tag1"]
     params:
       script: exit 1
 `
@@ -847,7 +850,10 @@ post:
 	s.Equal(evergreen.TaskSucceeded, s.mockCommunicator.EndTaskResult.Detail.Status)
 	s.Zero(s.mockCommunicator.EndTaskResult.Detail.Description, "should not include command failure description for a successful task")
 	s.Zero(s.mockCommunicator.EndTaskResult.Detail.Type, "should not include command failure type for a successful task")
-	s.Empty(s.mockCommunicator.EndTaskResult.Detail.FailureMetadataTags, "failure metadata tags should not be set for post command that fails but does not fail the task")
+	s.Empty(s.mockCommunicator.EndTaskResult.Detail.FailureMetadataTags, "failure metadata tags should not be set when task succeeds")
+	s.Require().Len(s.mockCommunicator.EndTaskResult.Detail.OtherFailingCommands, 1)
+	s.Contains("'shell.exec' (step 1 of 1) in block 'post'", s.mockCommunicator.EndTaskResult.Detail.OtherFailingCommands[0].FullDisplayName, "should set failing post command that does not fail the task")
+	s.ElementsMatch([]string{"failure_tag1"}, s.mockCommunicator.EndTaskResult.Detail.OtherFailingCommands[0].FailureMetadataTags, "should set failure metadata tags for failing post command that does not fail the task")
 
 	s.NoError(s.tc.logger.Close())
 	checkMockLogs(s.T(), s.mockCommunicator, s.tc.taskConfig.Task.Id, []string{
@@ -900,7 +906,8 @@ post:
 	s.Equal(evergreen.TaskSucceeded, s.mockCommunicator.EndTaskResult.Detail.Status)
 	s.Zero(s.mockCommunicator.EndTaskResult.Detail.Description, "should not include command failure description for a successful task")
 	s.Zero(s.mockCommunicator.EndTaskResult.Detail.Type, "should not include command failure type for a successful task")
-	s.Empty(s.mockCommunicator.EndTaskResult.Detail.FailureMetadataTags, "failure metadata tags should not be set for all successful commands")
+	s.Empty(s.mockCommunicator.EndTaskResult.Detail.FailureMetadataTags, "failure metadata tags should not be set if task succeeds")
+	s.Empty(s.mockCommunicator.EndTaskResult.Detail.OtherFailingCommands, "should not include other failing commands for a successful task")
 
 	s.NoError(s.tc.logger.Close())
 	checkMockLogs(s.T(), s.mockCommunicator, s.tc.taskConfig.Task.Id, []string{
@@ -955,6 +962,9 @@ post:
 	s.Equal("'shell.exec' (step 1 of 1)", s.mockCommunicator.EndTaskResult.Detail.Description, "should show main block command as the failing command if both main and post block commands fail")
 	s.True(s.mockCommunicator.EndTaskResult.Detail.TimedOut, "should show main block command hitting timeout")
 	s.ElementsMatch([]string{"failure_tag0"}, s.mockCommunicator.EndTaskResult.Detail.FailureMetadataTags, "failure tags should be set for failing main task command")
+	s.Require().Len(s.mockCommunicator.EndTaskResult.Detail.OtherFailingCommands, 1)
+	s.Equal("'shell.exec' (step 1 of 1) in block 'post'", s.mockCommunicator.EndTaskResult.Detail.OtherFailingCommands[0].FullDisplayName, "failing post command should be set")
+	s.ElementsMatch([]string{"failure_tag1"}, s.mockCommunicator.EndTaskResult.Detail.OtherFailingCommands[0].FailureMetadataTags, "failure tags should be set for failing post command")
 
 	s.NoError(s.tc.logger.Close())
 	checkMockLogs(s.T(), s.mockCommunicator, s.tc.taskConfig.Task.Id, []string{
@@ -1005,6 +1015,7 @@ post:
 	s.Equal(evergreen.TaskFailed, s.mockCommunicator.EndTaskResult.Detail.Status)
 	s.Equal("'shell.exec' (step 1 of 1)", s.mockCommunicator.EndTaskResult.Detail.Description)
 	s.ElementsMatch([]string{"failure_tag0"}, s.mockCommunicator.EndTaskResult.Detail.FailureMetadataTags, "failure tags should be set for failing main task command")
+	s.Empty(s.mockCommunicator.EndTaskResult.Detail.OtherFailingCommands, "should not include other failing commands when post command succeeds")
 
 	s.NoError(s.tc.logger.Close())
 	checkMockLogs(s.T(), s.mockCommunicator, s.tc.taskConfig.Task.Id, []string{

agent/command.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/evergreen-ci/evergreen"
 	"github.com/evergreen-ci/evergreen/agent/command"
-	"github.com/evergreen-ci/evergreen/agent/internal/client"
 	"github.com/evergreen-ci/evergreen/apimodels"
 	"github.com/evergreen-ci/evergreen/model"
 	"github.com/evergreen-ci/evergreen/util"
@@ -165,24 +164,6 @@ func (a *Agent) blockToLegacyName(block command.BlockType) string {
 func (a *Agent) runCommandOrFunc(ctx context.Context, tc *taskContext, commandInfo model.PluginCommandConf,
 	cmds []command.Command, options runCommandsOptions) error {
 
-	var err error
-	var logger client.LoggerProducer
-	// if there is a command-specific logger, make it here otherwise use the task-level logger
-	if commandInfo.Loggers == nil {
-		logger = tc.logger
-	} else {
-		logger, err = a.makeLoggerProducer(ctx, tc, commandInfo.Loggers, getCommandNameForFileLogger(commandInfo))
-		if err != nil {
-			return errors.Wrap(err, "making command logger")
-		}
-		defer func() {
-			// If the logger is a command-specific logger, when the command
-			// finishes, the loggers should have no more logs to send. Closing
-			// it ensure that the command logger flushes all logs and cleans up.
-			grip.Error(errors.Wrap(logger.Close(), "closing command logger"))
-		}()
-	}
-
 	if commandInfo.Function != "" {
 		var commandSetSpan trace.Span
 		ctx, commandSetSpan = a.tracer.Start(ctx, "function", trace.WithAttributes(
@@ -213,14 +194,14 @@ func (a *Agent) runCommandOrFunc(ctx context.Context, tc *taskContext, commandIn
 
 		cmd.SetJasperManager(a.jasper)
 
-		if err := a.runCommand(ctx, tc, logger, commandInfo, cmd, options); err != nil {
+		if err := a.runCommand(ctx, tc, commandInfo, cmd, options); err != nil {
 			commandSpan.SetStatus(codes.Error, "running command")
 			commandSpan.RecordError(err, trace.WithAttributes(tc.taskConfig.TaskAttributes()...))
 			commandSpan.End()
 			if cmd.RetryOnFailure() {
-				logger.Task().Infof("Command is set to automatically restart on completion, this can be done %d total times per task.", evergreen.MaxAutomaticRestarts)
+				tc.logger.Task().Infof("Command is set to automatically restart on completion, this can be done %d total times per task.", evergreen.MaxAutomaticRestarts)
 				if restartErr := a.comm.MarkFailedTaskToRestart(ctx, tc.task); restartErr != nil {
-					logger.Task().Errorf("Encountered error marking task to restart upon completion: %s", restartErr)
+					tc.logger.Task().Errorf("Encountered error marking task to restart upon completion: %s", restartErr)
 				}
 			}
 			return errors.Wrap(err, "running command")
@@ -232,7 +213,7 @@ func (a *Agent) runCommandOrFunc(ctx context.Context, tc *taskContext, commandIn
 
 // runCommand runs a single command, which is either a standalone command or a
 // single sub-command within a function.
-func (a *Agent) runCommand(ctx context.Context, tc *taskContext, logger client.LoggerProducer, commandInfo model.PluginCommandConf,
+func (a *Agent) runCommand(ctx context.Context, tc *taskContext, commandInfo model.PluginCommandConf,
 	cmd command.Command, options runCommandsOptions) error {
 	prevExp := map[string]string{}
 	for key, val := range commandInfo.Vars {
@@ -296,13 +277,14 @@ func (a *Agent) runCommand(ctx context.Context, tc *taskContext, logger client.L
 			cmdChan <- pErr
 		}()
 
-		cmdChan <- cmd.Execute(ctx, a.comm, logger, tc.taskConfig)
+		cmdChan <- cmd.Execute(ctx, a.comm, tc.logger, tc.taskConfig)
 	}()
 
 	select {
 	case err := <-cmdChan:
 		if err != nil {
 			tc.logger.Task().Errorf("Command %s failed: %s.", cmd.FullDisplayName(), err)
+			tc.addFailingCommand(cmd)
 			if options.block == command.PostBlock {
 				tc.setPostErrored(true)
 			}
@@ -323,6 +305,11 @@ func (a *Agent) runCommand(ctx context.Context, tc *taskContext, logger client.L
 		case <-cmdChan:
 		}
 
+		tc.addFailingCommand(cmd)
+		if options.block == command.PostBlock {
+			tc.setPostErrored(true)
+		}
+
 		tc.logger.Task().Errorf("Command %s stopped early: %s.", cmd.FullDisplayName(), ctx.Err())
 		return errors.Wrap(ctx.Err(), "command stopped early")
 	}
@@ -336,21 +323,6 @@ func (a *Agent) runCommand(ctx context.Context, tc *taskContext, logger client.L
 	return nil
 }
 
-// getCommandNameForFileLogger gets the name of the command that should be used
-// when the file logger is being used.
-func getCommandNameForFileLogger(commandInfo model.PluginCommandConf) string {
-	if commandInfo.DisplayName != "" {
-		return commandInfo.DisplayName
-	}
-	if commandInfo.Function != "" {
-		return commandInfo.Function
-	}
-	if commandInfo.Command != "" {
-		return commandInfo.Command
-	}
-	return "unknown function"
-}
-
 // endTaskSyncCommands returns the commands to sync the task to S3 if it was
 // requested when the task completes.
 func endTaskSyncCommands(tc *taskContext, detail *apimodels.TaskEndDetail) *model.YAMLCommandSet {