fix: continue jobs + steps after failure (#840)

* fix: continue jobs + steps after failure

To allow proper if expression handling on jobs and steps (like always,
success, failure, ...) we need to continue running all executors in
the prepared chain.
To keep the error handling intact we add an occurred error to the
go context and handle it later in the pipeline/chain.

Also we add the job result to the needs context to give expressions
access to it.
The needs object, failure and success functions are split between
run context (on jobs) and step context.

Closes #442

Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de>

* style: correct linter warnings

Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de>

* fix: job if value defaults to success()

As described in the documentation, a default value of "success()" is
applied when no "if" value is present on the job.

https://docs.github.com/en/actions/learn-github-actions/expressions#job-status-check-functions

Co-authored-by: Markus Wolf <mail@markus-wolf.de>

* fix: check job needs recursively

Ensure job result includes results of previous jobs

Co-authored-by: Markus Wolf <markus.wolf@new-work.se>

* test: add runner test for job status check functions

Co-authored-by: Markus Wolf <markus.wolf@new-work.se>

* test: add unit tests for run context if evaluation

Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de>

* refactor: move if expression evaluation

Move if expression evaluation into own function (step context) to
better support unit testing.

Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de>

* test: add unit tests for step context if evaluation

Co-authored-by: Markus Wolf <markus.wolf@new-work.se>

* fix: handle job error more resilient

The job error is not stored in a context map instead of a context
added value.
Since context values are immutable an added value requires to keep
the new context in all cases. This is fragile since it might slip
unnoticed to other parts of the code.

Storing the error of a job in the context map will make it more stable,
since the map is always there and the context of the pipeline is stable
for the whole run.

* feat: steps should use a default if expression of success()

* test: add integration test for if-expressions

* chore: disable editorconfig-checker for yaml multiline string

Co-authored-by: Björn Brauer <zaubernerd@zaubernerd.de>
Co-authored-by: Björn Brauer <bjoern.brauer@new-work.se>

pkg/common/executor.go

@@ -136,13 +136,12 @@ func (e Executor) Then(then Executor) Executor {
 			case Warning:
 				log.Warning(err.Error())
 			default:
-				log.Debugf("%+v", err)
-				return err
+				SetJobError(ctx, err)
 			}
+		} else if ctx.Err() != nil {
+			SetJobError(ctx, ctx.Err())
 		}
-		if ctx.Err() != nil {
-			return ctx.Err()
-		}
+
 		return then(ctx)
 	}
 }

pkg/common/job_error.go

@@ -0,0 +1,30 @@
+package common
+
+import (
+	"context"
+)
+
+type jobErrorContextKey string
+
+const jobErrorContextKeyVal = jobErrorContextKey("job.error")
+
+// JobError returns the job error for current context if any
+func JobError(ctx context.Context) error {
+	val := ctx.Value(jobErrorContextKeyVal)
+	if val != nil {
+		if container, ok := val.(map[string]error); ok {
+			return container["error"]
+		}
+	}
+	return nil
+}
+
+func SetJobError(ctx context.Context, err error) {
+	ctx.Value(jobErrorContextKeyVal).(map[string]error)["error"] = err
+}
+
+// WithJobErrorContainer adds a value to the context as a container for an error
+func WithJobErrorContainer(ctx context.Context) context.Context {
+	container := map[string]error{}
+	return context.WithValue(ctx, jobErrorContextKeyVal, container)
+}

pkg/model/planner.go

@@ -78,6 +78,9 @@ func FixIfStatement(content []byte, wr *Workflow) error {
 			if err != nil {
 				return err
 			}
+			if val == "" {
+				val = "success()"
+			}
 			jobs[j].Steps[i].If.Value = val
 		}
 	}

pkg/model/workflow.go

@@ -69,6 +69,7 @@ type Job struct {
 	RawContainer   yaml.Node                 `yaml:"container"`
 	Defaults       Defaults                  `yaml:"defaults"`
 	Outputs        map[string]string         `yaml:"outputs"`
+	Result         string
 }
 
 // Strategy for the job
@@ -433,6 +434,9 @@ func (w *Workflow) GetJob(jobID string) *Job {
 			if j.Name == "" {
 				j.Name = id
 			}
+			if j.If.Value == "" {
+				j.If.Value = "success()"
+			}
 			return j
 		}
 	}

pkg/runner/expression.go

@@ -4,6 +4,8 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -25,6 +27,7 @@ func init() {
 // NewExpressionEvaluator creates a new evaluator
 func (rc *RunContext) NewExpressionEvaluator() ExpressionEvaluator {
 	vm := rc.newVM()
+
 	return &expressionEvaluator{
 		vm,
 	}
@@ -36,6 +39,10 @@ func (sc *StepContext) NewExpressionEvaluator() ExpressionEvaluator {
 	configers := []func(*otto.Otto){
 		sc.vmEnv(),
 		sc.vmInputs(),
+
+		sc.vmNeeds(),
+		sc.vmSuccess(),
+		sc.vmFailure(),
 	}
 	for _, configer := range configers {
 		configer(vm)
@@ -373,14 +380,33 @@ func (rc *RunContext) vmHashFiles() func(*otto.Otto) {
 func (rc *RunContext) vmSuccess() func(*otto.Otto) {
 	return func(vm *otto.Otto) {
 		_ = vm.Set("success", func() bool {
-			return rc.getJobContext().Status == "success"
+			jobs := rc.Run.Workflow.Jobs
+			jobNeeds := rc.getNeedsTransitive(rc.Run.Job())
+
+			for _, needs := range jobNeeds {
+				if jobs[needs].Result != "success" {
+					return false
+				}
+			}
+
+			return true
 		})
 	}
 }
+
 func (rc *RunContext) vmFailure() func(*otto.Otto) {
 	return func(vm *otto.Otto) {
 		_ = vm.Set("failure", func() bool {
-			return rc.getJobContext().Status == "failure"
+			jobs := rc.Run.Workflow.Jobs
+			jobNeeds := rc.getNeedsTransitive(rc.Run.Job())
+
+			for _, needs := range jobNeeds {
+				if jobs[needs].Result == "failure" {
+					return true
+				}
+			}
+
+			return false
 		})
 	}
 }
@@ -440,9 +466,9 @@ func (sc *StepContext) vmInputs() func(*otto.Otto) {
 	}
 }
 
-func (rc *RunContext) vmNeeds() func(*otto.Otto) {
-	jobs := rc.Run.Workflow.Jobs
-	jobNeeds := rc.Run.Job().Needs()
+func (sc *StepContext) vmNeeds() func(*otto.Otto) {
+	jobs := sc.RunContext.Run.Workflow.Jobs
+	jobNeeds := sc.RunContext.Run.Job().Needs()
 
 	using := make(map[string]map[string]map[string]string)
 	for _, needs := range jobNeeds {
@@ -457,6 +483,70 @@ func (rc *RunContext) vmNeeds() func(*otto.Otto) {
 	}
 }
 
+func (sc *StepContext) vmSuccess() func(*otto.Otto) {
+	return func(vm *otto.Otto) {
+		_ = vm.Set("success", func() bool {
+			return sc.RunContext.getJobContext().Status == "success"
+		})
+	}
+}
+
+func (sc *StepContext) vmFailure() func(*otto.Otto) {
+	return func(vm *otto.Otto) {
+		_ = vm.Set("failure", func() bool {
+			return sc.RunContext.getJobContext().Status == "failure"
+		})
+	}
+}
+
+type vmNeedsStruct struct {
+	Outputs map[string]string `json:"outputs"`
+	Result  string            `json:"result"`
+}
+
+func (rc *RunContext) vmNeeds() func(*otto.Otto) {
+	return func(vm *otto.Otto) {
+		needsFunc := func() otto.Value {
+			jobs := rc.Run.Workflow.Jobs
+			jobNeeds := rc.Run.Job().Needs()
+
+			using := make(map[string]vmNeedsStruct)
+			for _, needs := range jobNeeds {
+				using[needs] = vmNeedsStruct{
+					Outputs: jobs[needs].Outputs,
+					Result:  jobs[needs].Result,
+				}
+			}
+
+			log.Debugf("context needs => %+v", using)
+
+			value, err := vm.ToValue(using)
+			if err != nil {
+				return vm.MakeTypeError(err.Error())
+			}
+
+			return value
+		}
+
+		// Results might change after the Otto VM was created
+		// and initialized. To access the current state
+		// we can't just pass a copy to Otto - instead we
+		// created a 'live-binding'.
+		// Technical Note: We don't want to pollute the global
+		// js namespace (and add things github actions hasn't)
+		// we delete the helper function after installing it
+		// as a getter.
+		global, _ := vm.Run("this")
+		_ = global.Object().Set("__needs__", needsFunc)
+		_, _ = vm.Run(`
+			(function (global) {
+				Object.defineProperty(global, 'needs', { get: global.__needs__ });
+				delete global.__needs__;
+			})(this)
+		`)
+	}
+}
+
 func (rc *RunContext) vmJob() func(*otto.Otto) {
 	job := rc.getJobContext()
 
@@ -518,3 +608,48 @@ func (rc *RunContext) vmMatrix() func(*otto.Otto) {
 		_ = vm.Set("matrix", rc.Matrix)
 	}
 }
+
+// EvalBool evaluates an expression against given evaluator
+func EvalBool(evaluator ExpressionEvaluator, expr string) (bool, error) {
+	if splitPattern == nil {
+		splitPattern = regexp.MustCompile(fmt.Sprintf(`%s|%s|\S+`, expressionPattern.String(), operatorPattern.String()))
+	}
+	if strings.HasPrefix(strings.TrimSpace(expr), "!") {
+		return false, errors.New("expressions starting with ! must be wrapped in ${{ }}")
+	}
+	if expr != "" {
+		parts := splitPattern.FindAllString(expr, -1)
+		var evaluatedParts []string
+		for i, part := range parts {
+			if operatorPattern.MatchString(part) {
+				evaluatedParts = append(evaluatedParts, part)
+				continue
+			}
+
+			interpolatedPart, isString := evaluator.InterpolateWithStringCheck(part)
+
+			// This peculiar transformation has to be done because the GitHub parser
+			// treats false returned from contexts as a string, not a boolean.
+			// Hence env.SOMETHING will be evaluated to true in an if: expression
+			// regardless if SOMETHING is set to false, true or any other string.
+			// It also handles some other weirdness that I found by trial and error.
+			if (expressionPattern.MatchString(part) && // it is an expression
+				!strings.Contains(part, "!")) && // but it's not negated
+				interpolatedPart == "false" && // and the interpolated string is false
+				(isString || previousOrNextPartIsAnOperator(i, parts)) { // and it's of type string or has an logical operator before or after
+				interpolatedPart = fmt.Sprintf("'%s'", interpolatedPart) // then we have to quote the false expression
+			}
+
+			evaluatedParts = append(evaluatedParts, interpolatedPart)
+		}
+
+		joined := strings.Join(evaluatedParts, " ")
+		v, _, err := evaluator.Evaluate(fmt.Sprintf("Boolean(%s)", joined))
+		if err != nil {
+			return false, err
+		}
+		log.Debugf("expression '%s' evaluated to '%s'", expr, v)
+		return v == "true", nil
+	}
+	return true, nil
+}

pkg/runner/run_context.go

@@ -3,7 +3,6 @@ package runner
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -288,7 +287,20 @@ func (rc *RunContext) Executor() common.Executor {
 		}
 		steps = append(steps, rc.newStepExecutor(step))
 	}
-	steps = append(steps, rc.stopJobContainer())
+	steps = append(steps, func(ctx context.Context) error {
+		err := rc.stopJobContainer()(ctx)
+		if err != nil {
+			return err
+		}
+
+		rc.Run.Job().Result = "success"
+		jobError := common.JobError(ctx)
+		if jobError != nil {
+			rc.Run.Job().Result = "failure"
+		}
+
+		return nil
+	})
 
 	return common.NewPipelineExecutor(steps...).Finally(rc.interpolateOutputs()).Finally(func(ctx context.Context) error {
 		if rc.JobContainer != nil {
@@ -310,15 +322,9 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor {
 			Conclusion: stepStatusSuccess,
 			Outputs:    make(map[string]string),
 		}
-		runStep, err := rc.EvalBool(sc.Step.If.Value)
 
+		runStep, err := sc.isEnabled(ctx)
 		if err != nil {
-			common.Logger(ctx).Errorf("  \u274C  Error in if: expression - %s", sc.Step)
-			exprEval, err := sc.setupEnv(ctx)
-			if err != nil {
-				return err
-			}
-			rc.ExprEval = exprEval
 			rc.StepResults[rc.CurrentStep].Conclusion = stepStatusFailure
 			rc.StepResults[rc.CurrentStep].Outcome = stepStatusFailure
 			return err
@@ -403,7 +409,7 @@ func (rc *RunContext) hostname() string {
 func (rc *RunContext) isEnabled(ctx context.Context) bool {
 	job := rc.Run.Job()
 	l := common.Logger(ctx)
-	runJob, err := rc.EvalBool(job.If.Value)
+	runJob, err := EvalBool(rc.ExprEval, job.If.Value)
 	if err != nil {
 		common.Logger(ctx).Errorf("  \u274C  Error in if: expression - %s", job.Name)
 		return false
@@ -430,51 +436,6 @@ func (rc *RunContext) isEnabled(ctx context.Context) bool {
 
 var splitPattern *regexp.Regexp
 
-// EvalBool evaluates an expression against current run context
-func (rc *RunContext) EvalBool(expr string) (bool, error) {
-	if splitPattern == nil {
-		splitPattern = regexp.MustCompile(fmt.Sprintf(`%s|%s|\S+`, expressionPattern.String(), operatorPattern.String()))
-	}
-	if strings.HasPrefix(strings.TrimSpace(expr), "!") {
-		return false, errors.New("expressions starting with ! must be wrapped in ${{ }}")
-	}
-	if expr != "" {
-		parts := splitPattern.FindAllString(expr, -1)
-		var evaluatedParts []string
-		for i, part := range parts {
-			if operatorPattern.MatchString(part) {
-				evaluatedParts = append(evaluatedParts, part)
-				continue
-			}
-
-			interpolatedPart, isString := rc.ExprEval.InterpolateWithStringCheck(part)
-
-			// This peculiar transformation has to be done because the GitHub parser
-			// treats false returned from contexts as a string, not a boolean.
-			// Hence env.SOMETHING will be evaluated to true in an if: expression
-			// regardless if SOMETHING is set to false, true or any other string.
-			// It also handles some other weirdness that I found by trial and error.
-			if (expressionPattern.MatchString(part) && // it is an expression
-				!strings.Contains(part, "!")) && // but it's not negated
-				interpolatedPart == "false" && // and the interpolated string is false
-				(isString || previousOrNextPartIsAnOperator(i, parts)) { // and it's of type string or has an logical operator before or after
-				interpolatedPart = fmt.Sprintf("'%s'", interpolatedPart) // then we have to quote the false expression
-			}
-
-			evaluatedParts = append(evaluatedParts, interpolatedPart)
-		}
-
-		joined := strings.Join(evaluatedParts, " ")
-		v, _, err := rc.ExprEval.Evaluate(fmt.Sprintf("Boolean(%s)", joined))
-		if err != nil {
-			return false, err
-		}
-		log.Debugf("expression '%s' evaluated to '%s'", expr, v)
-		return v == "true", nil
-	}
-	return true, nil
-}
-
 func previousOrNextPartIsAnOperator(i int, parts []string) bool {
 	operator := false
 	if i > 0 {
@@ -557,6 +518,17 @@ func (rc *RunContext) getStepsContext() map[string]*stepResult {
 	return rc.StepResults
 }
 
+func (rc *RunContext) getNeedsTransitive(job *model.Job) []string {
+	needs := job.Needs()
+
+	for _, need := range needs {
+		parentNeeds := rc.getNeedsTransitive(rc.Run.Workflow.GetJob(need))
+		needs = append(needs, parentNeeds...)
+	}
+
+	return needs
+}
+
 type githubContext struct {
 	Event            map[string]interface{} `json:"event"`
 	EventPath        string                 `json:"event_path"`