Merge branch 'master' into share-env-file-parser-container-host

.github/workflows/checks.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: golangci/golangci-lint-action@v3.3.1
         with:
           version: v1.47.2
-      - uses: megalinter/megalinter/flavors/go@v6.14.0
+      - uses: megalinter/megalinter/flavors/go@v6.15.0
         env:
           DEFAULT_BRANCH: master
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -148,6 +148,9 @@ act pull_request
 # Run a specific job:
 act -j test
 
+# Run a job in a specific workflow (useful if you have duplicate job names)
+act -j lint -W .github/workflows/checks.yml
+
 # Run in dry-run mode:
 act -n
 
@@ -339,10 +342,41 @@ MY_ENV_VAR=MY_ENV_VAR_VALUE
 MY_2ND_ENV_VAR="my 2nd env var value"
 ```
 
+# Skipping jobs
+
+You cannot use the `env` context in job level if conditions, but you can add a custom event property to the `github` context. You can use this method also on step level if conditions.
+
+```yml
+on: push
+jobs:
+  deploy:
+    if: ${{ !github.event.act }} # skip during local actions testing
+    runs-on: ubuntu-latest
+    steps:
+    - run: exit 0
+```
+
+And use this `event.json` file with act otherwise the Job will run:
+
+```json
+{
+    "act": true
+}
+```
+
+Run act like
+
+```sh
+act -e event.json
+```
+
+_Hint: you can add / append `-e event.json` as a line into `./.actrc`_
+
 # Skipping steps
 
 Act adds a special environment variable `ACT` that can be used to skip a step that you
 don't want to run locally. E.g. a step that posts a Slack message or bumps a version number.
+**You cannot use this method in job level if conditions, see [Skipping jobs](#skipping-jobs)**
 
 ```yml
 - name: Some step

cmd/input.go

@@ -30,6 +30,7 @@ type Input struct {
 	usernsMode                         string
 	containerArchitecture              string
 	containerDaemonSocket              string
+	containerOptions                   string
 	noWorkflowRecurse                  bool
 	useGitIgnore                       bool
 	githubInstance                     string

cmd/root.go

@@ -76,6 +76,7 @@ func Execute(ctx context.Context, version string) {
 	rootCmd.PersistentFlags().StringVarP(&input.envfile, "env-file", "", ".env", "environment file to read and use as env in the containers")
 	rootCmd.PersistentFlags().StringVarP(&input.containerArchitecture, "container-architecture", "", "", "Architecture which should be used to run containers, e.g.: linux/amd64. If not specified, will use host default architecture. Requires Docker server API Version 1.41+. Ignored on earlier Docker server platforms.")
 	rootCmd.PersistentFlags().StringVarP(&input.containerDaemonSocket, "container-daemon-socket", "", "/var/run/docker.sock", "Path to Docker daemon socket which will be mounted to containers")
+	rootCmd.PersistentFlags().StringVarP(&input.containerOptions, "container-options", "", "", "Custom docker container options for the job container without an options property in the job definition")
 	rootCmd.PersistentFlags().StringVarP(&input.githubInstance, "github-instance", "", "github.com", "GitHub instance to use. Don't use this if you are not using GitHub Enterprise Server.")
 	rootCmd.PersistentFlags().StringVarP(&input.artifactServerPath, "artifact-server-path", "", "", "Defines the path where the artifact server stores uploads and retrieves downloads from. If not specified the artifact server will not start.")
 	rootCmd.PersistentFlags().StringVarP(&input.artifactServerPort, "artifact-server-port", "", "34567", "Defines the port where the artifact server listens (will only bind to localhost).")
@@ -414,6 +415,22 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str
 				input.platforms = readArgsFile(cfgLocations[0], true)
 			}
 		}
+		deprecationWarning := "--%s is deprecated and will be removed soon, please switch to cli: `--container-options \"%[2]s\"` or `.actrc`: `--container-options %[2]s`."
+		if input.privileged {
+			log.Warnf(deprecationWarning, "privileged", "--privileged")
+		}
+		if len(input.usernsMode) > 0 {
+			log.Warnf(deprecationWarning, "userns", fmt.Sprintf("--userns=%s", input.usernsMode))
+		}
+		if len(input.containerArchitecture) > 0 {
+			log.Warnf(deprecationWarning, "container-architecture", fmt.Sprintf("--platform=%s", input.containerArchitecture))
+		}
+		if len(input.containerCapAdd) > 0 {
+			log.Warnf(deprecationWarning, "container-cap-add", fmt.Sprintf("--cap-add=%s", input.containerCapAdd))
+		}
+		if len(input.containerCapDrop) > 0 {
+			log.Warnf(deprecationWarning, "container-cap-drop", fmt.Sprintf("--cap-drop=%s", input.containerCapDrop))
+		}
 
 		// run the plan
 		config := &runner.Config{
@@ -437,6 +454,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str
 			UsernsMode:                         input.usernsMode,
 			ContainerArchitecture:              input.containerArchitecture,
 			ContainerDaemonSocket:              input.containerDaemonSocket,
+			ContainerOptions:                   input.containerOptions,
 			UseGitIgnore:                       input.useGitIgnore,
 			GitHubInstance:                     input.githubInstance,
 			ContainerCapAdd:                    input.containerCapAdd,

pkg/container/docker_run.go

@@ -411,10 +411,16 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
 
 	logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig)
 
+	hostConfig.Binds = append(hostConfig.Binds, containerConfig.HostConfig.Binds...)
+	hostConfig.Mounts = append(hostConfig.Mounts, containerConfig.HostConfig.Mounts...)
+	binds := hostConfig.Binds
+	mounts := hostConfig.Mounts
 	err = mergo.Merge(hostConfig, containerConfig.HostConfig, mergo.WithOverride)
 	if err != nil {
 		return nil, nil, fmt.Errorf("Cannot merge container.HostConfig options: '%s': '%w'", input.Options, err)
 	}
+	hostConfig.Binds = binds
+	hostConfig.Mounts = mounts
 	logger.Debugf("Merged container.HostConfig ==> %+v", hostConfig)
 
 	return config, hostConfig, nil

pkg/model/workflow.go

@@ -495,7 +495,7 @@ func (s *Step) ShellCommand() string {
 	case "python":
 		shellCommand = "python {0}"
 	case "sh":
-		shellCommand = "sh -e -c {0}"
+		shellCommand = "sh -e {0}"
 	case "cmd":
 		shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\""
 	case "powershell":

pkg/runner/action.go

@@ -366,6 +366,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd []string
 		Privileged:  rc.Config.Privileged,
 		UsernsMode:  rc.Config.UsernsMode,
 		Platform:    rc.Config.ContainerArchitecture,
+		Options:     rc.Config.ContainerOptions,
 	})
 	return stepContainer
 }

pkg/runner/job_executor.go

@@ -95,21 +95,16 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
 	}
 
 	postExecutor = postExecutor.Finally(func(ctx context.Context) error {
-		logger := common.Logger(ctx)
 		jobError := common.JobError(ctx)
-		if jobError != nil {
-			info.result("failure")
-			logger.WithField("jobResult", "failure").Infof("\U0001F3C1  Job failed")
-		} else {
-			err := info.stopContainer()(ctx)
-			if err != nil {
-				return err
-			}
-			info.result("success")
-			logger.WithField("jobResult", "success").Infof("\U0001F3C1  Job succeeded")
+		var err error
+		if rc.Config.AutoRemove || jobError == nil {
+			// always allow 1 min for stopping and removing the runner, even if we were cancelled
+			ctx, cancel := context.WithTimeout(common.WithLogger(context.Background(), common.Logger(ctx)), time.Minute)
+			defer cancel()
+			err = info.stopContainer()(ctx)
 		}
-
-		return nil
+		setJobResult(ctx, info, rc, jobError == nil)
+		return err
 	})
 
 	pipeline := make([]common.Executor, 0)
@@ -122,7 +117,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
 			if ctx.Err() == context.Canceled {
 				// in case of an aborted run, we still should execute the
 				// post steps to allow cleanup.
-				ctx, cancel = context.WithTimeout(WithJobLogger(context.Background(), rc.Run.JobID, rc.String(), rc.Config, &rc.Masks, rc.Matrix), 5*time.Minute)
+				ctx, cancel = context.WithTimeout(common.WithLogger(context.Background(), common.Logger(ctx)), 5*time.Minute)
 				defer cancel()
 			}
 			return postExecutor(ctx)
@@ -131,6 +126,18 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
 		Finally(info.closeContainer()))
 }
 
+func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success bool) {
+	logger := common.Logger(ctx)
+	jobResult := "success"
+	jobResultMessage := "succeeded"
+	if !success {
+		jobResult = "failure"
+		jobResultMessage = "failed"
+	}
+	info.result(jobResult)
+	logger.WithField("jobResult", jobResult).Infof("\U0001F3C1  Job %s", jobResultMessage)
+}
+
 func useStepLogger(rc *RunContext, stepModel *model.Step, stage stepStage, executor common.Executor) common.Executor {
 	return func(ctx context.Context) error {
 		ctx = withStepLogger(ctx, stepModel.ID, rc.ExprEval.Interpolate(ctx, stepModel.String()), stage.String())

pkg/runner/run_context.go

@@ -225,6 +225,7 @@ func (rc *RunContext) startJobContainer() common.Executor {
 		envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_OS", "Linux"))
 		envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx)))
 		envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp"))
+		envList = append(envList, fmt.Sprintf("%s=%s", "LANG", "C.UTF-8")) // Use same locale as GitHub Actions
 
 		ext := container.LinuxContainerEnvironmentExtensions{}
 		binds, mounts := rc.GetBindsAndMounts()
@@ -409,7 +410,7 @@ func (rc *RunContext) options(ctx context.Context) string {
 	job := rc.Run.Job()
 	c := job.Container()
 	if c == nil {
-		return ""
+		return rc.Config.ContainerOptions
 	}
 
 	return c.Options

pkg/runner/runner.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"time"
 
 	log "github.com/sirupsen/logrus"
 
@@ -40,6 +39,7 @@ type Config struct {
 	UsernsMode                         string            // user namespace to use
 	ContainerArchitecture              string            // Desired OS/architecture platform for running containers
 	ContainerDaemonSocket              string            // Path to Docker daemon socket
+	ContainerOptions                   string            // Options for the job container
 	UseGitIgnore                       bool              // controls if paths in .gitignore should not be copied into container, default true
 	GitHubInstance                     string            // GitHub instance to use, default "github.com"
 	ContainerCapAdd                    []string          // list of kernel capabilities to add to the containers
@@ -77,18 +77,15 @@ func New(runnerConfig *Config) (Runner, error) {
 }
 
 // NewPlanExecutor ...
-//
-//nolint:gocyclo
 func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
 	maxJobNameLen := 0
 
 	stagePipeline := make([]common.Executor, 0)
 	for i := range plan.Stages {
-		s := i
 		stage := plan.Stages[i]
 		stagePipeline = append(stagePipeline, func(ctx context.Context) error {
 			pipeline := make([]common.Executor, 0)
-			for r, run := range stage.Runs {
+			for _, run := range stage.Runs {
 				stageExecutor := make([]common.Executor, 0)
 				job := run.Job()
 
@@ -123,29 +120,8 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
 						maxJobNameLen = len(rc.String())
 					}
 					stageExecutor = append(stageExecutor, func(ctx context.Context) error {
-						logger := common.Logger(ctx)
 						jobName := fmt.Sprintf("%-*s", maxJobNameLen, rc.String())
-						return rc.Executor().Finally(func(ctx context.Context) error {
-							isLastRunningContainer := func(currentStage int, currentRun int) bool {
-								return currentStage == len(plan.Stages)-1 && currentRun == len(stage.Runs)-1
-							}
-
-							if runner.config.AutoRemove && isLastRunningContainer(s, r) {
-								var cancel context.CancelFunc
-								if ctx.Err() == context.Canceled {
-									ctx, cancel = context.WithTimeout(context.Background(), 5*time.Minute)
-									defer cancel()
-								}
-
-								log.Infof("Cleaning up container for job %s", rc.JobName)
-
-								if err := rc.stopJobContainer()(ctx); err != nil {
-									logger.Errorf("Error while cleaning container: %v", err)
-								}
-							}
-
-							return nil
-						})(common.WithJobErrorContainer(WithJobLogger(ctx, rc.Run.JobID, jobName, rc.Config, &rc.Masks, matrix)))
+						return rc.Executor()(common.WithJobErrorContainer(WithJobLogger(ctx, rc.Run.JobID, jobName, rc.Config, &rc.Masks, matrix)))
 					})
 				}
 				pipeline = append(pipeline, common.NewParallelExecutor(maxParallel, stageExecutor...))