Move action execution related code to campaigns package

Author: mrnugget

cmd/src/actions_exec.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bufio"
-	"bytes"
 	"context"
 	"encoding/json"
 	"flag"
@@ -19,43 +18,11 @@ import (
 	"time"
 
 	"github.com/fatih/color"
-	"github.com/hashicorp/go-multierror"
 	"github.com/mattn/go-isatty"
 	"github.com/pkg/errors"
-	"github.com/sourcegraph/src-cli/schema"
-	"github.com/xeipuuv/gojsonschema"
+	"github.com/sourcegraph/src-cli/internal/campaigns"
 )
 
-type Action struct {
-	ScopeQuery string        `json:"scopeQuery,omitempty"`
-	Steps      []*ActionStep `json:"steps"`
-}
-
-type ActionStep struct {
-	Type      string   `json:"type"`            // "command"
-	Image     string   `json:"image,omitempty"` // Docker image
-	CacheDirs []string `json:"cacheDirs,omitempty"`
-	Args      []string `json:"args,omitempty"`
-
-	// ImageContentDigest is an internal field that should not be set by users.
-	ImageContentDigest string
-}
-
-type PatchInput struct {
-	Repository   string `json:"repository"`
-	BaseRevision string `json:"baseRevision"`
-	BaseRef      string `json:"baseRef"`
-	Patch        string `json:"patch"`
-}
-
-func userCacheDir() (string, error) {
-	userCacheDir, err := os.UserCacheDir()
-	if err != nil {
-		return "", err
-	}
-	return filepath.Join(userCacheDir, "sourcegraph-src"), nil
-}
-
 const defaultTimeout = 60 * time.Minute
 
 func init() {
@@ -132,7 +99,7 @@ Format of the action JSON files:
 		fmt.Println(usage)
 	}
 
-	cacheDir, _ := userCacheDir()
+	cacheDir, _ := campaigns.UserCacheDir()
 	if cacheDir != "" {
 		cacheDir = filepath.Join(cacheDir, "action-exec")
 	}
@@ -211,12 +178,12 @@ Format of the action JSON files:
 			}
 		}
 
-		err = validateActionDefinition(actionFile)
+		err = campaigns.ValidateActionDefinition(actionFile)
 		if err != nil {
 			return err
 		}
 
-		var action Action
+		var action campaigns.Action
 		if err := jsonxUnmarshal(string(actionFile), &action); err != nil {
 			return errors.Wrap(err, "invalid JSON action file")
 		}
@@ -238,19 +205,21 @@ Format of the action JSON files:
 			os.Exit(2)
 		}()
 
-		logger := newActionLogger(*verbose, *keepLogsFlag)
+		logger := campaigns.NewActionLogger(*verbose, *keepLogsFlag)
 
 		// Fetch Docker images etc.
-		err = prepareAction(ctx, action, logger)
+		err = campaigns.PrepareAction(ctx, action, logger)
 		if err != nil {
 			return errors.Wrap(err, "Failed to prepare action")
 		}
 
-		opts := actionExecutorOptions{
-			timeout:    *timeoutFlag,
-			keepLogs:   *keepLogsFlag,
-			clearCache: *clearCacheFlag,
-			cache:      actionExecutionDiskCache{dir: *cacheDirFlag},
+		opts := campaigns.ExecutorOpts{
+			Endpoint:    cfg.Endpoint,
+			AccessToken: cfg.AccessToken,
+			Timeout:     *timeoutFlag,
+			KeepLogs:    *keepLogsFlag,
+			ClearCache:  *clearCacheFlag,
+			Cache:       campaigns.ExecutionDiskCache{Dir: *cacheDirFlag},
 		}
 
 		// Query repos over which to run action
@@ -264,15 +233,15 @@ Format of the action JSON files:
 		totalSteps := len(repos) * len(action.Steps)
 		logger.Start(totalSteps)
 
-		executor := newActionExecutor(action, *parallelismFlag, logger, opts)
+		executor := campaigns.NewExecutor(action, *parallelismFlag, logger, opts)
 		for _, repo := range repos {
-			executor.enqueueRepo(repo)
+			executor.EnqueueRepo(repo)
 		}
 
-		go executor.start(ctx)
-		err = executor.wait()
+		go executor.Start(ctx)
+		err = executor.Wait()
 
-		patches := executor.allPatches()
+		patches := executor.AllPatches()
 		if len(patches) == 0 {
 			// We call os.Exit because we don't want to return the error
 			// and have it printed.
@@ -344,115 +313,7 @@ Format of the action JSON files:
 	})
 }
 
-func formatValidationErrs(es []error) string {
-	points := make([]string, len(es))
-	for i, err := range es {
-		points[i] = fmt.Sprintf("- %s", err)
-	}
-
-	return fmt.Sprintf(
-		"Validating action definition failed:\n%s\n",
-		strings.Join(points, "\n"))
-}
-
-func validateActionDefinition(def []byte) error {
-	sl := gojsonschema.NewSchemaLoader()
-	sc, err := sl.Compile(gojsonschema.NewStringLoader(schema.ActionSchemaJSON))
-	if err != nil {
-		return errors.Wrapf(err, "failed to compile actions schema")
-	}
-
-	normalized, err := jsonxToJSON(string(def))
-	if err != nil {
-		return err
-	}
-
-	res, err := sc.Validate(gojsonschema.NewBytesLoader(normalized))
-	if err != nil {
-		return errors.Wrap(err, "failed to validate config against schema")
-	}
-
-	errs := &multierror.Error{ErrorFormat: formatValidationErrs}
-	for _, err := range res.Errors() {
-		e := err.String()
-		// Remove `(root): ` from error formatting since these errors are
-		// presented to users.
-		e = strings.TrimPrefix(e, "(root): ")
-		errs = multierror.Append(errs, errors.New(e))
-	}
-
-	return errs.ErrorOrNil()
-}
-
-func prepareAction(ctx context.Context, action Action, logger *actionLogger) error {
-	// Build any Docker images.
-	for _, step := range action.Steps {
-		if step.Type == "docker" {
-			// Set digests for Docker images so we don't cache action runs in 2 different images with
-			// the same tag.
-			var err error
-			step.ImageContentDigest, err = getDockerImageContentDigest(ctx, step.Image, logger)
-			if err != nil {
-				return errors.Wrap(err, "Failed to get Docker image content digest")
-			}
-		}
-	}
-
-	return nil
-}
-
-// getDockerImageContentDigest gets the content digest for the image. Note that this
-// is different from the "distribution digest" (which is what you can use to specify
-// an image to `docker run`, as in `my/image@sha256:xxx`). We need to use the
-// content digest because the distribution digest is only computed for images that
-// have been pulled from or pushed to a registry. See
-// https://windsock.io/explaining-docker-image-ids/ under "A Final Twist" for a good
-// explanation.
-func getDockerImageContentDigest(ctx context.Context, image string, logger *actionLogger) (string, error) {
-	// TODO!(sqs): is image id the right thing to use here? it is NOT the
-	// digest. but the digest is not calculated for all images (unless they are
-	// pulled/pushed from/to a registry), see
-	// https://github.com/moby/moby/issues/32016.
-	out, err := exec.CommandContext(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", "--", image).CombinedOutput()
-	if err != nil {
-		if !strings.Contains(string(out), "No such image") {
-			return "", fmt.Errorf("error inspecting docker image %q: %s", image, bytes.TrimSpace(out))
-		}
-		logger.Infof("Pulling Docker image %q...\n", image)
-		pullCmd := exec.CommandContext(ctx, "docker", "image", "pull", image)
-		prefix := fmt.Sprintf("docker image pull %s", image)
-		pullCmd.Stdout = logger.InfoPipe(prefix)
-		pullCmd.Stderr = logger.ErrorPipe(prefix)
-
-		err = pullCmd.Start()
-		if err != nil {
-			return "", fmt.Errorf("error pulling docker image %q: %s", image, err)
-		}
-		err = pullCmd.Wait()
-		if err != nil {
-			return "", fmt.Errorf("error pulling docker image %q: %s", image, err)
-		}
-	}
-	out, err = exec.CommandContext(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", "--", image).CombinedOutput()
-	// This time, the image MUST be present, so the issue must be something else.
-	if err != nil {
-		return "", fmt.Errorf("error inspecting docker image %q: %s", image, bytes.TrimSpace(out))
-	}
-	id := string(bytes.TrimSpace(out))
-	if id == "" {
-		return "", fmt.Errorf("unexpected empty docker image content ID for %q", image)
-	}
-	return id, nil
-}
-
-type ActionRepo struct {
-	ID      string
-	Name    string
-	Rev     string
-	BaseRef string
-}
-
-func actionRepos(ctx context.Context, scopeQuery string, includeUnsupported bool, logger *actionLogger) ([]ActionRepo, error) {
+func actionRepos(ctx context.Context, scopeQuery string, includeUnsupported bool, logger *campaigns.ActionLogger) ([]campaigns.ActionRepo, error) {
 	hasCount, err := regexp.MatchString(`count:\d+`, scopeQuery)
 	if err != nil {
 		return nil, err
@@ -563,7 +424,7 @@ fragment repositoryFields on Repository {
 
 	skipped := []string{}
 	unsupported := []string{}
-	reposByID := map[string]ActionRepo{}
+	reposByID := map[string]campaigns.ActionRepo{}
 	for _, searchResult := range result.Data.Search.Results.Results {
 
 		var repo Repository
@@ -595,7 +456,7 @@ fragment repositoryFields on Repository {
 		}
 
 		if _, ok := reposByID[repo.ID]; !ok {
-			reposByID[repo.ID] = ActionRepo{
+			reposByID[repo.ID] = campaigns.ActionRepo{
 				ID:      repo.ID,
 				Name:    repo.Name,
 				Rev:     repo.DefaultBranch.Target.OID,
@@ -604,7 +465,7 @@ fragment repositoryFields on Repository {
 		}
 	}
 
-	repos := make([]ActionRepo, 0, len(reposByID))
+	repos := make([]campaigns.ActionRepo, 0, len(reposByID))
 	for _, repo := range reposByID {
 		repos = append(repos, repo)
 	}
@@ -640,6 +501,8 @@ fragment repositoryFields on Repository {
 	return repos, nil
 }
 
+var yellow = color.New(color.FgYellow)
+
 func isGitAvailable() bool {
 	cmd := exec.Command("git", "version")
 	if err := cmd.Run(); err != nil {