Skip to content

Move action execution related code to campaigns package #230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 24, 2020
Merged

Move action execution related code to campaigns package #230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 24 additions & 161 deletions cmd/src/actions_exec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package main

import (
"bufio"
"bytes"
"context"
"encoding/json"
"flag"
Expand All @@ -19,43 +18,11 @@ import (
"time"

"github.com/fatih/color"
"github.com/hashicorp/go-multierror"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/sourcegraph/src-cli/schema"
"github.com/xeipuuv/gojsonschema"
"github.com/sourcegraph/src-cli/internal/campaigns"
)

type Action struct {
ScopeQuery string `json:"scopeQuery,omitempty"`
Steps []*ActionStep `json:"steps"`
}

type ActionStep struct {
Type string `json:"type"` // "command"
Image string `json:"image,omitempty"` // Docker image
CacheDirs []string `json:"cacheDirs,omitempty"`
Args []string `json:"args,omitempty"`

// ImageContentDigest is an internal field that should not be set by users.
ImageContentDigest string
}

type PatchInput struct {
Repository string `json:"repository"`
BaseRevision string `json:"baseRevision"`
BaseRef string `json:"baseRef"`
Patch string `json:"patch"`
}

func userCacheDir() (string, error) {
userCacheDir, err := os.UserCacheDir()
if err != nil {
return "", err
}
return filepath.Join(userCacheDir, "sourcegraph-src"), nil
}

const defaultTimeout = 60 * time.Minute

func init() {
Expand Down Expand Up @@ -132,7 +99,7 @@ Format of the action JSON files:
fmt.Println(usage)
}

cacheDir, _ := userCacheDir()
cacheDir, _ := campaigns.UserCacheDir()
if cacheDir != "" {
cacheDir = filepath.Join(cacheDir, "action-exec")
}
Expand Down Expand Up @@ -211,12 +178,12 @@ Format of the action JSON files:
}
}

err = validateActionDefinition(actionFile)
err = campaigns.ValidateActionDefinition(actionFile)
if err != nil {
return err
}

var action Action
var action campaigns.Action
if err := jsonxUnmarshal(string(actionFile), &action); err != nil {
return errors.Wrap(err, "invalid JSON action file")
}
Expand All @@ -238,19 +205,21 @@ Format of the action JSON files:
os.Exit(2)
}()

logger := newActionLogger(*verbose, *keepLogsFlag)
logger := campaigns.NewActionLogger(*verbose, *keepLogsFlag)

// Fetch Docker images etc.
err = prepareAction(ctx, action, logger)
err = campaigns.PrepareAction(ctx, action, logger)
if err != nil {
return errors.Wrap(err, "Failed to prepare action")
}

opts := actionExecutorOptions{
timeout: *timeoutFlag,
keepLogs: *keepLogsFlag,
clearCache: *clearCacheFlag,
cache: actionExecutionDiskCache{dir: *cacheDirFlag},
opts := campaigns.ExecutorOpts{
Endpoint: cfg.Endpoint,
AccessToken: cfg.AccessToken,
Timeout: *timeoutFlag,
KeepLogs: *keepLogsFlag,
ClearCache: *clearCacheFlag,
Cache: campaigns.ExecutionDiskCache{Dir: *cacheDirFlag},
}

// Query repos over which to run action
Expand All @@ -264,15 +233,15 @@ Format of the action JSON files:
totalSteps := len(repos) * len(action.Steps)
logger.Start(totalSteps)

executor := newActionExecutor(action, *parallelismFlag, logger, opts)
executor := campaigns.NewExecutor(action, *parallelismFlag, logger, opts)
for _, repo := range repos {
executor.enqueueRepo(repo)
executor.EnqueueRepo(repo)
}

go executor.start(ctx)
err = executor.wait()
go executor.Start(ctx)
err = executor.Wait()

patches := executor.allPatches()
patches := executor.AllPatches()
if len(patches) == 0 {
// We call os.Exit because we don't want to return the error
// and have it printed.
Expand Down Expand Up @@ -344,115 +313,7 @@ Format of the action JSON files:
})
}

func formatValidationErrs(es []error) string {
points := make([]string, len(es))
for i, err := range es {
points[i] = fmt.Sprintf("- %s", err)
}

return fmt.Sprintf(
"Validating action definition failed:\n%s\n",
strings.Join(points, "\n"))
}

func validateActionDefinition(def []byte) error {
sl := gojsonschema.NewSchemaLoader()
sc, err := sl.Compile(gojsonschema.NewStringLoader(schema.ActionSchemaJSON))
if err != nil {
return errors.Wrapf(err, "failed to compile actions schema")
}

normalized, err := jsonxToJSON(string(def))
if err != nil {
return err
}

res, err := sc.Validate(gojsonschema.NewBytesLoader(normalized))
if err != nil {
return errors.Wrap(err, "failed to validate config against schema")
}

errs := &multierror.Error{ErrorFormat: formatValidationErrs}
for _, err := range res.Errors() {
e := err.String()
// Remove `(root): ` from error formatting since these errors are
// presented to users.
e = strings.TrimPrefix(e, "(root): ")
errs = multierror.Append(errs, errors.New(e))
}

return errs.ErrorOrNil()
}

func prepareAction(ctx context.Context, action Action, logger *actionLogger) error {
// Build any Docker images.
for _, step := range action.Steps {
if step.Type == "docker" {
// Set digests for Docker images so we don't cache action runs in 2 different images with
// the same tag.
var err error
step.ImageContentDigest, err = getDockerImageContentDigest(ctx, step.Image, logger)
if err != nil {
return errors.Wrap(err, "Failed to get Docker image content digest")
}
}
}

return nil
}

// getDockerImageContentDigest gets the content digest for the image. Note that this
// is different from the "distribution digest" (which is what you can use to specify
// an image to `docker run`, as in `my/image@sha256:xxx`). We need to use the
// content digest because the distribution digest is only computed for images that
// have been pulled from or pushed to a registry. See
// https://windsock.io/explaining-docker-image-ids/ under "A Final Twist" for a good
// explanation.
func getDockerImageContentDigest(ctx context.Context, image string, logger *actionLogger) (string, error) {
// TODO!(sqs): is image id the right thing to use here? it is NOT the
// digest. but the digest is not calculated for all images (unless they are
// pulled/pushed from/to a registry), see
// https://github.com/moby/moby/issues/32016.
out, err := exec.CommandContext(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", "--", image).CombinedOutput()
if err != nil {
if !strings.Contains(string(out), "No such image") {
return "", fmt.Errorf("error inspecting docker image %q: %s", image, bytes.TrimSpace(out))
}
logger.Infof("Pulling Docker image %q...\n", image)
pullCmd := exec.CommandContext(ctx, "docker", "image", "pull", image)
prefix := fmt.Sprintf("docker image pull %s", image)
pullCmd.Stdout = logger.InfoPipe(prefix)
pullCmd.Stderr = logger.ErrorPipe(prefix)

err = pullCmd.Start()
if err != nil {
return "", fmt.Errorf("error pulling docker image %q: %s", image, err)
}
err = pullCmd.Wait()
if err != nil {
return "", fmt.Errorf("error pulling docker image %q: %s", image, err)
}
}
out, err = exec.CommandContext(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", "--", image).CombinedOutput()
// This time, the image MUST be present, so the issue must be something else.
if err != nil {
return "", fmt.Errorf("error inspecting docker image %q: %s", image, bytes.TrimSpace(out))
}
id := string(bytes.TrimSpace(out))
if id == "" {
return "", fmt.Errorf("unexpected empty docker image content ID for %q", image)
}
return id, nil
}

type ActionRepo struct {
ID string
Name string
Rev string
BaseRef string
}

func actionRepos(ctx context.Context, scopeQuery string, includeUnsupported bool, logger *actionLogger) ([]ActionRepo, error) {
func actionRepos(ctx context.Context, scopeQuery string, includeUnsupported bool, logger *campaigns.ActionLogger) ([]campaigns.ActionRepo, error) {
hasCount, err := regexp.MatchString(`count:\d+`, scopeQuery)
if err != nil {
return nil, err
Expand Down Expand Up @@ -563,7 +424,7 @@ fragment repositoryFields on Repository {

skipped := []string{}
unsupported := []string{}
reposByID := map[string]ActionRepo{}
reposByID := map[string]campaigns.ActionRepo{}
for _, searchResult := range result.Data.Search.Results.Results {

var repo Repository
Expand Down Expand Up @@ -595,7 +456,7 @@ fragment repositoryFields on Repository {
}

if _, ok := reposByID[repo.ID]; !ok {
reposByID[repo.ID] = ActionRepo{
reposByID[repo.ID] = campaigns.ActionRepo{
ID: repo.ID,
Name: repo.Name,
Rev: repo.DefaultBranch.Target.OID,
Expand All @@ -604,7 +465,7 @@ fragment repositoryFields on Repository {
}
}

repos := make([]ActionRepo, 0, len(reposByID))
repos := make([]campaigns.ActionRepo, 0, len(reposByID))
for _, repo := range reposByID {
repos = append(repos, repo)
}
Expand Down Expand Up @@ -640,6 +501,8 @@ fragment repositoryFields on Repository {
return repos, nil
}

var yellow = color.New(color.FgYellow)

func isGitAvailable() bool {
cmd := exec.Command("git", "version")
if err := cmd.Run(); err != nil {
Expand Down
117 changes: 0 additions & 117 deletions cmd/src/actions_exec_backend.go
View file
Open in desktop

This file was deleted.

Loading