diff --git a/CHANGELOG.md b/CHANGELOG.md index bd9606c987..4cf7b40ff0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ All notable changes to `src-cli` are documented in this file. ### Added +- EXPERIMENTAL: Templated campaign specs and file mounting. The campaign specs evaluated by `src campaign [preview|apply]` can now include template variables in `steps.run`, `steps.env`, and the new `steps.files` property, which allows users to create files inside the container in which the step is executed. The feature is marked as EXPERIMENTAL because it might change in the near future until we deem it non-experimental. See [#361](https://github.com/sourcegraph/src-cli/pull/361) for details. + ### Changed ### Fixed diff --git a/README.md b/README.markdown similarity index 100% rename from README.md rename to README.markdown diff --git a/internal/campaigns/campaign_spec.go b/internal/campaigns/campaign_spec.go index 8fbb1d45e8..2ba4fff08d 100644 --- a/internal/campaigns/campaign_spec.go +++ b/internal/campaigns/campaign_spec.go @@ -66,6 +66,7 @@ type Step struct { Run string `json:"run,omitempty" yaml:"run"` Container string `json:"container,omitempty" yaml:"container"` Env map[string]string `json:"env,omitempty" yaml:"env"` + Files map[string]string `json:"files,omitempty" yaml:"files,omitempty"` image string } diff --git a/internal/campaigns/executor_test.go b/internal/campaigns/executor_test.go index 4f54240824..6cdd3d1782 100644 --- a/internal/campaigns/executor_test.go +++ b/internal/campaigns/executor_test.go @@ -98,6 +98,24 @@ func TestExecutor_Integration(t *testing.T) { executorTimeout: 100 * time.Millisecond, wantErrInclude: "execution in github.com/sourcegraph/src-cli failed: Timeout reached. Execution took longer than 100ms.", }, + { + name: "templated", + repos: []*graphql.Repository{srcCLIRepo}, + archives: []mockRepoArchive{ + {repo: srcCLIRepo, files: map[string]string{ + "README.md": "# Welcome to the README\n", + "main.go": "package main\n\nfunc main() {\n\tfmt.Println( \"Hello World\")\n}\n", + }}, + }, + steps: []Step{ + {Run: `go fmt main.go`, Container: "doesntmatter:13"}, + {Run: `touch modified-${{ join previous_step.modified_files " " }}.md`, Container: "alpine:13"}, + {Run: `touch added-${{ join previous_step.added_files " " }}`, Container: "alpine:13"}, + }, + wantFilesChanged: map[string][]string{ + srcCLIRepo.ID: []string{"main.go", "modified-main.go.md", "added-modified-main.go.md"}, + }, + }, } for _, tc := range tests { @@ -170,11 +188,16 @@ func TestExecutor_Integration(t *testing.T) { diffsByName := map[string]*diff.FileDiff{} for _, fd := range fileDiffs { - diffsByName[fd.OrigName] = fd + if fd.NewName == "/dev/null" { + diffsByName[fd.OrigName] = fd + } else { + diffsByName[fd.NewName] = fd + } } + for _, file := range wantFiles { if _, ok := diffsByName[file]; !ok { - t.Errorf("%s was not changed", file) + t.Errorf("%s was not changed (diffsByName=%#v)", file, diffsByName) } } } diff --git a/internal/campaigns/graphql/repository.go b/internal/campaigns/graphql/repository.go index 584f3f0e66..6147bae78b 100644 --- a/internal/campaigns/graphql/repository.go +++ b/internal/campaigns/graphql/repository.go @@ -1,6 +1,9 @@ package graphql -import "strings" +import ( + "sort" + "strings" +) const RepositoryFieldsFragment = ` fragment repositoryFields on Repository { @@ -30,6 +33,8 @@ type Repository struct { URL string ExternalRepository struct{ ServiceType string } DefaultBranch *Branch + + FileMatches map[string]bool } func (r *Repository) BaseRef() string { @@ -43,3 +48,16 @@ func (r *Repository) Rev() string { func (r *Repository) Slug() string { return strings.ReplaceAll(r.Name, "/", "-") } + +func (r *Repository) SearchResultPaths() (list fileMatchPathList) { + var files []string + for f := range r.FileMatches { + files = append(files, f) + } + sort.Strings(files) + return fileMatchPathList(files) +} + +type fileMatchPathList []string + +func (f fileMatchPathList) String() string { return strings.Join(f, " ") } diff --git a/internal/campaigns/run_steps.go b/internal/campaigns/run_steps.go index 42b844d9b7..211fef81e7 100644 --- a/internal/campaigns/run_steps.go +++ b/internal/campaigns/run_steps.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "strings" + "text/template" "time" "github.com/hashicorp/go-multierror" @@ -60,9 +61,15 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor return nil, errors.Wrap(err, "git commit failed") } + results := make([]StepResult, len(steps)) + for i, step := range steps { logger.Logf("[Step %d] docker run %s %q", i+1, step.Container, step.Run) - reportProgress(step.Run) + + stepContext := StepContext{Repository: *repo} + if i > 0 { + stepContext.PreviousStep = results[i-1] + } cidFile, err := ioutil.TempFile(tempDir, repo.Slug()+"-container-id") if err != nil { @@ -87,29 +94,81 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor // Set up a temporary file on the host filesystem to contain the // script. - fp, err := ioutil.TempFile(tempDir, "") + runScriptFile, err := ioutil.TempFile(tempDir, "") if err != nil { return nil, errors.Wrap(err, "creating temporary file") } - hostTemp := fp.Name() - defer os.Remove(hostTemp) - if _, err := fp.WriteString(step.Run); err != nil { - return nil, errors.Wrapf(err, "writing to temporary file %q", hostTemp) + defer os.Remove(runScriptFile.Name()) + + // Parse step.Run as a template... + tmpl, err := parseAsTemplate("step-run", step.Run, &stepContext) + if err != nil { + return nil, errors.Wrap(err, "parsing step run") + } + + // ... and render it into a buffer and the temp file we just created. + var runScript bytes.Buffer + if err := tmpl.Execute(io.MultiWriter(&runScript, runScriptFile), stepContext); err != nil { + return nil, errors.Wrap(err, "executing template") + } + if err := runScriptFile.Close(); err != nil { + return nil, errors.Wrap(err, "closing temporary file") + } + + // Parse and render the step.Files. + files, err := renderStepFiles(step.Files, &stepContext) + if err != nil { + return nil, errors.Wrap(err, "parsing step files") + } + + // Create temp files with the rendered content of step.Files so that we + // can mount them into the container. + filesToMount := make(map[string]*os.File, len(files)) + for name, content := range files { + fp, err := ioutil.TempFile(tempDir, "") + if err != nil { + return nil, errors.Wrap(err, "creating temporary file") + } + defer os.Remove(fp.Name()) + + if _, err := io.Copy(fp, content); err != nil { + return nil, errors.Wrap(err, "writing to temporary file") + } + + if err := fp.Close(); err != nil { + return nil, errors.Wrap(err, "closing temporary file") + } + + filesToMount[name] = fp + } + + // Render the step.Env variables as templates. + env, err := renderStepEnv(step.Env, &stepContext) + if err != nil { + return nil, errors.Wrap(err, "parsing step files") } - fp.Close() + reportProgress(runScript.String()) const workDir = "/work" - cmd := exec.CommandContext(ctx, "docker", "run", + args := []string{ + "run", "--rm", "--cidfile", cidFile.Name(), "--workdir", workDir, "--mount", fmt.Sprintf("type=bind,source=%s,target=%s", volumeDir, workDir), - "--mount", fmt.Sprintf("type=bind,source=%s,target=%s,ro", hostTemp, containerTemp), - "--entrypoint", shell, - ) - for k, v := range step.Env { - cmd.Args = append(cmd.Args, "-e", k+"="+v) + "--mount", fmt.Sprintf("type=bind,source=%s,target=%s,ro", runScriptFile.Name(), containerTemp), + } + for target, source := range filesToMount { + args = append(args, "--mount", fmt.Sprintf("type=bind,source=%s,target=%s,ro", source.Name(), target)) } + + for k, v := range env { + args = append(args, "-e", k+"="+v) + } + + args = append(args, "--entrypoint", shell) + + cmd := exec.CommandContext(ctx, "docker", args...) cmd.Args = append(cmd.Args, "--", step.image, containerTemp) cmd.Dir = volumeDir @@ -132,7 +191,7 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor return nil, stepFailedErr{ Err: err, Args: cmd.Args, - Run: step.Run, + Run: runScript.String(), Container: step.Container, TmpFilename: containerTemp, Stdout: strings.TrimSpace(stdoutBuffer.String()), @@ -141,10 +200,22 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor } logger.Logf("[Step %d] complete in %s", i+1, elapsed) - } - if _, err := runGitCmd("add", "--all"); err != nil { - return nil, errors.Wrap(err, "git add failed") + if _, err := runGitCmd("add", "--all"); err != nil { + return nil, errors.Wrap(err, "git add failed") + } + + statusOut, err := runGitCmd("status", "--porcelain") + if err != nil { + return nil, errors.Wrap(err, "git status failed") + } + + changes, err := parseGitStatus(statusOut) + if err != nil { + return nil, errors.Wrap(err, "parsing git status output") + } + + results[i] = StepResult{Files: changes, Stdout: &stdoutBuffer, Stderr: &stderrBuffer} } reportProgress("Calculating diff") @@ -271,3 +342,166 @@ func (e stepFailedErr) SingleLineError() string { return strings.Split(out, "\n")[0] } + +func parseAsTemplate(name, input string, stepCtx *StepContext) (*template.Template, error) { + return template.New(name).Delims("${{", "}}").Funcs(stepCtx.ToFuncMap()).Parse(input) +} + +func renderStepFiles(files map[string]string, stepCtx *StepContext) (map[string]io.Reader, error) { + containerFiles := make(map[string]io.Reader, len(files)) + + for fileName, fileRaw := range files { + // We treat the file contents as a template and render it + // into a buffer that we then mount into the code host. + var out bytes.Buffer + + tmpl, err := parseAsTemplate(fileName, fileRaw, stepCtx) + if err != nil { + return containerFiles, err + } + + if err := tmpl.Execute(&out, stepCtx); err != nil { + return containerFiles, err + } + + containerFiles[fileName] = &out + } + + return containerFiles, nil +} + +func renderStepEnv(env map[string]string, stepCtx *StepContext) (map[string]string, error) { + parsedEnv := make(map[string]string, len(env)) + + fnMap := stepCtx.ToFuncMap() + + for k, v := range env { + // We treat the file contents as a template and render it + // into a buffer that we then mount into the code host. + var out bytes.Buffer + + tmpl, err := template.New(k).Delims("${{", "}}").Funcs(fnMap).Parse(v) + if err != nil { + return parsedEnv, err + } + + if err := tmpl.Execute(&out, stepCtx); err != nil { + return parsedEnv, err + } + + parsedEnv[k] = out.String() + } + + return parsedEnv, nil +} + +// StepContext represents the contextual information available when executing a +// step that's defined in a campaign spec. +type StepContext struct { + PreviousStep StepResult + Repository graphql.Repository +} + +// ToFuncMap returns a template.FuncMap to access fields on the StepContext in a +// text/template. +func (stepCtx *StepContext) ToFuncMap() template.FuncMap { + return template.FuncMap{ + "join": func(list []string, sep string) string { + return strings.Join(list, sep) + }, + "split": func(s string, sep string) []string { + return strings.Split(s, sep) + }, + "previous_step": func() map[string]interface{} { + result := map[string]interface{}{ + "modified_files": stepCtx.PreviousStep.ModifiedFiles(), + "added_files": stepCtx.PreviousStep.AddedFiles(), + "deleted_files": stepCtx.PreviousStep.DeletedFiles(), + "renamed_files": stepCtx.PreviousStep.RenamedFiles(), + } + + if stepCtx.PreviousStep.Stdout != nil { + result["stdout"] = stepCtx.PreviousStep.Stdout.String() + } else { + result["stdout"] = "" + } + + if stepCtx.PreviousStep.Stderr != nil { + result["stderr"] = stepCtx.PreviousStep.Stderr.String() + } else { + result["stderr"] = "" + } + + return result + }, + "repository": func() map[string]interface{} { + return map[string]interface{}{ + "search_result_paths": stepCtx.Repository.SearchResultPaths(), + "name": stepCtx.Repository.Name, + } + }, + } +} + +// StepResult represents the result of a previously executed step. +type StepResult struct { + // Files are the changes made to files by the step. + Files StepChanges + + // Stdout is the output produced by the step on standard out. + Stdout *bytes.Buffer + // Stderr is the output produced by the step on standard error. + Stderr *bytes.Buffer +} + +// StepChanges are the changes made to files by a previous step in a repository. +type StepChanges struct { + Modified []string + Added []string + Deleted []string + Renamed []string +} + +// ModifiedFiles returns the files modified by a step. +func (r StepResult) ModifiedFiles() []string { return r.Files.Modified } + +// AddedFiles returns the files added by a step. +func (r StepResult) AddedFiles() []string { return r.Files.Added } + +// DeletedFiles returns the files deleted by a step. +func (r StepResult) DeletedFiles() []string { return r.Files.Deleted } + +// RenamedFiles returns the new name of files that have been renamed by a step. +func (r StepResult) RenamedFiles() []string { return r.Files.Renamed } + +func parseGitStatus(out []byte) (StepChanges, error) { + result := StepChanges{} + + stripped := strings.TrimSpace(string(out)) + if len(stripped) == 0 { + return result, nil + } + + for _, line := range strings.Split(stripped, "\n") { + if len(line) < 4 { + return result, fmt.Errorf("git status line has unrecognized format: %q", line) + } + + file := line[3:] + + switch line[0] { + case 'M': + result.Modified = append(result.Modified, file) + case 'A': + result.Added = append(result.Added, file) + case 'D': + result.Deleted = append(result.Deleted, file) + case 'R': + files := strings.Split(file, " -> ") + newFile := files[len(files)-1] + result.Renamed = append(result.Renamed, newFile) + } + } + + return result, nil +} diff --git a/internal/campaigns/run_steps_test.go b/internal/campaigns/run_steps_test.go new file mode 100644 index 0000000000..8d4db8415a --- /dev/null +++ b/internal/campaigns/run_steps_test.go @@ -0,0 +1,180 @@ +package campaigns + +import ( + "bytes" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/sourcegraph/src-cli/internal/campaigns/graphql" +) + +func TestParseGitStatus(t *testing.T) { + const input = `M README.md +M another_file.go +A new_file.txt +A barfoo/new_file.txt +D to_be_deleted.txt +R README.md -> README.markdown +` + parsed, err := parseGitStatus([]byte(input)) + if err != nil { + t.Fatal(err) + } + + want := StepChanges{ + Modified: []string{"README.md", "another_file.go"}, + Added: []string{"new_file.txt", "barfoo/new_file.txt"}, + Deleted: []string{"to_be_deleted.txt"}, + Renamed: []string{"README.markdown"}, + } + + if !cmp.Equal(want, parsed) { + t.Fatalf("wrong output:\n%s", cmp.Diff(want, parsed)) + } +} + +func TestParsingAndRenderingTemplates(t *testing.T) { + stepCtx := &StepContext{ + PreviousStep: StepResult{ + Files: StepChanges{ + Modified: []string{"go.mod"}, + Added: []string{"main.go.swp"}, + Deleted: []string{".DS_Store"}, + Renamed: []string{"new-filename.txt"}, + }, + Stdout: bytes.NewBufferString("this is stdout"), + Stderr: bytes.NewBufferString("this is stderr"), + }, + Repository: graphql.Repository{ + Name: "github.com/sourcegraph/src-cli", + FileMatches: map[string]bool{ + "README.md": true, + "main.go": true, + }, + }, + } + + tests := []struct { + name string + stepCtx *StepContext + run string + want string + }{ + { + name: "previous step file changes", + stepCtx: stepCtx, + run: `${{ .PreviousStep.ModifiedFiles }} +${{ .PreviousStep.AddedFiles }} +${{ .PreviousStep.DeletedFiles }} +${{ .PreviousStep.RenamedFiles }} +`, + want: `[go.mod] +[main.go.swp] +[.DS_Store] +[new-filename.txt] +`, + }, + { + name: "previous step output", + stepCtx: stepCtx, + run: `${{ .PreviousStep.Stdout }} ${{ .PreviousStep.Stderr }}`, + want: `this is stdout this is stderr`, + }, + { + name: "repository name", + stepCtx: stepCtx, + run: `${{ .Repository.Name }}`, + want: `github.com/sourcegraph/src-cli`, + }, + { + name: "search result paths", + stepCtx: stepCtx, + run: `${{ .Repository.SearchResultPaths }}`, + want: `README.md main.go`, + }, + { + name: "lower-case aliases", + stepCtx: stepCtx, + run: `${{ repository.search_result_paths }} + ${{ repository.name }} + ${{ previous_step.modified_files }} + ${{ previous_step.added_files }} + ${{ previous_step.deleted_files }} + ${{ previous_step.renamed_files }} + ${{ previous_step.stdout }} + ${{ previous_step.stderr}} + `, + want: `README.md main.go + github.com/sourcegraph/src-cli + [go.mod] + [main.go.swp] + [.DS_Store] + [new-filename.txt] + this is stdout + this is stderr + `, + }, + { + name: "empty context", + stepCtx: &StepContext{}, + run: `${{ .Repository.SearchResultPaths }} +${{ .Repository.Name }} +${{ previous_step.modified_files }} +${{ previous_step.added_files }} +${{ previous_step.deleted_files }} +${{ previous_step.renamed_files }} +${{ previous_step.stdout }} +${{ previous_step.stderr}} +`, + want: ` + +[] +[] +[] +[] + + +`, + }, + { + name: "empty context and aliases", + stepCtx: &StepContext{}, + run: `${{ repository.search_result_paths }} +${{ repository.name }} +${{ previous_step.modified_files }} +${{ previous_step.added_files }} +${{ previous_step.deleted_files }} +${{ previous_step.renamed_files }} +${{ previous_step.stdout }} +${{ previous_step.stderr}} +`, + want: ` + +[] +[] +[] +[] + + +`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + parsed, err := parseAsTemplate("testing", tc.run, tc.stepCtx) + if err != nil { + t.Fatal(err) + } + + var out bytes.Buffer + if err := parsed.Execute(&out, tc.stepCtx); err != nil { + t.Fatalf("executing template failed: %s", err) + } + + if out.String() != tc.want { + t.Fatalf("wrong output:\n%s", cmp.Diff(tc.want, out.String())) + } + }) + } +} diff --git a/internal/campaigns/service.go b/internal/campaigns/service.go index a55a1053d3..1a803f5f49 100644 --- a/internal/campaigns/service.go +++ b/internal/campaigns/service.go @@ -430,6 +430,7 @@ query ChangesetRepos( ...repositoryFields } ... on FileMatch { + file { path } repository { ...repositoryFields } @@ -454,13 +455,18 @@ func (svc *Service) resolveRepositorySearch(ctx context.Context, query string) ( return nil, err } - ids := map[string]struct{}{} + ids := map[string]*graphql.Repository{} var repos []*graphql.Repository for _, r := range result.Search.Results.Results { - if _, ok := ids[r.ID]; !ok { + existing, ok := ids[r.ID] + if !ok { repo := r.Repository repos = append(repos, &repo) - ids[r.ID] = struct{}{} + ids[r.ID] = &repo + } else { + for file := range r.FileMatches { + existing.FileMatches[file] = true + } } } return repos, nil @@ -492,12 +498,18 @@ func (sr *searchResult) UnmarshalJSON(data []byte) error { switch tn.Typename { case "FileMatch": - var result struct{ Repository graphql.Repository } + var result struct { + Repository graphql.Repository + File struct { + Path string + } + } if err := json.Unmarshal(data, &result); err != nil { return err } sr.Repository = result.Repository + sr.Repository.FileMatches = map[string]bool{result.File.Path: true} return nil case "Repository": diff --git a/schema/campaign_spec.schema.json b/schema/campaign_spec.schema.json index ac922eed16..985af68a43 100644 --- a/schema/campaign_spec.schema.json +++ b/schema/campaign_spec.schema.json @@ -81,6 +81,11 @@ "additionalProperties": { "type": "string" } + }, + "files": { + "type": "object", + "description": "Files that should be mounted into or be created inside the Docker container.", + "additionalProperties": {"type": "string"} } } } diff --git a/schema/campaign_spec_stringdata.go b/schema/campaign_spec_stringdata.go index 93455e06e3..e4e05dc641 100644 --- a/schema/campaign_spec_stringdata.go +++ b/schema/campaign_spec_stringdata.go @@ -86,6 +86,11 @@ const CampaignSpecJSON = `{ "additionalProperties": { "type": "string" } + }, + "files": { + "type": "object", + "description": "Files that should be mounted into or be created inside the Docker container.", + "additionalProperties": {"type": "string"} } } }