From 41f8104d273a01d43c557b0f91dd0d4aba945927 Mon Sep 17 00:00:00 2001 From: Erik Seliger Date: Wed, 1 Sep 2021 12:23:47 +0200 Subject: [PATCH] Move template pkg to sg/sg/lib --- go.mod | 2 +- go.sum | 4 +- internal/batches/executor/changeset_specs.go | 2 +- .../batches/executor/changeset_specs_test.go | 2 +- internal/batches/executor/coordinator_test.go | 2 +- internal/batches/executor/executor_test.go | 2 +- internal/batches/executor/run_steps.go | 2 +- internal/batches/executor/task.go | 2 +- internal/batches/service/build_tasks.go | 2 +- internal/batches/template/main_test.go | 11 - internal/batches/template/partial_eval.go | 383 ------------------ .../batches/template/partial_eval_test.go | 294 -------------- internal/batches/template/templating.go | 311 -------------- internal/batches/template/templating_test.go | 361 ----------------- internal/batches/util/repo.go | 2 +- 15 files changed, 11 insertions(+), 1371 deletions(-) delete mode 100644 internal/batches/template/main_test.go delete mode 100644 internal/batches/template/partial_eval.go delete mode 100644 internal/batches/template/partial_eval_test.go delete mode 100644 internal/batches/template/templating.go delete mode 100644 internal/batches/template/templating_test.go diff --git a/go.mod b/go.mod index 86517eed9e..27fd864927 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/sourcegraph/batch-change-utils v0.0.0-20210708162152-c9f35b905d94 github.com/sourcegraph/go-diff v0.6.1 github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf - github.com/sourcegraph/sourcegraph/lib v0.0.0-20210901132614-5780c9c4b466 + github.com/sourcegraph/sourcegraph/lib v0.0.0-20210901142905-cc5a3f7fd279 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e diff --git a/go.sum b/go.sum index 735b720c57..4b8795e208 100644 --- a/go.sum +++ b/go.sum @@ -263,8 +263,8 @@ github.com/sourcegraph/go-diff v0.6.1 h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0H github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf h1:oAdWFqhStsWiiMP/vkkHiMXqFXzl1XfUNOdxKJbd6bI= github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf/go.mod h1:ppFaPm6kpcHnZGqQTFhUIAQRIEhdQDWP1PCv4/ON354= -github.com/sourcegraph/sourcegraph/lib v0.0.0-20210901132614-5780c9c4b466 h1:hyo3Vr63aieMyWqrfy0skzZHT3ACjajrd3ksVxmMl7k= -github.com/sourcegraph/sourcegraph/lib v0.0.0-20210901132614-5780c9c4b466/go.mod h1:lSNpzAxCBx40MpkM/DbMH7ZUMURcpAstABzhbc4O5V8= +github.com/sourcegraph/sourcegraph/lib v0.0.0-20210901142905-cc5a3f7fd279 h1:RVrhk8J8Ob4J59BAFA5koKlC5pYfz3GCPpzN5R0+M3g= +github.com/sourcegraph/sourcegraph/lib v0.0.0-20210901142905-cc5a3f7fd279/go.mod h1:8A6WlyQouL7Y5MUdL+2+An5rsysLYlm+DaZN/T3egwU= github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152 h1:z/MpntplPaW6QW95pzcAR/72Z5TWDyDnSo0EOcyij9o= github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= diff --git a/internal/batches/executor/changeset_specs.go b/internal/batches/executor/changeset_specs.go index a2a56cd701..345d6bf634 100644 --- a/internal/batches/executor/changeset_specs.go +++ b/internal/batches/executor/changeset_specs.go @@ -7,8 +7,8 @@ import ( "github.com/pkg/errors" "github.com/sourcegraph/go-diff/diff" batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" + "github.com/sourcegraph/sourcegraph/lib/batches/template" "github.com/sourcegraph/src-cli/internal/batches" - "github.com/sourcegraph/src-cli/internal/batches/template" "github.com/sourcegraph/src-cli/internal/batches/util" ) diff --git a/internal/batches/executor/changeset_specs_test.go b/internal/batches/executor/changeset_specs_test.go index 6e626fa4a4..c4d081d81f 100644 --- a/internal/batches/executor/changeset_specs_test.go +++ b/internal/batches/executor/changeset_specs_test.go @@ -9,8 +9,8 @@ import ( "github.com/sourcegraph/batch-change-utils/overridable" batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" "github.com/sourcegraph/sourcegraph/lib/batches/git" + "github.com/sourcegraph/sourcegraph/lib/batches/template" "github.com/sourcegraph/src-cli/internal/batches" - "github.com/sourcegraph/src-cli/internal/batches/template" ) func TestCreateChangesetSpecs(t *testing.T) { diff --git a/internal/batches/executor/coordinator_test.go b/internal/batches/executor/coordinator_test.go index ec95e6876e..890d37c25d 100644 --- a/internal/batches/executor/coordinator_test.go +++ b/internal/batches/executor/coordinator_test.go @@ -12,9 +12,9 @@ import ( "github.com/sourcegraph/batch-change-utils/overridable" batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" "github.com/sourcegraph/sourcegraph/lib/batches/git" + "github.com/sourcegraph/sourcegraph/lib/batches/template" "github.com/sourcegraph/src-cli/internal/batches/graphql" "github.com/sourcegraph/src-cli/internal/batches/mock" - "github.com/sourcegraph/src-cli/internal/batches/template" ) func TestCoordinator_Execute(t *testing.T) { diff --git a/internal/batches/executor/executor_test.go b/internal/batches/executor/executor_test.go index 78366665ff..ca91b15927 100644 --- a/internal/batches/executor/executor_test.go +++ b/internal/batches/executor/executor_test.go @@ -19,11 +19,11 @@ import ( "github.com/sourcegraph/go-diff/diff" batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" "github.com/sourcegraph/sourcegraph/lib/batches/git" + "github.com/sourcegraph/sourcegraph/lib/batches/template" "github.com/sourcegraph/src-cli/internal/api" "github.com/sourcegraph/src-cli/internal/batches" "github.com/sourcegraph/src-cli/internal/batches/docker" "github.com/sourcegraph/src-cli/internal/batches/mock" - "github.com/sourcegraph/src-cli/internal/batches/template" "github.com/sourcegraph/src-cli/internal/batches/workspace" ) diff --git a/internal/batches/executor/run_steps.go b/internal/batches/executor/run_steps.go index 8cfa43dfaf..0e2b08c0a4 100644 --- a/internal/batches/executor/run_steps.go +++ b/internal/batches/executor/run_steps.go @@ -16,8 +16,8 @@ import ( "github.com/pkg/errors" batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" "github.com/sourcegraph/sourcegraph/lib/batches/git" + "github.com/sourcegraph/sourcegraph/lib/batches/template" "github.com/sourcegraph/src-cli/internal/batches/log" - "github.com/sourcegraph/src-cli/internal/batches/template" "github.com/sourcegraph/src-cli/internal/batches/util" "github.com/sourcegraph/src-cli/internal/batches/workspace" diff --git a/internal/batches/executor/task.go b/internal/batches/executor/task.go index 1faec41177..ffbb820c11 100644 --- a/internal/batches/executor/task.go +++ b/internal/batches/executor/task.go @@ -2,9 +2,9 @@ package executor import ( batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" + "github.com/sourcegraph/sourcegraph/lib/batches/template" "github.com/sourcegraph/src-cli/internal/batches" "github.com/sourcegraph/src-cli/internal/batches/graphql" - "github.com/sourcegraph/src-cli/internal/batches/template" ) type Task struct { diff --git a/internal/batches/service/build_tasks.go b/internal/batches/service/build_tasks.go index cb5c6dc90f..61e6fcecac 100644 --- a/internal/batches/service/build_tasks.go +++ b/internal/batches/service/build_tasks.go @@ -5,9 +5,9 @@ import ( "github.com/pkg/errors" batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" + "github.com/sourcegraph/sourcegraph/lib/batches/template" "github.com/sourcegraph/src-cli/internal/batches/executor" "github.com/sourcegraph/src-cli/internal/batches/graphql" - "github.com/sourcegraph/src-cli/internal/batches/template" "github.com/sourcegraph/src-cli/internal/batches/util" ) diff --git a/internal/batches/template/main_test.go b/internal/batches/template/main_test.go deleted file mode 100644 index 5113074ad0..0000000000 --- a/internal/batches/template/main_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package template - -var testRepo1 = &TemplatingRepository{ - ID: "src-cli", - Name: "github.com/sourcegraph/src-cli", - DefaultBranch: TemplatingBranch{Name: "main", TargetOID: "d34db33f"}, - FileMatches: map[string]bool{ - "README.md": true, - "main.go": true, - }, -} diff --git a/internal/batches/template/partial_eval.go b/internal/batches/template/partial_eval.go deleted file mode 100644 index d1276c6b98..0000000000 --- a/internal/batches/template/partial_eval.go +++ /dev/null @@ -1,383 +0,0 @@ -package template - -import ( - "bytes" - "fmt" - "reflect" - "strings" - "text/template" - "text/template/parse" -) - -// IsStaticBool parses the input as a text/template and attempts to evaluate it -// with only the ahead-of-execution information available in StepContext. -// -// To do that it first calls parseAndPartialEval to evaluate the template as -// much as possible. -// -// If, after evaluation, more than text is left (i.e. because the template -// requires information that's only available later) the function returns with -// the first return value being false, because the template is not "static". -// first return value is true. -// -// If only text is left we check whether that text equals "true". The result of -// that check is the second return value. -func IsStaticBool(input string, ctx *StepContext) (isStatic bool, boolVal bool, err error) { - t, err := parseAndPartialEval(input, ctx) - if err != nil { - return false, false, err - } - - isStatic = true - for _, n := range t.Tree.Root.Nodes { - if n.Type() != parse.NodeText { - isStatic = false - break - } - } - if !isStatic { - return isStatic, false, nil - } - - return true, isTrueOutput(t.Tree.Root), nil -} - -// parseAndPartialEval parses input as a text/template and then attempts to -// partially evaluate the parts of the template it can evaluate ahead of time -// (meaning: before we've executed any batch spec steps and have a full -// StepContext available). -// -// If it's possible to evaluate a parse.ActionNode (which is what sits between -// delimiters in a text/template), the node is rewritten into a parse.TextNode, -// to make it look like it's always been text in the template. -// -// Partial evaluation is done in a best effort manner: if it's not possible to -// evaluate a node (because it requires information that we only later get, or -// because it's too complex, etc.) we degrade gracefully and simply abort the -// partial evaluation and leave the node as is. -// -// It also should be noted that we don't do "full" partial evaluation: if we -// come across value that we can't partially evaluate we abort the process *for -// the whole node* without replacing the sub-nodes that we've successfully -// evaluated. Why? Because we can't construct correct `*parse.Node` from -// outside the `parse` package. In other words: we evaluate -// all-parse.ActionNode-or-nothing. -func parseAndPartialEval(input string, ctx *StepContext) (*template.Template, error) { - t, err := template. - New("partial-eval"). - Delims(startDelim, endDelim). - Funcs(builtins). - Funcs(ctx.ToFuncMap()). - Parse(input) - - if err != nil { - return nil, err - } - - for i, n := range t.Tree.Root.Nodes { - t.Tree.Root.Nodes[i] = rewriteNode(n, ctx) - } - - return t, nil -} - -// rewriteNode takes the given parse.Parse and tries to partially evaluate it. -// If that's possible, the output of the evaluation is turned into text and -// instead of the node that was passed in a new parse.TextNode is returned that -// represents the output of the evaluation. -func rewriteNode(n parse.Node, ctx *StepContext) parse.Node { - switch n := n.(type) { - case *parse.ActionNode: - if val, ok := evalPipe(ctx, n.Pipe); ok { - var out bytes.Buffer - fmt.Fprint(&out, val.Interface()) - return &parse.TextNode{ - Text: out.Bytes(), - Pos: n.Pos, - NodeType: parse.NodeText, - } - } - - return n - - default: - return n - } -} - -// noValue is returned by the functions that partially evaluate a parse.Node -// to signify that evaluation was not possible or did not yield a value. -var noValue reflect.Value - -func evalPipe(ctx *StepContext, p *parse.PipeNode) (finalVal reflect.Value, ok bool) { - // If the pipe contains declaration we abort evaluation. - if len(p.Decl) > 0 { - return noValue, false - } - - // TODO: Support finalVal - // finalVal is the value of the previous Cmd in a pipe (i.e. `${{ 3 + 3 | eq 6 }}`) - // It needs to be the final (fixed) argument of a call if it's set. - - for _, c := range p.Cmds { - finalVal, ok = evalCmd(ctx, c, finalVal) - if !ok { - return noValue, false - } - } - - return finalVal, ok -} - -func evalCmd(ctx *StepContext, c *parse.CommandNode, previousValue reflect.Value) (reflect.Value, bool) { - switch first := c.Args[0].(type) { - case *parse.BoolNode, *parse.NumberNode, *parse.StringNode, *parse.ChainNode: - if len(c.Args) == 1 { - return evalNode(ctx, first) - } - return noValue, false - - case *parse.IdentifierNode: - // A function call always starts with an identifier - return evalFunction(ctx, first, first.Ident, c.Args, previousValue) - - default: - // Node type that we don't care about, so we don't even try to evaluate it - return noValue, false - } -} - -func evalNode(ctx *StepContext, n parse.Node) (reflect.Value, bool) { - switch n := n.(type) { - case *parse.BoolNode: - return reflect.ValueOf(n.True), true - - case *parse.NumberNode: - // This case branch is lifted from Go's text/template execution engine: - // https://sourcegraph.com/github.com/golang/go@2c9f5a1da823773c436f8b2c119635797d6db2d3/-/blob/src/text/template/exec.go#L493-530 - // The difference is that we don't do any error handling but simply abort. - switch { - case n.IsComplex: - return reflect.ValueOf(n.Complex128), true - - case n.IsFloat && - !isHexInt(n.Text) && !isRuneInt(n.Text) && - strings.ContainsAny(n.Text, ".eEpP"): - return reflect.ValueOf(n.Float64), true - - case n.IsInt: - num := int(n.Int64) - if int64(num) != n.Int64 { - return noValue, false - } - return reflect.ValueOf(num), true - - case n.IsUint: - return noValue, false - } - - case *parse.StringNode: - return reflect.ValueOf(n.Text), true - - case *parse.ChainNode: - // For now we only support fields that are 1 level deep (see below). - // Should we ever want to support more than one level, we need to - // revise this. - if len(n.Field) != 1 { - return noValue, false - } - - if ident, ok := n.Node.(*parse.IdentifierNode); ok { - switch ident.Ident { - case "repository": - switch n.Field[0] { - case "search_result_paths": - // TODO: We don't eval search_result_paths for now, since it's a - // "complex" value, a slice of strings, and turning that - // into text might not be useful to the user. So we abort. - return noValue, false - case "name": - return reflect.ValueOf(ctx.Repository.Name), true - } - - case "batch_change": - switch n.Field[0] { - case "name": - return reflect.ValueOf(ctx.BatchChange.Name), true - case "description": - return reflect.ValueOf(ctx.BatchChange.Description), true - } - } - } - return noValue, false - - case *parse.PipeNode: - return evalPipe(ctx, n) - } - - return noValue, false -} - -func isRuneInt(s string) bool { - return len(s) > 0 && s[0] == '\'' -} - -func isHexInt(s string) bool { - return len(s) > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X') && !strings.ContainsAny(s, "pP") -} - -func evalFunction(ctx *StepContext, fn *parse.IdentifierNode, name string, args []parse.Node, previousValue reflect.Value) (val reflect.Value, success bool) { - defer func() { - if r := recover(); r != nil { - val = noValue - success = false - } - }() - - switch name { - case "eq": - return evalEqCall(ctx, args[1:]) - - case "ne": - equal, ok := evalEqCall(ctx, args[1:]) - if !ok { - return noValue, false - } - return reflect.ValueOf(!equal.Bool()), true - - case "not": - return evalNotCall(ctx, args[1:]) - - default: - concreteFn, ok := builtins[name] - if !ok { - return noValue, false - } - - fn := reflect.ValueOf(concreteFn) - - // We can eval only if all args are static: - var evaluatedArgs []reflect.Value - for _, a := range args[1:] { - v, ok := evalNode(ctx, a) - if !ok { - // One of the args is not static, abort - return noValue, false - } - evaluatedArgs = append(evaluatedArgs, v) - - } - - ret := fn.Call(evaluatedArgs) - if len(ret) == 2 && !ret[1].IsNil() { - return noValue, false - } - return ret[0], true - } -} - -func evalNotCall(ctx *StepContext, args []parse.Node) (reflect.Value, bool) { - // We only support 1 arg for now: - if len(args) != 1 { - return noValue, false - } - - arg, ok := evalNode(ctx, args[0]) - if !ok { - return noValue, false - } - - return reflect.ValueOf(!isTrue(arg)), true -} - -func evalEqCall(ctx *StepContext, args []parse.Node) (reflect.Value, bool) { - // We only support 2 args for now: - if len(args) != 2 { - return noValue, false - } - - // We only eval `eq` if all args are static: - var evaluatedArgs []reflect.Value - for _, a := range args { - v, ok := evalNode(ctx, a) - if !ok { - // One of the args is not static, abort - return noValue, false - } - evaluatedArgs = append(evaluatedArgs, v) - } - - if len(evaluatedArgs) != 2 { - // safety check - return noValue, false - } - - isEqual := evaluatedArgs[0].Interface() == evaluatedArgs[1].Interface() - return reflect.ValueOf(isEqual), true -} - -// isTrue is taken from Go's text/template/exec.go and simplified -func isTrue(val reflect.Value) (truth bool) { - if !val.IsValid() { - // Something like var x interface{}, never set. It's a form of nil. - return false - } - switch val.Kind() { - case reflect.Array, reflect.Map, reflect.Slice, reflect.String: - return val.Len() > 0 - case reflect.Bool: - return val.Bool() - case reflect.Complex64, reflect.Complex128: - return val.Complex() != 0 - case reflect.Chan, reflect.Func, reflect.Ptr, reflect.Interface: - return !val.IsNil() - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return val.Int() != 0 - case reflect.Float32, reflect.Float64: - return val.Float() != 0 - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - return val.Uint() != 0 - case reflect.Struct: - return true // Struct values are always true. - default: - return false - } -} - -// debugParseNode can be used to produce a debug-friendly version of a parse.Node. -func debugParseNode(n parse.Node) string { - switch n := n.(type) { - case *parse.ActionNode: - return fmt.Sprintf("parse.ActionNode. n.Pipe=%#v", n.Pipe) - - case *parse.IdentifierNode: - return fmt.Sprintf("parse.IdentifierNode: %+v", n) - - case *parse.FieldNode: - return fmt.Sprintf("parse.FieldNode: %+v", n) - - case *parse.ChainNode: - return fmt.Sprintf("parse.ChainNode. n.Node=(%s), chain.Field: %+v", debugParseNode(n.Node), n.Field) - - case *parse.TextNode: - return fmt.Sprintf("parse.TextNode. n.Text=%q", n.Text) - - case *parse.BoolNode: - return fmt.Sprintf("parse.BoolNode") - - case *parse.CommandNode: - return fmt.Sprintf("parse.CommandNode") - - case *parse.NilNode: - return fmt.Sprintf("parse.NilNode") - - case *parse.NumberNode: - return fmt.Sprintf("parse.NumberNode") - - case *parse.PipeNode: - return fmt.Sprintf("parse.PipeNode") - - default: - return fmt.Sprintf("UNHANDLED parse.Node TYPE: %s", n.String()) - } -} diff --git a/internal/batches/template/partial_eval_test.go b/internal/batches/template/partial_eval_test.go deleted file mode 100644 index 403d7ed4b7..0000000000 --- a/internal/batches/template/partial_eval_test.go +++ /dev/null @@ -1,294 +0,0 @@ -package template - -import ( - "testing" - - "github.com/google/go-cmp/cmp" -) - -var partialEvalStepCtx = &StepContext{ - BatchChange: BatchChangeAttributes{ - Name: "test-batch-change", - Description: "test-description", - }, - // Step is not set when evalStepCondition is called - Repository: TemplatingRepository{ - Name: "github.com/sourcegraph/src-cli", - FileMatches: map[string]bool{ - "README.md": true, - "main.go": true, - }, - }, -} - -func runParseAndPartialTest(t *testing.T, in, want string) { - t.Helper() - - tmpl, err := parseAndPartialEval(in, partialEvalStepCtx) - if err != nil { - t.Fatal(err) - } - - tmplStr := tmpl.Tree.Root.String() - if tmplStr != want { - t.Fatalf("wrong output:\n%s", cmp.Diff(want, tmplStr)) - } -} - -func TestParseAndPartialEval(t *testing.T) { - t.Run("evaluated", func(t *testing.T) { - for _, tt := range []struct{ input, want string }{ - { - // Literal constants: - `this is my template ${{ "hardcoded string" }}`, - `this is my template hardcoded string`, - }, - { - `${{ 1234 }}`, - `1234`, - }, - { - `${{ true }} ${{ false }}`, - `true false`, - }, - { - // Evaluated, since they're static values: - `${{ repository.name }} ${{ batch_change.name }} ${{ batch_change.description }}`, - `github.com/sourcegraph/src-cli test-batch-change test-description`, - }, - { - `AAA${{ repository.name }}BBB${{ batch_change.name }}CCC${{ batch_change.description }}DDD`, - `AAAgithub.com/sourcegraph/src-cliBBBtest-batch-changeCCCtest-descriptionDDD`, - }, - { - // Function call with static value and runtime value: - `${{ eq repository.name outputs.repo.name }}`, - // Aborts, since one of them is runtime value - `{{eq repository.name outputs.repo.name}}`, - }, - { - // "eq" call with 2 static values: - `${{ eq repository.name "github.com/sourcegraph/src-cli" }}`, - `true`, - }, - { - // "eq" call with 2 literal values: - `${{ eq 5 5 }}`, - `true`, - }, - { - // "not" call: - `${{ not (eq repository.name "bitbucket-repo") }}`, - `true`, - }, - { - // "not" call: - `${{ not 1234 }} ${{ not false }} ${{ not true }}`, - `false true false`, - }, - { - // "ne" call with 2 static values: - `${{ ne repository.name "github.com/sourcegraph/src-cli" }}`, - `false`, - }, - { - // "ne" call with 2 literal values: - `${{ ne 5 5 }}`, - `false`, - }, - { - // Function call with builtin function and static values: - `${{ matches repository.name "github.com*" }}`, - `true`, - }, - { - // Nested function call with builtin function and static values: - `${{ eq false (matches repository.name "github.com*") }}`, - `false`, - }, - { - // Nested nested function call with builtin function and static values: - `${{ eq false (eq false (matches repository.name "github.com*")) }}`, - `true`, - }, - } { - runParseAndPartialTest(t, tt.input, tt.want) - } - }) - - t.Run("aborted", func(t *testing.T) { - for _, tt := range []struct{ input, want string }{ - { - // Field that doesn't exist - `${{ repository.secretlocation }}`, - `{{repository.secretlocation}}`, - }, - { - // Field access that's too deep - `${{ repository.name.doesnotexist }}`, - `{{repository.name.doesnotexist}}`, - }, - { - // Complex value - `${{ repository.search_result_paths }}`, - // String representation of templates uses standard delimiters - `{{repository.search_result_paths}}`, - }, - { - // Runtime value - `${{ outputs.runtime.value }}`, - `{{outputs.runtime.value}}`, - }, - { - // Runtime value - `${{ step.modified_files }}`, - `{{step.modified_files}}`, - }, - { - // Runtime value - `${{ previous_step.modified_files }}`, - `{{previous_step.modified_files}}`, - }, - { - // "eq" call with static value and runtime value: - `${{ eq repository.name outputs.repo.name }}`, - // Aborts, since one of them is runtime value - `{{eq repository.name outputs.repo.name}}`, - }, - { - // "eq" call with more than 2 arguments: - `${{ eq repository.name "github.com/sourcegraph/src-cli" "github.com/sourcegraph/sourcegraph" }}`, - `{{eq repository.name "github.com/sourcegraph/src-cli" "github.com/sourcegraph/sourcegraph"}}`, - }, - { - // Nested nested function call with builtin function but runtime values: - `${{ eq false (eq false (matches outputs.runtime.value "github.com*")) }}`, - `{{eq false (eq false (matches outputs.runtime.value "github.com*"))}}`, - }, - } { - runParseAndPartialTest(t, tt.input, tt.want) - } - }) -} - -func TestParseAndPartialEval_BuiltinFunctions(t *testing.T) { - t.Run("success", func(t *testing.T) { - for _, tt := range []struct{ input, want string }{ - { - `${{ join (split repository.name "/") "-" }}`, - `github.com-sourcegraph-src-cli`, - }, - { - `${{ split repository.name "/" "-" }}`, - `{{split repository.name "/" "-"}}`, - }, - { - `${{ replace repository.name "github" "foobar" }}`, - `foobar.com/sourcegraph/src-cli`, - }, - { - `${{ join_if "SEP" repository.name "postfix" }}`, - `github.com/sourcegraph/src-cliSEPpostfix`, - }, - { - `${{ matches repository.name "github.com*" }}`, - `true`, - }, - } { - runParseAndPartialTest(t, tt.input, tt.want) - } - }) - - t.Run("aborted", func(t *testing.T) { - for _, tt := range []struct{ input, want string }{ - { - // Wrong argument type - `${{ join "foobar" "-" }}`, - `{{join "foobar" "-"}}`, - }, - { - // Wrong argument count - `${{ join (split repository.name "/") "-" "too" "many" "args" }}`, - `{{join (split repository.name "/") "-" "too" "many" "args"}}`, - }, - } { - runParseAndPartialTest(t, tt.input, tt.want) - } - }) -} - -func TestIsStaticBool(t *testing.T) { - tests := []struct { - name string - template string - wantIsStatic bool - wantBoolVal bool - }{ - - { - name: "true literal", - template: `true`, - wantIsStatic: true, - wantBoolVal: true, - }, - { - name: "false literal", - template: `false`, - wantIsStatic: true, - wantBoolVal: false, - }, - { - name: "static non bool value", - template: `${{ repository.name }}`, - wantIsStatic: true, - wantBoolVal: false, - }, - { - name: "function call true val", - template: `${{ eq repository.name "github.com/sourcegraph/src-cli" }}`, - wantIsStatic: true, - wantBoolVal: true, - }, - { - name: "function call false val", - template: `${{ eq repository.name "hans wurst" }}`, - wantIsStatic: true, - wantBoolVal: false, - }, - { - name: "nested function call and whitespace", - template: ` ${{ eq false (eq false (matches repository.name "github.com*")) }} `, - wantIsStatic: true, - wantBoolVal: true, - }, - { - name: "nested function call with runtime value", - template: `${{ eq false (eq false (matches outputs.repo.name "github.com*")) }}`, - wantIsStatic: false, - wantBoolVal: false, - }, - { - name: "random string", - template: `adfadsfasdfadsfasdfasdfadsf`, - wantIsStatic: true, - wantBoolVal: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - isStatic, boolVal, err := IsStaticBool(tt.template, partialEvalStepCtx) - if err != nil { - t.Fatal(err) - } - - if isStatic != tt.wantIsStatic { - t.Fatalf("wrong isStatic value. want=%t, got=%t", tt.wantIsStatic, isStatic) - } - if boolVal != tt.wantBoolVal { - t.Fatalf("wrong boolVal value. want=%t, got=%t", tt.wantBoolVal, boolVal) - } - }) - } -} diff --git a/internal/batches/template/templating.go b/internal/batches/template/templating.go deleted file mode 100644 index 2fc5aea408..0000000000 --- a/internal/batches/template/templating.go +++ /dev/null @@ -1,311 +0,0 @@ -package template - -import ( - "bytes" - "io" - "sort" - "strings" - "text/template" - - "github.com/gobwas/glob" - "github.com/pkg/errors" - "github.com/sourcegraph/sourcegraph/lib/batches/git" -) - -const startDelim = "${{" -const endDelim = "}}" - -var builtins = template.FuncMap{ - "join": strings.Join, - "split": strings.Split, - "replace": strings.ReplaceAll, - "join_if": func(sep string, elems ...string) string { - var nonBlank []string - for _, e := range elems { - if e != "" { - nonBlank = append(nonBlank, e) - } - } - return strings.Join(nonBlank, sep) - }, - "matches": func(in, pattern string) (bool, error) { - g, err := glob.Compile(pattern) - if err != nil { - return false, err - } - return g.Match(in), nil - }, -} - -func isTrueOutput(output interface{ String() string }) bool { - return strings.TrimSpace(output.String()) == "true" -} - -func EvalStepCondition(condition string, stepCtx *StepContext) (bool, error) { - if condition == "" { - return true, nil - } - - var out bytes.Buffer - if err := RenderStepTemplate("step-condition", condition, &out, stepCtx); err != nil { - return false, errors.Wrap(err, "parsing step if") - } - - return isTrueOutput(&out), nil -} - -func RenderStepTemplate(name, tmpl string, out io.Writer, stepCtx *StepContext) error { - t, err := template.New(name).Delims(startDelim, endDelim).Funcs(builtins).Funcs(stepCtx.ToFuncMap()).Parse(tmpl) - if err != nil { - return errors.Wrap(err, "parsing step run") - } - - return t.Execute(out, stepCtx) -} - -func RenderStepMap(m map[string]string, stepCtx *StepContext) (map[string]string, error) { - rendered := make(map[string]string, len(m)) - - for k, v := range m { - var out bytes.Buffer - - if err := RenderStepTemplate(k, v, &out, stepCtx); err != nil { - return rendered, err - } - - rendered[k] = out.String() - } - - return rendered, nil -} - -// TODO(mrnugget): This is bad and should be (a) removed or (b) moved to batches package -type BatchChangeAttributes struct { - Name string - Description string -} - -type TemplatingBranch struct { - Name string - TargetOID string -} - -type TemplatingRepository struct { - ID string - Name string - DefaultBranch TemplatingBranch - FileMatches map[string]bool -} - -func (r TemplatingRepository) SearchResultPaths() (list fileMatchPathList) { - var files []string - for f := range r.FileMatches { - files = append(files, f) - } - sort.Strings(files) - return fileMatchPathList(files) -} - -type fileMatchPathList []string - -func (f fileMatchPathList) String() string { return strings.Join(f, " ") } - -// StepContext represents the contextual information available when rendering a -// step's fields, such as "run" or "outputs", as templates. -type StepContext struct { - // BatchChange are the attributes in the BatchSpec that are set on the BatchChange. - BatchChange BatchChangeAttributes - // Outputs are the outputs set by the current and all previous steps. - Outputs map[string]interface{} - // Step is the result of the current step. Empty when evaluating the "run" field - // but filled when evaluating the "outputs" field. - Step StepResult - // Steps contains the path in which the steps are being executed and the - // changes made by all steps that were executed up until the current step. - Steps StepsContext - // PreviousStep is the result of the previous step. Empty when there is no - // previous step. - PreviousStep StepResult - // Repository is the Sourcegraph repository in which the steps are executed. - Repository TemplatingRepository -} - -// ToFuncMap returns a template.FuncMap to access fields on the StepContext in a -// text/template. -func (stepCtx *StepContext) ToFuncMap() template.FuncMap { - newStepResult := func(res *StepResult) map[string]interface{} { - m := map[string]interface{}{ - "modified_files": "", - "added_files": "", - "deleted_files": "", - "renamed_files": "", - "stdout": "", - "stderr": "", - } - if res == nil { - return m - } - - m["modified_files"] = res.ModifiedFiles() - m["added_files"] = res.AddedFiles() - m["deleted_files"] = res.DeletedFiles() - m["renamed_files"] = res.RenamedFiles() - - if res.Stdout != nil { - m["stdout"] = res.Stdout.String() - } - - if res.Stderr != nil { - m["stderr"] = res.Stderr.String() - } - - return m - } - - return template.FuncMap{ - "previous_step": func() map[string]interface{} { - return newStepResult(&stepCtx.PreviousStep) - }, - "step": func() map[string]interface{} { - return newStepResult(&stepCtx.Step) - }, - "steps": func() map[string]interface{} { - res := newStepResult(&StepResult{Files: stepCtx.Steps.Changes}) - res["path"] = stepCtx.Steps.Path - return res - }, - "outputs": func() map[string]interface{} { - return stepCtx.Outputs - }, - "repository": func() map[string]interface{} { - return map[string]interface{}{ - "search_result_paths": stepCtx.Repository.SearchResultPaths(), - "name": stepCtx.Repository.Name, - } - }, - "batch_change": func() map[string]interface{} { - return map[string]interface{}{ - "name": stepCtx.BatchChange.Name, - "description": stepCtx.BatchChange.Description, - } - }, - } -} - -// StepResult represents the result of a previously executed step. -type StepResult struct { - // Files are the changes made to Files by the step. - Files *git.Changes - - // Stdout is the output produced by the step on standard out. - Stdout *bytes.Buffer - // Stderr is the output produced by the step on standard error. - Stderr *bytes.Buffer -} - -// ModifiedFiles returns the files modified by a step. -func (r StepResult) ModifiedFiles() []string { - if r.Files != nil { - return r.Files.Modified - } - return []string{} -} - -// AddedFiles returns the files added by a step. -func (r StepResult) AddedFiles() []string { - if r.Files != nil { - return r.Files.Added - } - return []string{} -} - -// DeletedFiles returns the files deleted by a step. -func (r StepResult) DeletedFiles() []string { - if r.Files != nil { - return r.Files.Deleted - } - return []string{} -} - -// RenamedFiles returns the new name of files that have been renamed by a step. -func (r StepResult) RenamedFiles() []string { - if r.Files != nil { - return r.Files.Renamed - } - return []string{} -} - -type StepsContext struct { - // Changes that have been made by executing all steps. - Changes *git.Changes - // Path is the relative-to-root directory in which the steps have been - // executed. Default is "". No leading "/". - Path string -} - -// ChangesetTemplateContext represents the contextual information available -// when rendering a field of the ChangesetTemplate as a template. -type ChangesetTemplateContext struct { - // BatchChangeAttributes are the attributes of the BatchChange that will be - // created/updated. - BatchChangeAttributes BatchChangeAttributes - - // Steps are the changes made by all steps that were executed. - Steps StepsContext - - // Outputs are the outputs defined and initialized by the steps. - Outputs map[string]interface{} - - // Repository is the repository in which the steps were executed. - Repository TemplatingRepository -} - -// ToFuncMap returns a template.FuncMap to access fields on the StepContext in a -// text/template. -func (tmplCtx *ChangesetTemplateContext) ToFuncMap() template.FuncMap { - return template.FuncMap{ - "repository": func() map[string]interface{} { - return map[string]interface{}{ - "search_result_paths": tmplCtx.Repository.SearchResultPaths(), - "name": tmplCtx.Repository.Name, - } - }, - "batch_change": func() map[string]interface{} { - return map[string]interface{}{ - "name": tmplCtx.BatchChangeAttributes.Name, - "description": tmplCtx.BatchChangeAttributes.Description, - } - }, - "outputs": func() map[string]interface{} { - return tmplCtx.Outputs - }, - "steps": func() map[string]interface{} { - // Wrap the *StepChanges in a StepResult so we can use nil-safe - // methods. - res := StepResult{Files: tmplCtx.Steps.Changes} - - return map[string]interface{}{ - "modified_files": res.ModifiedFiles(), - "added_files": res.AddedFiles(), - "deleted_files": res.DeletedFiles(), - "renamed_files": res.RenamedFiles(), - "path": tmplCtx.Steps.Path, - } - }, - } -} - -func RenderChangesetTemplateField(name, tmpl string, tmplCtx *ChangesetTemplateContext) (string, error) { - var out bytes.Buffer - - t, err := template.New(name).Delims(startDelim, endDelim).Funcs(builtins).Funcs(tmplCtx.ToFuncMap()).Parse(tmpl) - if err != nil { - return "", err - } - - if err := t.Execute(&out, tmplCtx); err != nil { - return "", err - } - - return strings.TrimSpace(out.String()), nil -} diff --git a/internal/batches/template/templating_test.go b/internal/batches/template/templating_test.go deleted file mode 100644 index b6990f9fc4..0000000000 --- a/internal/batches/template/templating_test.go +++ /dev/null @@ -1,361 +0,0 @@ -package template - -import ( - "bytes" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/sourcegraph/sourcegraph/lib/batches/git" - "gopkg.in/yaml.v3" -) - -var testChanges = &git.Changes{ - Modified: []string{"go.mod"}, - Added: []string{"main.go.swp"}, - Deleted: []string{".DS_Store"}, - Renamed: []string{"new-filename.txt"}, -} - -func TestEvalStepCondition(t *testing.T) { - stepCtx := &StepContext{ - BatchChange: BatchChangeAttributes{ - Name: "test-batch-change", - Description: "This batch change is just an experiment", - }, - PreviousStep: StepResult{ - Files: testChanges, - Stdout: bytes.NewBufferString("this is previous step's stdout"), - Stderr: bytes.NewBufferString("this is previous step's stderr"), - }, - Steps: StepsContext{ - Changes: testChanges, - Path: "sub/directory/of/repo", - }, - Outputs: map[string]interface{}{}, - // Step is not set when evalStepCondition is called - Repository: *testRepo1, - } - - tests := []struct { - run string - want bool - }{ - {run: `true`, want: true}, - {run: ` true `, want: true}, - {run: `TRUE`, want: false}, - {run: `false`, want: false}, - {run: `FALSE`, want: false}, - {run: `${{ eq repository.name "github.com/sourcegraph/src-cli" }}`, want: true}, - {run: `${{ eq steps.path "sub/directory/of/repo" }}`, want: true}, - {run: `${{ matches repository.name "github.com/sourcegraph/*" }}`, want: true}, - } - - for _, tc := range tests { - got, err := EvalStepCondition(tc.run, stepCtx) - if err != nil { - t.Fatal(err) - } - - if got != tc.want { - t.Fatalf("wrong value. want=%t, got=%t", tc.want, got) - } - } -} - -const rawYaml = `dist: release -env: - - GO111MODULE=on - - CGO_ENABLED=0 -before: - hooks: - - go mod download - - go mod tidy - - go generate ./schema -` - -func TestRenderStepTemplate(t *testing.T) { - // To avoid bugs due to differences between test setup and actual code, we - // do the actual parsing of YAML here to get an interface{} which we'll put - // in the StepContext. - var parsedYaml interface{} - if err := yaml.Unmarshal([]byte(rawYaml), &parsedYaml); err != nil { - t.Fatalf("failed to parse YAML: %s", err) - } - - stepCtx := &StepContext{ - BatchChange: BatchChangeAttributes{ - Name: "test-batch-change", - Description: "This batch change is just an experiment", - }, - PreviousStep: StepResult{ - Files: testChanges, - Stdout: bytes.NewBufferString("this is previous step's stdout"), - Stderr: bytes.NewBufferString("this is previous step's stderr"), - }, - Outputs: map[string]interface{}{ - "lastLine": "lastLine is this", - "project": parsedYaml, - }, - Step: StepResult{ - Files: testChanges, - Stdout: bytes.NewBufferString("this is current step's stdout"), - Stderr: bytes.NewBufferString("this is current step's stderr"), - }, - Steps: StepsContext{Changes: testChanges, Path: "sub/directory/of/repo"}, - Repository: *testRepo1, - } - - tests := []struct { - name string - stepCtx *StepContext - run string - want string - }{ - { - name: "lower-case aliases", - stepCtx: stepCtx, - run: `${{ repository.search_result_paths }} -${{ repository.name }} -${{ batch_change.name }} -${{ batch_change.description }} -${{ previous_step.modified_files }} -${{ previous_step.added_files }} -${{ previous_step.deleted_files }} -${{ previous_step.renamed_files }} -${{ previous_step.stdout }} -${{ previous_step.stderr}} -${{ outputs.lastLine }} -${{ index outputs.project.env 1 }} -${{ step.modified_files }} -${{ step.added_files }} -${{ step.deleted_files }} -${{ step.renamed_files }} -${{ step.stdout}} -${{ step.stderr}} -${{ steps.modified_files }} -${{ steps.added_files }} -${{ steps.deleted_files }} -${{ steps.renamed_files }} -${{ steps.path }} -`, - want: `README.md main.go -github.com/sourcegraph/src-cli -test-batch-change -This batch change is just an experiment -[go.mod] -[main.go.swp] -[.DS_Store] -[new-filename.txt] -this is previous step's stdout -this is previous step's stderr -lastLine is this -CGO_ENABLED=0 -[go.mod] -[main.go.swp] -[.DS_Store] -[new-filename.txt] -this is current step's stdout -this is current step's stderr -[go.mod] -[main.go.swp] -[.DS_Store] -[new-filename.txt] -sub/directory/of/repo -`, - }, - { - name: "empty context", - stepCtx: &StepContext{}, - run: `${{ repository.search_result_paths }} -${{ repository.name }} -${{ previous_step.modified_files }} -${{ previous_step.added_files }} -${{ previous_step.deleted_files }} -${{ previous_step.renamed_files }} -${{ previous_step.stdout }} -${{ previous_step.stderr}} -${{ outputs.lastLine }} -${{ outputs.project }} -${{ step.modified_files }} -${{ step.added_files }} -${{ step.deleted_files }} -${{ step.renamed_files }} -${{ step.stdout}} -${{ step.stderr}} -${{ steps.modified_files }} -${{ steps.added_files }} -${{ steps.deleted_files }} -${{ steps.renamed_files }} -${{ steps.path }} -`, - want: ` - -[] -[] -[] -[] - - - - -[] -[] -[] -[] - - -[] -[] -[] -[] - -`, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var out bytes.Buffer - - err := RenderStepTemplate("testing", tc.run, &out, tc.stepCtx) - if err != nil { - t.Fatal(err) - } - - if out.String() != tc.want { - t.Fatalf("wrong output:\n%s", cmp.Diff(tc.want, out.String())) - } - }) - } -} - -func TestRenderStepMap(t *testing.T) { - stepCtx := &StepContext{ - PreviousStep: StepResult{ - Files: testChanges, - Stdout: bytes.NewBufferString("this is previous step's stdout"), - Stderr: bytes.NewBufferString("this is previous step's stderr"), - }, - Outputs: map[string]interface{}{}, - Repository: *testRepo1, - } - - input := map[string]string{ - "/tmp/my-file.txt": `${{ previous_step.modified_files }}`, - "/tmp/my-other-file.txt": `${{ previous_step.added_files }}`, - "/tmp/my-other-file2.txt": `${{ previous_step.deleted_files }}`, - } - - have, err := RenderStepMap(input, stepCtx) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } - - want := map[string]string{ - "/tmp/my-file.txt": "[go.mod]", - "/tmp/my-other-file.txt": "[main.go.swp]", - "/tmp/my-other-file2.txt": "[.DS_Store]", - } - - if diff := cmp.Diff(want, have); diff != "" { - t.Fatalf("wrong output:\n%s", diff) - } -} - -func TestRenderChangesetTemplateField(t *testing.T) { - // To avoid bugs due to differences between test setup and actual code, we - // do the actual parsing of YAML here to get an interface{} which we'll put - // in the StepContext. - var parsedYaml interface{} - if err := yaml.Unmarshal([]byte(rawYaml), &parsedYaml); err != nil { - t.Fatalf("failed to parse YAML: %s", err) - } - - tmplCtx := &ChangesetTemplateContext{ - BatchChangeAttributes: BatchChangeAttributes{ - Name: "test-batch-change", - Description: "This batch change is just an experiment", - }, - Outputs: map[string]interface{}{ - "lastLine": "lastLine is this", - "project": parsedYaml, - }, - Repository: *testRepo1, - Steps: StepsContext{ - Changes: &git.Changes{ - Modified: []string{"modified-file.txt"}, - Added: []string{"added-file.txt"}, - Deleted: []string{"deleted-file.txt"}, - Renamed: []string{"renamed-file.txt"}, - }, - Path: "infrastructure/sub-project", - }, - } - - tests := []struct { - name string - tmplCtx *ChangesetTemplateContext - tmpl string - want string - }{ - { - name: "lower-case aliases", - tmplCtx: tmplCtx, - tmpl: `${{ repository.search_result_paths }} -${{ repository.name }} -${{ batch_change.name }} -${{ batch_change.description }} -${{ outputs.lastLine }} -${{ index outputs.project.env 1 }} -${{ steps.modified_files }} -${{ steps.added_files }} -${{ steps.deleted_files }} -${{ steps.renamed_files }} -${{ steps.path }} -`, - want: `README.md main.go -github.com/sourcegraph/src-cli -test-batch-change -This batch change is just an experiment -lastLine is this -CGO_ENABLED=0 -[modified-file.txt] -[added-file.txt] -[deleted-file.txt] -[renamed-file.txt] -infrastructure/sub-project`, - }, - { - name: "empty context", - tmplCtx: &ChangesetTemplateContext{}, - tmpl: `${{ repository.search_result_paths }} -${{ repository.name }} -${{ outputs.lastLine }} -${{ outputs.project }} -${{ steps.modified_files }} -${{ steps.added_files }} -${{ steps.deleted_files }} -${{ steps.renamed_files }} -`, - want: ` - -[] -[] -[] -[]`, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - out, err := RenderChangesetTemplateField("testing", tc.tmpl, tc.tmplCtx) - if err != nil { - t.Fatal(err) - } - - if out != tc.want { - t.Fatalf("wrong output:\n%s", cmp.Diff(tc.want, out)) - } - }) - } -} diff --git a/internal/batches/util/repo.go b/internal/batches/util/repo.go index eaea63cfdc..ccab38fc80 100644 --- a/internal/batches/util/repo.go +++ b/internal/batches/util/repo.go @@ -1,8 +1,8 @@ package util import ( + "github.com/sourcegraph/sourcegraph/lib/batches/template" "github.com/sourcegraph/src-cli/internal/batches/graphql" - "github.com/sourcegraph/src-cli/internal/batches/template" ) // GraphQLRepoToTemplatingRepo transforms a given *graphql.Repository into a