Basic Starlark support

go.mod

@@ -27,11 +27,13 @@ require (
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.6.1
+	go.starlark.net v0.0.0-20200821142938-949cc6f4b097
 	golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect
 	golang.org/x/sys v0.0.0-20200819141100-7c7a22168250 // indirect
 	google.golang.org/genproto v0.0.0-20200815001618-f69a88009b70 // indirect
 	google.golang.org/grpc v1.31.0
 	google.golang.org/protobuf v1.25.0
+	gopkg.in/yaml.v2 v2.3.0
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
 	gotest.tools v2.2.0+incompatible // indirect
 )

go.sum

@@ -25,6 +25,9 @@ github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjw
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cirruslabs/cirrus-ci-agent v1.7.0 h1:YyWvlGVE3T7i9zE5Enk/bFurQjCCYSjO5fyfBOcmotg=
 github.com/cirruslabs/cirrus-ci-agent v1.7.0/go.mod h1:gx+B38qqOeK6MMlXYpKkuamrG/LxSpnhWx5/pMvUaAc=
 github.com/cirruslabs/cirrus-ci-annotations v0.0.0-20200716124710-e1adb4748203/go.mod h1:98qD7HLlBx5aNqWiCH80OTTqTTsbXT69wxnlnrnoL0E=
@@ -243,6 +246,8 @@ github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0B
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.starlark.net v0.0.0-20200821142938-949cc6f4b097 h1:YiRMXXgG+Pg26t1fjq+iAjaauKWMC9cmGFrtOEuwDDg=
+go.starlark.net v0.0.0-20200821142938-949cc6f4b097/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
@@ -302,6 +307,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed h1:WBkVNH1zd9jg/dK4HCM4lNANnmd12EHC9z+LmcCG4ns=
 golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200817155316-9781c653f443 h1:X18bCaipMcoJGm27Nv7zr4XYPKGUy92GtqboKC2Hxaw=

internal/commands/run.go

@@ -1,24 +1,26 @@
 package commands
 
 import (
+	"context"
 	"errors"
 	"github.com/cirruslabs/cirrus-cli/internal/executor"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/taskfilter"
+	"github.com/cirruslabs/cirrus-cli/pkg/larker"
+	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
 	"github.com/cirruslabs/cirrus-cli/pkg/parser"
 	"github.com/cirruslabs/echelon"
 	"github.com/cirruslabs/echelon/renderers"
 	"github.com/spf13/cobra"
+	"io/ioutil"
 	"log"
 	"os"
-	"path/filepath"
 	"strings"
 )
 
 var ErrRun = errors.New("run failed")
 
 var dirty bool
 var environment []string
-var runFile string
 var verbose bool
 
 // envArgsToMap parses and expands environment arguments like "A=B" (set operation)
@@ -51,15 +53,53 @@ func envArgsToMap(arguments []string) map[string]string {
 	return result
 }
 
+func readYAMLConfig() (string, error) {
+	yamlConfig, err := ioutil.ReadFile(".cirrus.yml")
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", nil
+		}
+		return "", err
+	}
+
+	return string(yamlConfig), nil
+}
+
+func readStarlarkConfig(ctx context.Context) (string, error) {
+	starlarkSource, err := ioutil.ReadFile(".cirrus.star")
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", nil
+		}
+		return "", err
+	}
+
+	lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(".")))
+	return lrk.Main(ctx, string(starlarkSource))
+}
+
 func run(cmd *cobra.Command, args []string) error {
 	// https://github.com/spf13/cobra/issues/340#issuecomment-374617413
 	cmd.SilenceUsage = true
 
 	envMap := envArgsToMap(environment)
 
+	// Retrieve configurations and merge them
+	yamlConfig, err := readYAMLConfig()
+	if err != nil {
+		return err
+	}
+
+	starlarkConfig, err := readStarlarkConfig(cmd.Context())
+	if err != nil {
+		return err
+	}
+
+	mergedYAML := yamlConfig + "\n" + starlarkConfig
+
 	// Parse
 	p := parser.Parser{Environment: envMap}
-	result, err := p.ParseFromFile(runFile)
+	result, err := p.Parse(mergedYAML)
 	if err != nil {
 		return err
 	}
@@ -103,7 +143,7 @@ func run(cmd *cobra.Command, args []string) error {
 	executorOpts = append(executorOpts, executor.WithEnvironment(envMap))
 
 	// Run
-	e, err := executor.New(filepath.Dir(runFile), result.Tasks, executorOpts...)
+	e, err := executor.New(".", result.Tasks, executorOpts...)
 	if err != nil {
 		return err
 	}
@@ -123,7 +163,6 @@ func newRunCmd() *cobra.Command {
 		"in read-write mode, otherwise the project directory files are copied, taking .gitignore into account")
 	cmd.PersistentFlags().StringArrayVarP(&environment, "environment", "e", []string{},
 		"set (-e A=B) or pass-through (-e A) an environment variable")
-	cmd.PersistentFlags().StringVarP(&runFile, "file", "f", ".cirrus.yml", "use file as the configuration file")
 	cmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "")
 
 	return cmd

internal/commands/run_test.go

@@ -133,3 +133,24 @@ func TestRunEnvironmentOnlyIf(t *testing.T) {
 
 	require.Nil(t, err)
 }
+
+// TestRunYAMLAndStarlarkMerged ensures that CLI merges multiple configurations.
+func TestRunYAMLAndStarlarkMerged(t *testing.T) {
+	testutil.TempChdirPopulatedWith(t, "testdata/run-yaml-and-starlark")
+
+	// Create os.Stderr writer that duplicates it's output to buf
+	buf := bytes.NewBufferString("")
+	writer := io.MultiWriter(os.Stderr, buf)
+
+	command := commands.NewRootCmd()
+	command.SetArgs([]string{"run", "-v"})
+	command.SetOut(writer)
+	command.SetErr(writer)
+	err := command.Execute()
+
+	require.Nil(t, err)
+	// nolint:misspell // https://github.com/cirruslabs/echelon/pull/7
+	assert.Contains(t, buf.String(), "'from_yaml' succeded")
+	// nolint:misspell // https://github.com/cirruslabs/echelon/pull/7
+	assert.Contains(t, buf.String(), "'from_starlark' succeded")
+}

internal/commands/testdata/run-yaml-and-starlark/.cirrus.star

@@ -0,0 +1,6 @@
+def main(ctx):
+    return [
+        {
+            "from_starlark_script": "true",
+        }
+    ]

internal/commands/testdata/run-yaml-and-starlark/.cirrus.yml

@@ -0,0 +1,5 @@
+container:
+  image: debian:latest
+
+task:
+  from_yaml_script: true

pkg/larker/context.go

@@ -0,0 +1,23 @@
+package larker
+
+import "go.starlark.net/starlark"
+
+type Context struct{}
+
+func (context *Context) String() string {
+	return "context"
+}
+
+func (context *Context) Type() string {
+	return "context"
+}
+
+func (context *Context) Freeze() {}
+
+func (context *Context) Truth() starlark.Bool {
+	return true
+}
+
+func (context *Context) Hash() (uint32, error) {
+	return 0, nil
+}

pkg/larker/convert.go

@@ -0,0 +1,49 @@
+package larker
+
+import (
+	"go.starlark.net/starlark"
+	"gopkg.in/yaml.v2"
+)
+
+func convertList(l *starlark.List) (result []interface{}) {
+	iter := l.Iterate()
+	defer iter.Done()
+
+	var listValue starlark.Value
+
+	for iter.Next(&listValue) {
+		switch value := listValue.(type) {
+		case *starlark.List:
+			result = append(result, convertList(value))
+		case *starlark.Dict:
+			result = append(result, convertDict(value))
+		default:
+			result = append(result, value)
+		}
+	}
+
+	return
+}
+
+func convertDict(d *starlark.Dict) yaml.MapSlice {
+	var slice yaml.MapSlice
+
+	for _, dictTuple := range d.Items() {
+		var sliceItem yaml.MapItem
+
+		key := dictTuple[0]
+
+		switch value := dictTuple[1].(type) {
+		case *starlark.List:
+			sliceItem = yaml.MapItem{Key: key, Value: convertList(value)}
+		case *starlark.Dict:
+			sliceItem = yaml.MapItem{Key: key, Value: convertDict(value)}
+		default:
+			sliceItem = yaml.MapItem{Key: key, Value: value}
+		}
+
+		slice = append(slice, sliceItem)
+	}
+
+	return slice
+}

pkg/larker/fs/dummy.go

@@ -0,0 +1,15 @@
+package fs
+
+import (
+	"os"
+)
+
+type Dummy struct{}
+
+func NewDummyFileSystem() *Dummy {
+	return &Dummy{}
+}
+
+func (dfs *Dummy) Get(path string) ([]byte, error) {
+	return nil, os.ErrNotExist
+}

pkg/larker/fs/fs.go

@@ -0,0 +1,5 @@
+package fs
+
+type FileSystem interface {
+	Get(path string) ([]byte, error)
+}

pkg/larker/fs/local.go

@@ -0,0 +1,25 @@
+package fs
+
+import (
+	"io/ioutil"
+	"path/filepath"
+)
+
+type Local struct {
+	root string
+}
+
+func NewLocalFileSystem(root string) *Local {
+	return &Local{
+		root: root,
+	}
+}
+
+func (lfs *Local) Get(path string) ([]byte, error) {
+	// To make Starlark scripts cross-platform, load statements are expected to always use slashes,
+	// but to actually make this work on non-Unix platforms we need to adapt the path
+	// to the current platform
+	adaptedPath := filepath.FromSlash(path)
+
+	return ioutil.ReadFile(filepath.Join(lfs.root, adaptedPath))
+}