[RD-29] Change v2 schema releases structure to keep the order of rele…

…ases when converting (#494)

.circleci/config.yml

@@ -242,14 +242,14 @@ workflows:
           name: "End-To-End Kubernetes 1.20 - Python"
           kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab"
           <<: *e2e_configuration_python
-      # - rok8s/kubernetes_e2e_tests:
-      #     name: "End-To-End Kubernetes 1.19 - Go"
-      #     kind_node_image: "kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca"
-      #     <<: *e2e_configuration_go
-      # - rok8s/kubernetes_e2e_tests:
-      #     name: "End-To-End Kubernetes 1.20 - Go"
-      #     kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab"
-      #     <<: *e2e_configuration_go
+      - rok8s/kubernetes_e2e_tests:
+          name: "End-To-End Kubernetes 1.19 - Go"
+          kind_node_image: "kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca"
+          <<: *e2e_configuration_go
+      - rok8s/kubernetes_e2e_tests:
+          name: "End-To-End Kubernetes 1.20 - Go"
+          kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab"
+          <<: *e2e_configuration_go
   release:
     jobs:
       - release:

.gitignore

@@ -17,3 +17,5 @@ node_modules
 
 coverage.txt
 /reckoner-go
+cover-report.html
+govet-report.out

cmd/root.go

@@ -43,6 +43,8 @@ var (
 	onlyRun []string
 	// createNamespaces contains the boolean flag to create namespaces
 	createNamespaces bool
+	// inPlaceConvert contains the boolean flag to update the course file in place
+	inPlaceConvert bool
 )
 
 func init() {
@@ -51,6 +53,8 @@ func init() {
 	rootCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Implies helm --dry-run --debug and skips any hooks")
 	rootCmd.PersistentFlags().BoolVar(&createNamespaces, "create-namespaces", true, "If true, allow reckoner to create namespaces.")
 
+	convertCmd.Flags().BoolVarP(&inPlaceConvert, "in-place", "i", false, "If specified, will update the file in place, otherwise outputs to stdout.")
+
 	// add commands here
 	rootCmd.AddCommand(plotCmd)
 	rootCmd.AddCommand(convertCmd)
@@ -140,19 +144,31 @@ var lintCmd = &cobra.Command{
 }
 
 var convertCmd = &cobra.Command{
-	Use:     "convert",
-	Short:   "convert <course file> from v1 to v2 schema",
-	Long:    "Converts a course file from the v1 python schema to v2 go schema",
-	PreRunE: validateArgs,
+	Use:   "convert",
+	Short: "convert <course file> from v1 to v2 schema",
+	Long:  "Converts a course file from the v1 python schema to v2 go schema",
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		runAll = true
+		return validateArgs(cmd, args)
+	},
 	Run: func(cmd *cobra.Command, args []string) {
 		newCourse, err := course.ConvertV1toV2(courseFile)
 		if err != nil {
 			klog.Fatal(err)
 		}
-		// We prefer 2 spaces in yaml
 		w := os.Stdout
+		if inPlaceConvert {
+			f, err := os.OpenFile(courseFile, os.O_RDWR, 0644)
+			if err != nil {
+				klog.Fatal(err)
+			}
+			defer f.Close()
+			f.Truncate(0)
+			w = f
+		}
 		e := yaml.NewEncoder(w)
 		defer e.Close()
+		// We prefer 2 spaces in yaml
 		e.SetIndent(2)
 
 		err = e.Encode(newCourse)

end_to_end_testing/run_go.sh

@@ -4,9 +4,9 @@ set -x
 set -e
 
 # Install Go
-curl -LO https://golang.org/dl/go1.15.8.linux-amd64.tar.gz
+curl -LO https://go.dev/dl/go1.17.3.linux-amd64.tar.gz
 
-tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz
+tar -C /usr/local -xzf go1.17.3.linux-amd64.tar.gz
 export PATH=$PATH:/usr/local/go/bin
 go version
 

go.mod

@@ -7,9 +7,9 @@ require (
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
-	github.com/thoas/go-funk v0.7.0
+	github.com/thoas/go-funk v0.9.1
 	github.com/xeipuuv/gojsonschema v1.2.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 	k8s.io/api v0.20.2
 	k8s.io/apimachinery v0.20.2
 	k8s.io/client-go v0.20.2

go.sum

@@ -356,8 +356,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/thoas/go-funk v0.7.0 h1:GmirKrs6j6zJbhJIficOsz2aAI7700KsU/5YrdHRM1Y=
-github.com/thoas/go-funk v0.7.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
+github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
+github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -649,9 +649,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/course/course.go

@@ -54,7 +54,7 @@ type FileV2 struct {
 		Default NamespaceConfig `yaml:"default" json:"default"`
 	} `yaml:"namespace_management,omitempty" json:"namespace_management,omitempty"`
 	// Releases is the list of releases that should be maintained by this course file.
-	Releases ReleaseList `yaml:"releases" json:"releases"`
+	Releases []*Release `yaml:"releases" json:"releases"`
 }
 
 // Repository is a helm reposotory definition
@@ -89,6 +89,8 @@ type NamespaceConfig struct {
 
 // Release represents a helm release and all of its configuration
 type Release struct {
+	// Name is the name of the release
+	Name string `yaml:"name" json:"name"`
 	// Namespace is the namespace that this release should be placed in
 	Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"`
 	// NamespaceMgmt is a set of labels and annotations to be added to the namespace for this release
@@ -114,11 +116,10 @@ type Release struct {
 	Values map[string]interface{} `yaml:"values,omitempty" json:"values,omitempty"`
 }
 
-// ReleaseList is a set of releases
-type ReleaseList map[string]Release
-
 // ReleaseV1 represents a helm release and all of its configuration from v1 schema
 type ReleaseV1 struct {
+	// Name is the name of the release
+	Name string `yaml:"name" json:"name"`
 	// Namespace is the namespace that this release should be placed in
 	Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"`
 	// NamespaceMgmt is a set of labels and annotations to be added to the namespace for this release
@@ -165,9 +166,12 @@ type FileV1 struct {
 		Default NamespaceConfig `yaml:"default" json:"default"`
 	} `yaml:"namespace_management" json:"namespace_management"`
 	// Charts is the map of releases
-	Charts map[string]ReleaseV1 `yaml:"charts" json:"charts"`
+	Charts ChartsListV1 `yaml:"charts" json:"charts"`
 }
 
+// ChartsListV1 is a list of charts for the v1 schema
+type ChartsListV1 []ReleaseV1
+
 // RepositoryV1 is a helm reposotory definition
 type RepositoryV1 struct {
 	Name string `yaml:"name,omitempty" json:"name,omitempty"`
@@ -196,11 +200,11 @@ func ConvertV1toV2(fileName string) (*FileV2, error) {
 	newFile.NamespaceMgmt = oldFile.NamespaceMgmt
 	newFile.DefaultRepository = oldFile.DefaultRepository
 	newFile.Repositories = oldFile.Repositories
-	newFile.Releases = make(map[string]Release)
+	newFile.Releases = make([]*Release, len(oldFile.Charts))
 	newFile.Hooks = oldFile.Hooks
 	newFile.MinimumVersions = oldFile.MinimumVersions
 
-	for releaseName, release := range oldFile.Charts {
+	for releaseIndex, release := range oldFile.Charts {
 		repositoryName, ok := release.Repository.(string)
 		// The repository is not in the format repository: string. Need to handle that
 		if !ok {
@@ -217,7 +221,7 @@ func ConvertV1toV2(fileName string) (*FileV2, error) {
 			if addRepo.Git != "" {
 				klog.V(3).Infof("detected a git-based inline repository. Attempting to convert to repository in header")
 
-				repositoryName = fmt.Sprintf("%s-git-repository", releaseName)
+				repositoryName = fmt.Sprintf("%s-git-repository", release.Name)
 				newFile.Repositories[repositoryName] = Repository{
 					Git:  addRepo.Git,
 					Path: addRepo.Path,
@@ -228,7 +232,8 @@ func ConvertV1toV2(fileName string) (*FileV2, error) {
 				repositoryName = addRepo.Name
 			}
 		}
-		newFile.Releases[releaseName] = Release{
+		newFile.Releases[releaseIndex] = &Release{
+			Name:          release.Name,
 			Namespace:     release.Namespace,
 			NamespaceMgmt: release.NamespaceMgmt,
 			Repository:    repositoryName,
@@ -293,18 +298,36 @@ func OpenCourseV1(fileName string) (*FileV1, error) {
 	return courseFile, nil
 }
 
+// UnmarshalYAML implements the yaml.Unmarshaler interface to customize how we Unmarshal this particular field of the FileV1 struct
+func (cl *ChartsListV1) UnmarshalYAML(value *yaml.Node) error {
+	if value.Kind != yaml.MappingNode {
+		return fmt.Errorf("ChartsList must contain YAML mapping, has %v", value.Kind)
+	}
+	*cl = make([]ReleaseV1, len(value.Content)/2)
+	for i := 0; i < len(value.Content); i += 2 {
+		var res = &(*cl)[i/2]
+		if err := value.Content[i].Decode(&res.Name); err != nil {
+			return err
+		}
+		if err := value.Content[i+1].Decode(&res); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // populateDefaultNamespace sets the default namespace in each release
 // if the release does not have a namespace. If the DefaultNamespace is blank, simply returns
 func (f *FileV2) populateDefaultNamespace() {
 	if f.DefaultNamespace == "" {
 		klog.V(2).Info("no default namespace set - skipping filling out defaults")
 		return
 	}
-	for releaseName, release := range f.Releases {
+	for releaseIndex, release := range f.Releases {
 		if release.Namespace == "" {
-			klog.V(5).Infof("setting the default namespace of %s on release %s", f.DefaultNamespace, releaseName)
+			klog.V(5).Infof("setting the default namespace of %s on release %s", f.DefaultNamespace, release.Name)
 			release.Namespace = f.DefaultNamespace
-			f.Releases[releaseName] = release
+			f.Releases[releaseIndex] = release
 		}
 	}
 }
@@ -316,22 +339,22 @@ func (f *FileV2) populateDefaultRepository() {
 		klog.V(2).Info("no default repository set - skipping filling out defaults")
 		return
 	}
-	for releaseName, release := range f.Releases {
+	for releaseIndex, release := range f.Releases {
 		if release.Repository == "" {
-			klog.V(5).Infof("setting the default repository of %s on release %s", f.DefaultRepository, releaseName)
+			klog.V(5).Infof("setting the default repository of %s on release %s", f.DefaultRepository, release.Name)
 			release.Repository = f.DefaultRepository
-			f.Releases[releaseName] = release
+			f.Releases[releaseIndex] = release
 		}
 	}
 }
 
 // populateEmptyChartNames assumes that the chart name should be the release name if the chart name is empty
 func (f *FileV2) populateEmptyChartNames() {
-	for releaseName, release := range f.Releases {
+	for releaseIndex, release := range f.Releases {
 		if release.Chart == "" {
-			klog.V(5).Infof("assuming chart name is release name for release: %s", releaseName)
-			release.Chart = releaseName
-			f.Releases[releaseName] = release
+			klog.V(5).Infof("assuming chart name is release name for release: %s", release.Name)
+			release.Chart = release.Name
+			f.Releases[releaseIndex] = release
 		}
 	}
 }