diff --git a/.circleci/config.yml b/.circleci/config.yml index 78034e7a..990dac6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -242,14 +242,14 @@ workflows: name: "End-To-End Kubernetes 1.20 - Python" kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab" <<: *e2e_configuration_python - # - rok8s/kubernetes_e2e_tests: - # name: "End-To-End Kubernetes 1.19 - Go" - # kind_node_image: "kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca" - # <<: *e2e_configuration_go - # - rok8s/kubernetes_e2e_tests: - # name: "End-To-End Kubernetes 1.20 - Go" - # kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab" - # <<: *e2e_configuration_go + - rok8s/kubernetes_e2e_tests: + name: "End-To-End Kubernetes 1.19 - Go" + kind_node_image: "kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca" + <<: *e2e_configuration_go + - rok8s/kubernetes_e2e_tests: + name: "End-To-End Kubernetes 1.20 - Go" + kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab" + <<: *e2e_configuration_go release: jobs: - release: diff --git a/.gitignore b/.gitignore index 6679d416..188bd5c3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ node_modules coverage.txt /reckoner-go +cover-report.html +govet-report.out diff --git a/cmd/root.go b/cmd/root.go index 273f9037..2263f830 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -43,6 +43,8 @@ var ( onlyRun []string // createNamespaces contains the boolean flag to create namespaces createNamespaces bool + // inPlaceConvert contains the boolean flag to update the course file in place + inPlaceConvert bool ) func init() { @@ -51,6 +53,8 @@ func init() { rootCmd.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Implies helm --dry-run --debug and skips any hooks") rootCmd.PersistentFlags().BoolVar(&createNamespaces, "create-namespaces", true, "If true, allow reckoner to create namespaces.") + convertCmd.Flags().BoolVarP(&inPlaceConvert, "in-place", "i", false, "If specified, will update the file in place, otherwise outputs to stdout.") + // add commands here rootCmd.AddCommand(plotCmd) rootCmd.AddCommand(convertCmd) @@ -140,19 +144,31 @@ var lintCmd = &cobra.Command{ } var convertCmd = &cobra.Command{ - Use: "convert", - Short: "convert from v1 to v2 schema", - Long: "Converts a course file from the v1 python schema to v2 go schema", - PreRunE: validateArgs, + Use: "convert", + Short: "convert from v1 to v2 schema", + Long: "Converts a course file from the v1 python schema to v2 go schema", + PreRunE: func(cmd *cobra.Command, args []string) error { + runAll = true + return validateArgs(cmd, args) + }, Run: func(cmd *cobra.Command, args []string) { newCourse, err := course.ConvertV1toV2(courseFile) if err != nil { klog.Fatal(err) } - // We prefer 2 spaces in yaml w := os.Stdout + if inPlaceConvert { + f, err := os.OpenFile(courseFile, os.O_RDWR, 0644) + if err != nil { + klog.Fatal(err) + } + defer f.Close() + f.Truncate(0) + w = f + } e := yaml.NewEncoder(w) defer e.Close() + // We prefer 2 spaces in yaml e.SetIndent(2) err = e.Encode(newCourse) diff --git a/end_to_end_testing/run_go.sh b/end_to_end_testing/run_go.sh index b311ec42..06bfbc70 100644 --- a/end_to_end_testing/run_go.sh +++ b/end_to_end_testing/run_go.sh @@ -4,9 +4,9 @@ set -x set -e # Install Go -curl -LO https://golang.org/dl/go1.15.8.linux-amd64.tar.gz +curl -LO https://go.dev/dl/go1.17.3.linux-amd64.tar.gz -tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz +tar -C /usr/local -xzf go1.17.3.linux-amd64.tar.gz export PATH=$PATH:/usr/local/go/bin go version diff --git a/go.mod b/go.mod index 64510153..cdcc9272 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 - github.com/thoas/go-funk v0.7.0 + github.com/thoas/go-funk v0.9.1 github.com/xeipuuv/gojsonschema v1.2.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 k8s.io/api v0.20.2 k8s.io/apimachinery v0.20.2 k8s.io/client-go v0.20.2 diff --git a/go.sum b/go.sum index e19a4281..061ad58d 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/thoas/go-funk v0.7.0 h1:GmirKrs6j6zJbhJIficOsz2aAI7700KsU/5YrdHRM1Y= -github.com/thoas/go-funk v0.7.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -649,9 +649,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/course/course.go b/pkg/course/course.go index 823d51c7..49c8269c 100644 --- a/pkg/course/course.go +++ b/pkg/course/course.go @@ -54,7 +54,7 @@ type FileV2 struct { Default NamespaceConfig `yaml:"default" json:"default"` } `yaml:"namespace_management,omitempty" json:"namespace_management,omitempty"` // Releases is the list of releases that should be maintained by this course file. - Releases ReleaseList `yaml:"releases" json:"releases"` + Releases []*Release `yaml:"releases" json:"releases"` } // Repository is a helm reposotory definition @@ -89,6 +89,8 @@ type NamespaceConfig struct { // Release represents a helm release and all of its configuration type Release struct { + // Name is the name of the release + Name string `yaml:"name" json:"name"` // Namespace is the namespace that this release should be placed in Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` // NamespaceMgmt is a set of labels and annotations to be added to the namespace for this release @@ -114,11 +116,10 @@ type Release struct { Values map[string]interface{} `yaml:"values,omitempty" json:"values,omitempty"` } -// ReleaseList is a set of releases -type ReleaseList map[string]Release - // ReleaseV1 represents a helm release and all of its configuration from v1 schema type ReleaseV1 struct { + // Name is the name of the release + Name string `yaml:"name" json:"name"` // Namespace is the namespace that this release should be placed in Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` // NamespaceMgmt is a set of labels and annotations to be added to the namespace for this release @@ -165,9 +166,12 @@ type FileV1 struct { Default NamespaceConfig `yaml:"default" json:"default"` } `yaml:"namespace_management" json:"namespace_management"` // Charts is the map of releases - Charts map[string]ReleaseV1 `yaml:"charts" json:"charts"` + Charts ChartsListV1 `yaml:"charts" json:"charts"` } +// ChartsListV1 is a list of charts for the v1 schema +type ChartsListV1 []ReleaseV1 + // RepositoryV1 is a helm reposotory definition type RepositoryV1 struct { Name string `yaml:"name,omitempty" json:"name,omitempty"` @@ -196,11 +200,11 @@ func ConvertV1toV2(fileName string) (*FileV2, error) { newFile.NamespaceMgmt = oldFile.NamespaceMgmt newFile.DefaultRepository = oldFile.DefaultRepository newFile.Repositories = oldFile.Repositories - newFile.Releases = make(map[string]Release) + newFile.Releases = make([]*Release, len(oldFile.Charts)) newFile.Hooks = oldFile.Hooks newFile.MinimumVersions = oldFile.MinimumVersions - for releaseName, release := range oldFile.Charts { + for releaseIndex, release := range oldFile.Charts { repositoryName, ok := release.Repository.(string) // The repository is not in the format repository: string. Need to handle that if !ok { @@ -217,7 +221,7 @@ func ConvertV1toV2(fileName string) (*FileV2, error) { if addRepo.Git != "" { klog.V(3).Infof("detected a git-based inline repository. Attempting to convert to repository in header") - repositoryName = fmt.Sprintf("%s-git-repository", releaseName) + repositoryName = fmt.Sprintf("%s-git-repository", release.Name) newFile.Repositories[repositoryName] = Repository{ Git: addRepo.Git, Path: addRepo.Path, @@ -228,7 +232,8 @@ func ConvertV1toV2(fileName string) (*FileV2, error) { repositoryName = addRepo.Name } } - newFile.Releases[releaseName] = Release{ + newFile.Releases[releaseIndex] = &Release{ + Name: release.Name, Namespace: release.Namespace, NamespaceMgmt: release.NamespaceMgmt, Repository: repositoryName, @@ -293,6 +298,24 @@ func OpenCourseV1(fileName string) (*FileV1, error) { return courseFile, nil } +// UnmarshalYAML implements the yaml.Unmarshaler interface to customize how we Unmarshal this particular field of the FileV1 struct +func (cl *ChartsListV1) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.MappingNode { + return fmt.Errorf("ChartsList must contain YAML mapping, has %v", value.Kind) + } + *cl = make([]ReleaseV1, len(value.Content)/2) + for i := 0; i < len(value.Content); i += 2 { + var res = &(*cl)[i/2] + if err := value.Content[i].Decode(&res.Name); err != nil { + return err + } + if err := value.Content[i+1].Decode(&res); err != nil { + return err + } + } + return nil +} + // populateDefaultNamespace sets the default namespace in each release // if the release does not have a namespace. If the DefaultNamespace is blank, simply returns func (f *FileV2) populateDefaultNamespace() { @@ -300,11 +323,11 @@ func (f *FileV2) populateDefaultNamespace() { klog.V(2).Info("no default namespace set - skipping filling out defaults") return } - for releaseName, release := range f.Releases { + for releaseIndex, release := range f.Releases { if release.Namespace == "" { - klog.V(5).Infof("setting the default namespace of %s on release %s", f.DefaultNamespace, releaseName) + klog.V(5).Infof("setting the default namespace of %s on release %s", f.DefaultNamespace, release.Name) release.Namespace = f.DefaultNamespace - f.Releases[releaseName] = release + f.Releases[releaseIndex] = release } } } @@ -316,22 +339,22 @@ func (f *FileV2) populateDefaultRepository() { klog.V(2).Info("no default repository set - skipping filling out defaults") return } - for releaseName, release := range f.Releases { + for releaseIndex, release := range f.Releases { if release.Repository == "" { - klog.V(5).Infof("setting the default repository of %s on release %s", f.DefaultRepository, releaseName) + klog.V(5).Infof("setting the default repository of %s on release %s", f.DefaultRepository, release.Name) release.Repository = f.DefaultRepository - f.Releases[releaseName] = release + f.Releases[releaseIndex] = release } } } // populateEmptyChartNames assumes that the chart name should be the release name if the chart name is empty func (f *FileV2) populateEmptyChartNames() { - for releaseName, release := range f.Releases { + for releaseIndex, release := range f.Releases { if release.Chart == "" { - klog.V(5).Infof("assuming chart name is release name for release: %s", releaseName) - release.Chart = releaseName - f.Releases[releaseName] = release + klog.V(5).Infof("assuming chart name is release name for release: %s", release.Name) + release.Chart = release.Name + f.Releases[releaseIndex] = release } } } diff --git a/pkg/course/course_test.go b/pkg/course/course_test.go index bbd0a1d1..4e61719a 100644 --- a/pkg/course/course_test.go +++ b/pkg/course/course_test.go @@ -61,8 +61,9 @@ func TestConvertV1toV2(t *testing.T) { Path: "gitpath", }, }, - Releases: ReleaseList{ - "basic": { + Releases: []*Release{ + { + Name: "basic", Chart: "somechart", Version: "2.0.0", Repository: "helm-repo", @@ -70,13 +71,15 @@ func TestConvertV1toV2(t *testing.T) { "dummyvalue": false, }, }, - "gitrelease": { + { + Name: "gitrelease", Chart: "gitchart", Version: "main", Repository: "gitrelease-git-repository", Values: nil, }, - "standard": { + { + Name: "standard", Chart: "basic", Repository: "helm-repo", Values: nil, @@ -101,25 +104,27 @@ func TestConvertV1toV2(t *testing.T) { func TestFileV2_populateDefaultNamespace(t *testing.T) { type fields struct { DefaultNamespace string - Releases ReleaseList + Releases []*Release } tests := []struct { name string fields fields - want ReleaseList + want []*Release }{ { name: "basic test", fields: fields{ DefaultNamespace: "default-ns", - Releases: map[string]Release{ - "first-release": { + Releases: []*Release{ + { + Name: "first-release", Chart: "farglebargle", }, }, }, - want: map[string]Release{ - "first-release": { + want: []*Release{ + { + Name: "first-release", Chart: "farglebargle", Namespace: "default-ns", }, @@ -129,14 +134,16 @@ func TestFileV2_populateDefaultNamespace(t *testing.T) { name: "no default namespace", fields: fields{ DefaultNamespace: "", - Releases: map[string]Release{ - "first-release": { + Releases: []*Release{ + { + Name: "first-release", Chart: "farglebargle", }, }, }, - want: map[string]Release{ - "first-release": { + want: []*Release{ + { + Name: "first-release", Chart: "farglebargle", }, }, @@ -157,25 +164,27 @@ func TestFileV2_populateDefaultNamespace(t *testing.T) { func TestFileV2_populateDefaultRepository(t *testing.T) { type fields struct { DefaultRepository string - Releases ReleaseList + Releases []*Release } tests := []struct { name string fields fields - want ReleaseList + want []*Release }{ { name: "basic test", fields: fields{ DefaultRepository: "default-repo", - Releases: map[string]Release{ - "first-release": { + Releases: []*Release{ + { + Name: "first-release", Chart: "farglebargle", }, }, }, - want: map[string]Release{ - "first-release": { + want: []*Release{ + { + Name: "first-release", Chart: "farglebargle", Repository: "default-repo", }, @@ -185,14 +194,16 @@ func TestFileV2_populateDefaultRepository(t *testing.T) { name: "no default set", fields: fields{ DefaultRepository: "", - Releases: map[string]Release{ - "first-release": { + Releases: []*Release{ + { + Name: "first-release", Chart: "farglebargle", }, }, }, - want: map[string]Release{ - "first-release": { + want: []*Release{ + { + Name: "first-release", Chart: "farglebargle", }, }, diff --git a/pkg/course/coursev2.schema.json b/pkg/course/coursev2.schema.json index 742452af..5eb6a764 100644 --- a/pkg/course/coursev2.schema.json +++ b/pkg/course/coursev2.schema.json @@ -42,15 +42,16 @@ } }, "release": { - "type": "object", - "propertyNames": { - "pattern": "^[a-zA-Z0-9_-]{1,63}$", - "x-custom-error-message": "Chart release names must be alphanumeric with \"_\" and \"-\" and be between 1 and 63 characters" - }, + "type": "array", "additionalProperties": { "type": "object", "additionalProperties": false, "properties": { + "name": { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]{1,63}$", + "x-custom-error-message": "Chart release names must be alphanumeric with \"_\" and \"-\" and be between 1 and 63 characters" + }, "namespace": { "type": "string" }, diff --git a/pkg/reckoner/client.go b/pkg/reckoner/client.go index d16bf581..88f27f7b 100644 --- a/pkg/reckoner/client.go +++ b/pkg/reckoner/client.go @@ -65,13 +65,14 @@ func NewClient(fileName, version string, plotAll bool, releases []string, kubeCl } client := &Client{ - CourseFile: *courseFile, - PlotAll: plotAll, - Releases: releases, - Helm: *helmClient, - ReckonerVersion: version, - BaseDirectory: path.Dir(fileName), - DryRun: dryRun, + CourseFile: *courseFile, + PlotAll: plotAll, + Releases: releases, + Helm: *helmClient, + ReckonerVersion: version, + BaseDirectory: path.Dir(fileName), + DryRun: dryRun, + CreateNamespaces: createNamespaces, } // Check versions @@ -180,12 +181,18 @@ func (c Client) UpdateHelmRepos() error { func (c *Client) filterReleases() error { releases := c.CourseFile.Releases if len(c.Releases) > 0 { - selectedReleases := make(map[string]course.Release) + selectedReleases := []*course.Release{} for _, releaseName := range c.Releases { - if !funk.Contains(c.CourseFile.Releases, releaseName) { + contained := funk.Contains(c.CourseFile.Releases, func(rel *course.Release) bool { + return rel.Name == releaseName + }) + if !contained { continue } - selectedReleases[releaseName] = c.CourseFile.Releases[releaseName] + releaseIndex := funk.IndexOf(c.CourseFile.Releases, func(rel *course.Release) bool { + return rel.Name == releaseName + }) + selectedReleases = append(selectedReleases, c.CourseFile.Releases[releaseIndex]) } releases = selectedReleases } diff --git a/pkg/reckoner/plot.go b/pkg/reckoner/plot.go index b2fc89b0..392c1088 100644 --- a/pkg/reckoner/plot.go +++ b/pkg/reckoner/plot.go @@ -43,14 +43,14 @@ func (c Client) Plot() (string, error) { return "", err } - for releaseName, release := range c.CourseFile.Releases { + for _, release := range c.CourseFile.Releases { err = c.execHook(release.Hooks.PreInstall) if err != nil { return "", err } - args, tmpFile, err := buildHelmArgs(releaseName, "upgrade", release) + args, tmpFile, err := buildHelmArgs(release.Name, "upgrade", *release) if err != nil { klog.Error(err) continue @@ -93,8 +93,8 @@ func (c Client) Template() (string, error) { } var fullOutput string - for releaseName, release := range c.CourseFile.Releases { - args, tmpFile, err := buildHelmArgs(releaseName, "template", release) + for _, release := range c.CourseFile.Releases { + args, tmpFile, err := buildHelmArgs(release.Name, "template", *release) if err != nil { klog.Error(err) continue