From cc8c382675cf986697683f796d3a8edb2750fd15 Mon Sep 17 00:00:00 2001 From: Luke Reed Date: Mon, 28 Feb 2022 11:49:45 -0500 Subject: [PATCH 1/8] many changes for refactoring and e2e tests --- .circleci/config.yml | 29 ++- cmd/root.go | 29 ++- .../05_test_exit_on_post_install_hook.yml | 3 +- end_to_end_testing/tests/04_bad_chart.yaml | 4 +- end_to_end_testing/tests/07_good_hooks.yaml | 5 +- end_to_end_testing/tests/13_schema.yaml | 2 +- end_to_end_testing/tests/21_diff.yaml | 8 +- go.mod | 2 +- pkg/course/course.go | 193 ++++++++++++------ pkg/course/course_test.go | 2 +- pkg/helm/helm.go | 10 + pkg/reckoner/client.go | 54 +++-- pkg/reckoner/diff.go | 12 +- pkg/reckoner/git.go | 3 +- pkg/reckoner/hook.go | 7 +- pkg/reckoner/namespace.go | 9 +- pkg/reckoner/namespace_test.go | 8 +- pkg/reckoner/plot.go | 59 ++++-- pkg/reckoner/plot_test.go | 5 +- pkg/reckoner/update.go | 12 +- 20 files changed, 310 insertions(+), 146 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a7e7fc1a..b6bd4f91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,6 @@ jobs: pip install --user -e . reckoner --version pytest --cov ./ - build-3-8: executor: python-3-8 working_directory: ~/reckoner @@ -136,15 +135,13 @@ jobs: snapshot-go: working_directory: /go/src/github.com/fairwindsops/reckoner docker: - - image: quay.io/reactiveops/ci-images:v11-buster + - image: goreleaser/goreleaser:v1.5.0 steps: - checkout - - *install_go - - *install_goreleaser - setup_remote_docker - run: name: Goreleaser Snapshot - command: goreleaser --snapshot + command: goreleaser --snapshot --skip-sign - store_artifacts: path: dist destination: snapshot @@ -186,18 +183,32 @@ jobs: npm run check-links npm run build - run: - name: Install AWS CLI + name: Install Tools command: | + cd /tmp + echo "Installing AWS CLI" curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip sudo ./aws/install + echo "Installing Hashicorp Vault" + curl -LO https://releases.hashicorp.com/vault/1.9.3/vault_1.9.3_linux_amd64.zip + unzip vault_1.9.3_linux_amd64.zip + sudo mv vault /usr/bin/vault + sudo chmod +x /usr/bin/vault + vault --version + echo "Installing yq" + curl -LO https://github.com/mikefarah/yq/releases/download/v4.16.2/yq_linux_amd64.tar.gz + tar -zxvf yq_linux_amd64.tar.gz + sudo mv yq_linux_amd64 /usr/bin/yq + sudo chmod +x /usr/bin/yq + yq --version + - rok8s/get_vault_env: + vault_path: repo/pluto/env - run: name: Publish Docs Site to S3 command: | cd ./dist - aws s3 sync ./ s3://reckoner.docs.fairwinds.com --delete - - + aws s3 sync ./ s3://pluto.docs.fairwinds.com --delete workflows: version: 2 build_and_test: diff --git a/cmd/root.go b/cmd/root.go index 40d00705..8db3d03a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -44,6 +44,8 @@ var ( onlyRun []string // createNamespaces contains the boolean flag to create namespaces createNamespaces bool + // continueOnError contains the boolean flag to continue processing releases even if one errors + continueOnError bool // inPlaceConvert contains the boolean flag to update the course file in place inPlaceConvert bool // noColor contains the boolean flag to disable color output @@ -57,6 +59,10 @@ func init() { rootCmd.PersistentFlags().BoolVar(&createNamespaces, "create-namespaces", true, "If true, allow reckoner to create namespaces.") rootCmd.PersistentFlags().BoolVar(&noColor, "no-color", false, "If true, don't colorize output.") + plotCmd.PersistentFlags().BoolVar(&continueOnError, "continue-on-error", false, "If true, continue plotting releases even if one or more has errors.") + updateCmd.PersistentFlags().BoolVar(&continueOnError, "continue-on-error", false, "If true, continue plotting releases even if one or more has errors.") + diffCmd.PersistentFlags().BoolVar(&continueOnError, "continue-on-error", false, "If true, continue plotting releases even if one or more has errors.") + convertCmd.Flags().BoolVarP(&inPlaceConvert, "in-place", "i", false, "If specified, will update the file in place, otherwise outputs to stdout.") // add commands here @@ -92,7 +98,7 @@ var plotCmd = &cobra.Command{ Long: "Runs a helm install on a release or several releases.", PreRunE: validateArgs, Run: func(cmd *cobra.Command, args []string) { - client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, true, dryRun, createNamespaces, courseSchema) + client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, true, dryRun, createNamespaces, courseSchema, continueOnError) if err != nil { color.Red(err.Error()) os.Exit(1) @@ -102,6 +108,9 @@ var plotCmd = &cobra.Command{ color.Red(err.Error()) os.Exit(1) } + if client.Errors > 0 { + os.Exit(1) + } }, } @@ -111,7 +120,7 @@ var templateCmd = &cobra.Command{ Long: "Templates a helm chart for a release or several releases. Automatically sets --create-namespaces=false --dry-run=true", PreRunE: validateArgs, Run: func(cmd *cobra.Command, args []string) { - client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema) + client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, continueOnError) if err != nil { color.Red(err.Error()) os.Exit(1) @@ -131,7 +140,7 @@ var getManifestsCmd = &cobra.Command{ Long: "Gets the manifests currently in the cluster.", PreRunE: validateArgs, Run: func(cmd *cobra.Command, args []string) { - client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema) + client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, continueOnError) if err != nil { color.Red(err.Error()) os.Exit(1) @@ -151,7 +160,7 @@ var diffCmd = &cobra.Command{ Long: "Diffs the currently defined release and the one in the cluster", PreRunE: validateArgs, Run: func(cmd *cobra.Command, args []string) { - client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema) + client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, continueOnError) if err != nil { color.Red(err.Error()) os.Exit(1) @@ -165,6 +174,9 @@ var diffCmd = &cobra.Command{ color.Red(err.Error()) os.Exit(1) } + if client.Errors > 0 { + os.Exit(1) + } }, } @@ -177,12 +189,12 @@ var lintCmd = &cobra.Command{ return validateArgs(cmd, args) }, Run: func(cmd *cobra.Command, args []string) { - _, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema) + _, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, continueOnError) if err != nil { color.Red(err.Error()) os.Exit(1) } - klog.Infof("course file %s is good to go!", courseFile) + color.Green("No schema validation errors found in course file: %s", courseFile) }, } @@ -234,7 +246,7 @@ var updateCmd = &cobra.Command{ Long: "Only install/upgrade a release if there are changes.", PreRunE: validateArgs, Run: func(cmd *cobra.Command, args []string) { - client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, true, dryRun, createNamespaces, courseSchema) + client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, true, dryRun, createNamespaces, courseSchema, continueOnError) if err != nil { color.Red(err.Error()) os.Exit(1) @@ -244,6 +256,9 @@ var updateCmd = &cobra.Command{ color.Red(err.Error()) os.Exit(1) } + if client.Errors > 0 { + os.Exit(1) + } }, } diff --git a/end_to_end_testing/course_files/05_test_exit_on_post_install_hook.yml b/end_to_end_testing/course_files/05_test_exit_on_post_install_hook.yml index e3acb663..0a448675 100644 --- a/end_to_end_testing/course_files/05_test_exit_on_post_install_hook.yml +++ b/end_to_end_testing/course_files/05_test_exit_on_post_install_hook.yml @@ -12,4 +12,5 @@ charts: hooks: pre_install: - whoami - post_install: exit 1 + post_install: + - exit 1 diff --git a/end_to_end_testing/tests/04_bad_chart.yaml b/end_to_end_testing/tests/04_bad_chart.yaml index 5a4bf00a..4ff83d79 100644 --- a/end_to_end_testing/tests/04_bad_chart.yaml +++ b/end_to_end_testing/tests/04_bad_chart.yaml @@ -18,5 +18,5 @@ testcases: - name: 04 - cleanup steps: - script: | - helm -n {{.namespace}} delete {{.release}} - kubectl delete ns {{.namespace}} + helm -n {{.namespace}} delete {{.release}} || true + kubectl delete ns {{.namespace}} || true diff --git a/end_to_end_testing/tests/07_good_hooks.yaml b/end_to_end_testing/tests/07_good_hooks.yaml index acc23bf2..af106c99 100644 --- a/end_to_end_testing/tests/07_good_hooks.yaml +++ b/end_to_end_testing/tests/07_good_hooks.yaml @@ -11,9 +11,8 @@ testcases: reckoner plot -a {{.course}} assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring "Returned stdout" - - result.systemerr ShouldContainSubstring "Release basic-demo post install hook ran successfully" - - result.systemerr ShouldContainSubstring "Release basic-demo pre install hook ran successfully" + - result.systemout ShouldContainSubstring "Running release pre hook" + - result.systemout ShouldContainSubstring "Running release post hook" - script: | helm -n {{.namespace}} get all {{.release}} assertions: diff --git a/end_to_end_testing/tests/13_schema.yaml b/end_to_end_testing/tests/13_schema.yaml index 525662b0..254872a9 100644 --- a/end_to_end_testing/tests/13_schema.yaml +++ b/end_to_end_testing/tests/13_schema.yaml @@ -23,7 +23,7 @@ testcases: reckoner lint ../course_files/13_test_lint_bad_secret.yml assertions: - result.code ShouldEqual 1 - - result.systemout ShouldContainSubstring 'Course file has schema validation errors.' + - result.systemout ShouldContainSubstring 'Course file has schema validation errors' - name: 13 - lint 13_test_lint_good_secret.yml steps: diff --git a/end_to_end_testing/tests/21_diff.yaml b/end_to_end_testing/tests/21_diff.yaml index 433e3ae7..b1f35c8b 100644 --- a/end_to_end_testing/tests/21_diff.yaml +++ b/end_to_end_testing/tests/21_diff.yaml @@ -14,16 +14,12 @@ testcases: reckoner diff {{.course}} -o chart-with-namespace assertions: - result.code ShouldEqual 0 - # Python logs to stderr - - result.systemerr ShouldContainSubstring "There are no differences in release" - - result.systemout ShouldEqual "" + - result.systemout ShouldContainSubstring "There are no differences in release" - script: | reckoner diff {{.course}} -o chart-without-namespace assertions: - result.code ShouldEqual 0 - # Python logs to stderr - - result.systemerr ShouldContainSubstring "There are no differences in release" - - result.systemout ShouldEqual "" + - result.systemout ShouldContainSubstring "There are no differences in release" - name: 21 - cleanup steps: - script: | diff --git a/go.mod b/go.mod index d2bc1ef5..213da0a1 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v2 v2.4.0 k8s.io/klog/v2 v2.30.0 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect diff --git a/pkg/course/course.go b/pkg/course/course.go index 25e06a01..f86c44e4 100644 --- a/pkg/course/course.go +++ b/pkg/course/course.go @@ -16,17 +16,26 @@ package course import ( "encoding/json" + "errors" "fmt" "io/ioutil" "os" + "strconv" "strings" "github.com/fatih/color" + "github.com/thoas/go-funk" "github.com/xeipuuv/gojsonschema" + + // "gopkg.in/yaml.v2" "gopkg.in/yaml.v3" "k8s.io/klog/v2" ) +var ( + SchemaValidationError error = errors.New("Course file has schema validation errors") +) + // FileV2 is the heart of reckoner, it contains the definitions of the releases to be installed // as well as all other configuration. type FileV2 struct { @@ -34,15 +43,15 @@ type FileV2 struct { SchemaVersion string `yaml:"schema,omitempty" json:"schema,omitempty"` // DefaultNamespace is the namespace that releases will be installed into if // a namespace is not specified on the Release - DefaultNamespace string `yaml:"namespace" json:"namespace"` + DefaultNamespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` // DefaultRepository is the default repository that the release will be installed // from if one is not specified on the Release - DefaultRepository string `yaml:"repository" json:"repository"` + DefaultRepository string `yaml:"repository,omitempty" json:"repository,omitempty"` // Context is the kubeconfig context to use when installing // if that context is not available, then reckoner should fail Context string `yaml:"context,omitempty" json:"context,omitempty"` // Repositories is a list of helm repositories that can be used to look for charts - Repositories RepositoryList `yaml:"repositories" json:"repositories"` + Repositories RepositoryMap `yaml:"repositories,omitempty" json:"repositories,omitempty"` // MinimumVersions is a block that restricts this course file from being used with // outdated versions of helm or reckoner MinimumVersions struct { @@ -54,12 +63,15 @@ type FileV2 struct { // NamespaceMgmt contains the default namespace config for all namespaces managed by this course. NamespaceMgmt struct { // Default is the default namespace config for this course - Default NamespaceConfig `yaml:"default" json:"default"` + Default *NamespaceConfig `yaml:"default" json:"default"` } `yaml:"namespace_management,omitempty" json:"namespace_management,omitempty"` // Releases is the list of releases that should be maintained by this course file. - Releases []*Release `yaml:"releases" json:"releases"` + Releases []*Release `yaml:"releases,omitempty" json:"releases,omitempty"` } +// FileV2Unmarshal is a helper type that allows us to have a custom unmarshal function for the FileV2 struct +type FileV2Unmarshal FileV2 + // Repository is a helm reposotory definition type Repository struct { URL string `yaml:"url,omitempty" json:"url,omitempty"` @@ -67,8 +79,8 @@ type Repository struct { Path string `yaml:"path,omitempty" json:"path,omitempty"` } -// RepositoryList is a set of repositories -type RepositoryList map[string]Repository +// RepositoryMap is a set of repositories +type RepositoryMap map[string]Repository // Hooks are a set of short scripts to run before or after installation type Hooks struct { @@ -92,14 +104,18 @@ type NamespaceConfig struct { // Release represents a helm release and all of its configuration type Release struct { - GitClonePath *string - GitChartSubPath *string + // GitClonePath is the path where the repository should be cloned into + // ignored when parsing to and from yaml or json + GitClonePath *string `yaml:"-" json:"-"` + // GitChartSubPath is the sub path of the repository where the chart is located after being cloned + // ignored when parsing to and from yaml or json + GitChartSubPath *string `yaml:"-" json:"-"` // Name is the name of the release Name string `yaml:"name" json:"name"` // Namespace is the namespace that this release should be placed in Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` // NamespaceMgmt is a set of labels and annotations to be added to the namespace for this release - NamespaceMgmt NamespaceConfig `yaml:"namespace_management,omitempty" json:"namespace_management,omitempty"` + NamespaceMgmt *NamespaceConfig `yaml:"namespace_management,omitempty" json:"namespace_management,omitempty"` // Chart is the name of the chart used by this release. // If empty, then the release name is assumed to be the chart. Chart string `yaml:"chart,omitempty" json:"chart,omitempty"` @@ -123,12 +139,10 @@ type Release struct { // ReleaseV1 represents a helm release and all of its configuration from v1 schema type ReleaseV1 struct { - // Name is the name of the release - Name string `yaml:"name" json:"name"` // Namespace is the namespace that this release should be placed in Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` // NamespaceMgmt is a set of labels and annotations to be added to the namespace for this release - NamespaceMgmt NamespaceConfig `yaml:"namespace_management,omitempty" json:"namespace_management,omitempty"` + NamespaceMgmt *NamespaceConfig `yaml:"namespace_management,omitempty" json:"namespace_management,omitempty"` // Chart is the name of the chart used by this release Chart string `yaml:"chart" json:"chart"` // Hooks are pre and post install hooks @@ -156,7 +170,7 @@ type FileV1 struct { // if that context is not available, then reckoner should fail Context string `yaml:"context" json:"context"` // Repositories is a list of helm repositories that can be used to look for charts - Repositories RepositoryList `yaml:"repositories" json:"repositories"` + Repositories RepositoryMap `yaml:"repositories" json:"repositories"` // MinimumVersions is a block that restricts this course file from being used with // outdated versions of helm or reckoner MinimumVersions struct { @@ -168,14 +182,14 @@ type FileV1 struct { // NamespaceMgmt contains the default namespace config for all namespaces managed by this course. NamespaceMgmt struct { // Default is the default namespace config for this course - Default NamespaceConfig `yaml:"default" json:"default"` + Default *NamespaceConfig `yaml:"default" json:"default"` } `yaml:"namespace_management" json:"namespace_management"` // Charts is the map of releases - Charts ChartsListV1 `yaml:"charts" json:"charts"` + Charts map[string]ReleaseV1 `yaml:"charts" json:"charts"` } -// ChartsListV1 is a list of charts for the v1 schema -type ChartsListV1 []ReleaseV1 +// FileV1Unmarshal is a helper type that allows us to have a custom unmarshal function for the FileV2 struct +type FileV1Unmarshal FileV1 // RepositoryV1 is a helm reposotory definition type RepositoryV1 struct { @@ -205,11 +219,10 @@ func convertV1toV2(fileName string) (*FileV2, error) { newFile.NamespaceMgmt = oldFile.NamespaceMgmt newFile.DefaultRepository = oldFile.DefaultRepository newFile.Repositories = oldFile.Repositories - newFile.Releases = make([]*Release, len(oldFile.Charts)) newFile.Hooks = oldFile.Hooks newFile.MinimumVersions = oldFile.MinimumVersions - for releaseIndex, release := range oldFile.Charts { + for releaseName, release := range oldFile.Charts { repositoryName, ok := release.Repository.(string) // The repository is not in the format repository: string. Need to handle that if !ok { @@ -226,7 +239,7 @@ func convertV1toV2(fileName string) (*FileV2, error) { if addRepo.Git != "" { klog.V(3).Infof("detected a git-based inline repository. Attempting to convert to repository in header") - repositoryName = fmt.Sprintf("%s-git-repository", release.Name) + repositoryName = fmt.Sprintf("%s-git-repository", releaseName) newFile.Repositories[repositoryName] = Repository{ Git: addRepo.Git, Path: addRepo.Path, @@ -237,8 +250,8 @@ func convertV1toV2(fileName string) (*FileV2, error) { repositoryName = addRepo.Name } } - newFile.Releases[releaseIndex] = &Release{ - Name: release.Name, + newFile.Releases = append(newFile.Releases, &Release{ + Name: releaseName, Namespace: release.Namespace, NamespaceMgmt: release.NamespaceMgmt, Repository: repositoryName, @@ -246,7 +259,7 @@ func convertV1toV2(fileName string) (*FileV2, error) { Version: release.Version, Values: release.Values, Hooks: release.Hooks, - } + }) } return newFile, nil } @@ -258,10 +271,10 @@ func OpenCourseV2(fileName string, schema []byte) (*FileV2, error) { if err != nil { return nil, err } - err = yaml.Unmarshal(data, courseFile) if err != nil { - return nil, fmt.Errorf("failed to unmarshal file: %s", err.Error()) + klog.V(3).Infof("failed to unmarshal file: %s", err.Error()) + return nil, SchemaValidationError } if courseFile.SchemaVersion != "v2" { klog.V(2).Infof("did not detect v2 course file - trying conversion from v1") @@ -273,27 +286,23 @@ func OpenCourseV2(fileName string, schema []byte) (*FileV2, error) { courseFile = fileV2 } - // Marshal back here just so we can populate the env vars without any yaml comments present - data, _ = yaml.Marshal(courseFile) - data, err = parseEnv(data) - if err != nil { - return nil, fmt.Errorf("failed to parse env variables: %v", err) - } - err = yaml.Unmarshal(data, courseFile) if err != nil { - return nil, fmt.Errorf("failed to unmarshal file after parsing env vars: %s", err.Error()) + klog.V(3).Infof("failed to unmarshal file after parsing env vars: %s", err.Error()) + return nil, SchemaValidationError } courseFile.populateDefaultNamespace() courseFile.populateDefaultRepository() courseFile.populateEmptyChartNames() + courseFile.populateNamespaceManagement() if courseFile.SchemaVersion != "v2" { return nil, fmt.Errorf("unsupported schema version: %s", courseFile.SchemaVersion) } if err := courseFile.validateJsonSchema(schema); err != nil { - return nil, fmt.Errorf("failed to validate jsonSchema in course file: %s", fileName) + klog.V(3).Infof("failed to validate jsonSchema in course file: %s", fileName) + return nil, SchemaValidationError } return courseFile, nil @@ -313,24 +322,78 @@ func OpenCourseV1(fileName string) (*FileV1, error) { return courseFile, nil } -// UnmarshalYAML implements the yaml.Unmarshaler interface to customize how we Unmarshal this particular field of the FileV1 struct -func (cl *ChartsListV1) UnmarshalYAML(value *yaml.Node) error { - if value.Kind != yaml.MappingNode { - return fmt.Errorf("ChartsList must contain YAML mapping, has %v", value.Kind) +func (f *FileV1) UnmarshalYAML(value *yaml.Node) error { + err := decodeYamlWithEnv(value) + if err != nil { + return err + } + if err := value.Decode((*FileV1Unmarshal)(f)); err != nil { + return err } - *cl = make([]ReleaseV1, len(value.Content)/2) - for i := 0; i < len(value.Content); i += 2 { - var res = &(*cl)[i/2] - if err := value.Content[i].Decode(&res.Name); err != nil { - return err + return nil +} + +func decodeYamlWithEnv(value *yaml.Node) error { + v := *value + for i := 0; i < len(v.Content); i += 2 { + if v.Kind == yaml.SequenceNode { + for i := range v.Content { + var err error + v.Content[i].Value, err = parseEnv(v.Content[i].Value) + if err != nil { + return err + } + parseYamlTypes(v.Content[i]) + } + continue } - if err := value.Content[i+1].Decode(&res); err != nil { - return err + if v.Content[i+1].Kind != yaml.ScalarNode { + err := decodeYamlWithEnv(v.Content[i+1]) + if err != nil { + return err + } + continue + } + if v.Content[i+1].Tag == "!!str" { + var err error + v.Content[i+1].Value, err = parseEnv(v.Content[i+1].Value) + if err != nil { + return err + } + parseYamlTypes(v.Content[i+1]) + continue } } + *value = v return nil } +func parseYamlTypes(node *yaml.Node) { + quotedVariables := []yaml.Style{yaml.DoubleQuotedStyle, yaml.SingleQuotedStyle} + trueVals := []string{"true", "yes", "on"} + falseVals := []string{"false", "no", "off"} + if !funk.Contains(quotedVariables, node.Style) { + if funk.ContainsString(trueVals, node.Value) { + node.Tag = "!!bool" + node.Value = "true" + return + } + if funk.ContainsString(falseVals, node.Value) { + node.Tag = "!!bool" + node.Value = "false" + return + } + if _, err := strconv.ParseFloat(node.Value, 64); err == nil { + node.Tag = "!!float" + return + } + if _, err := strconv.Atoi(node.Value); err == nil { + node.Tag = "!!int" + return + } + } +} + // populateDefaultNamespace sets the default namespace in each release // if the release does not have a namespace. If the DefaultNamespace is blank, simply returns func (f *FileV2) populateDefaultNamespace() { @@ -374,7 +437,22 @@ func (f *FileV2) populateEmptyChartNames() { } } -func (f FileV2) validateJsonSchema(schemaData []byte) error { +// populateNamespaceManagement populates each release with the default namespace management settings if they are not set +func (f *FileV2) populateNamespaceManagement() { + var emptyNamespaceMgmt NamespaceConfig + if f.NamespaceMgmt.Default == nil { + f.NamespaceMgmt.Default = &emptyNamespaceMgmt + } + for releaseIndex, release := range f.Releases { + if release.NamespaceMgmt == nil { + klog.V(5).Infof("using default namespace management for release: %s", release.Name) + release.NamespaceMgmt = f.NamespaceMgmt.Default + f.Releases[releaseIndex] = release + } + } +} + +func (f *FileV2) validateJsonSchema(schemaData []byte) error { klog.V(10).Infof("validating course file against schema: \n%s", string(schemaData)) schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaData)) if err != nil { @@ -383,18 +461,18 @@ func (f FileV2) validateJsonSchema(schemaData []byte) error { jsonData, err := json.Marshal(f) if err != nil { - return err + return SchemaValidationError } result, err := schema.Validate(gojsonschema.NewBytesLoader(jsonData)) if err != nil { - return err + return SchemaValidationError } if len(result.Errors()) > 0 { for _, err := range result.Errors() { klog.Errorf("jsonSchema error: %s", err.String()) } - return fmt.Errorf("jsonSchema validation failed") + return SchemaValidationError } return nil } @@ -409,19 +487,18 @@ func (r *Release) SetGitPaths(clonePath, subPath string) error { return nil } -func parseEnv(data []byte) ([]byte, error) { +func parseEnv(data string) (string, error) { dataWithEnv := os.Expand(string(data), envMapper) if strings.Contains(dataWithEnv, "_ENV_NOT_SET_") { - return nil, fmt.Errorf("course has env variables that are not properly set") + return data, fmt.Errorf("course has env variables that are not properly set") } - return []byte(dataWithEnv), nil + return dataWithEnv, nil } func envMapper(key string) string { - v := os.Getenv(key) - if v == "" { - color.Red("ERROR: environment variable %s is not set", key) - return "_ENV_NOT_SET_" + if value, ok := os.LookupEnv(key); ok { + return value } - return v + color.Red("ERROR: environment variable %s is not set", key) + return "_ENV_NOT_SET_" } diff --git a/pkg/course/course_test.go b/pkg/course/course_test.go index 683ee99b..85350527 100644 --- a/pkg/course/course_test.go +++ b/pkg/course/course_test.go @@ -48,7 +48,7 @@ func TestConvertV1toV2(t *testing.T) { DefaultNamespace: "namespace", DefaultRepository: "stable", Context: "farglebargle", - Repositories: RepositoryList{ + Repositories: RepositoryMap{ "git-repo-test": { Git: "https://github.com/FairwindsOps/charts", Path: "stable", diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 003cda00..b576bc00 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -102,6 +102,16 @@ func (h Client) Cache() (string, error) { return "", fmt.Errorf("could not find HELM_REPOSITORY_CACHE in helm env output") } +// UpdateDependencies will update dependencies for a given release if it is stored locally (i.e. pulled from git) +func (h Client) UpdateDependencies(path string) error { + klog.V(5).Infof("updating chart dependencies for %s", path) + _, stdErr, _ := h.Exec("dependency", "update", path) + if stdErr != "" { + return fmt.Errorf("error running helm dependency update: %s", stdErr) + } + return nil +} + // get can run any 'helm get' command func (h Client) get(kind, namespace, release string) (string, error) { validKinds := []string{"all", "hooks", "manifest", "notes", "values"} diff --git a/pkg/reckoner/client.go b/pkg/reckoner/client.go index fc67214b..cf972e81 100644 --- a/pkg/reckoner/client.go +++ b/pkg/reckoner/client.go @@ -45,6 +45,8 @@ type Client struct { BaseDirectory string DryRun bool CreateNamespaces bool + ContinueOnError bool + Errors int } var once sync.Once @@ -52,7 +54,7 @@ var clientset *kubernetes.Clientset // NewClient returns a client. Attempts to open a v2 schema course file // If getClient is true, attempts to get a Kubernetes client from config -func NewClient(fileName, version string, plotAll bool, releases []string, kubeClient bool, dryRun bool, createNamespaces bool, schema []byte) (*Client, error) { +func NewClient(fileName, version string, plotAll bool, releases []string, kubeClient bool, dryRun bool, createNamespaces bool, schema []byte, continueOnError bool) (*Client, error) { // Get the course file courseFile, err := course.OpenCourseV2(fileName, schema) if err != nil { @@ -74,6 +76,7 @@ func NewClient(fileName, version string, plotAll bool, releases []string, kubeCl BaseDirectory: path.Dir(fileName), DryRun: dryRun, CreateNamespaces: createNamespaces, + ContinueOnError: continueOnError, } // Check versions @@ -95,6 +98,14 @@ func NewClient(fileName, version string, plotAll bool, releases []string, kubeCl return client, nil } +func (c *Client) Continue() bool { + if c.ContinueOnError { + c.Errors += 1 + return true + } + return false +} + func getKubeClient() *kubernetes.Clientset { once.Do(func() { kubeConf, err := config.GetConfig() @@ -192,21 +203,6 @@ func (c *Client) filterReleases() error { releaseIndex := funk.IndexOf(c.CourseFile.Releases, func(rel *course.Release) bool { return rel.Name == releaseName }) - repository := c.CourseFile.Repositories[c.CourseFile.Releases[releaseIndex].Repository] - if repository.Git != "" { - cacheDir, err := c.Helm.Cache() - if err != nil { - return err - } - re := regexp.MustCompile(`\:\/\/|\/|\.`) - repoPathName := re.ReplaceAllString(repository.Git, "_") - clonePath := fmt.Sprintf("%s/%s_goreckoner", cacheDir, repoPathName) - - err = c.CourseFile.Releases[releaseIndex].SetGitPaths(clonePath, repository.Path) - if err != nil { - return err - } - } selectedReleases = append(selectedReleases, c.CourseFile.Releases[releaseIndex]) } releases = selectedReleases @@ -217,6 +213,32 @@ func (c *Client) filterReleases() error { } return fmt.Errorf("no valid releases found in course that match input releases") } + releases, err := c.prepareGitRepositories(releases) + if err != nil { + return err + } c.CourseFile.Releases = releases return nil } + +// prepareGitRepositories checks each release provided for a git repository and prepares the struct with the proper information needed to clone +func (c *Client) prepareGitRepositories(releases []*course.Release) ([]*course.Release, error) { + for _, release := range releases { + repository := c.CourseFile.Repositories[release.Repository] + if repository.Git != "" { + cacheDir, err := c.Helm.Cache() + if err != nil { + return releases, err + } + re := regexp.MustCompile(`\:\/\/|\/|\.`) + repoPathName := re.ReplaceAllString(repository.Git, "_") + clonePath := fmt.Sprintf("%s/%s_goreckoner", cacheDir, repoPathName) + + err = release.SetGitPaths(clonePath, fmt.Sprintf("%s/%s", repository.Path, release.Chart)) + if err != nil { + return releases, err + } + } + } + return releases, nil +} diff --git a/pkg/reckoner/diff.go b/pkg/reckoner/diff.go index edde2006..a9404aed 100644 --- a/pkg/reckoner/diff.go +++ b/pkg/reckoner/diff.go @@ -49,14 +49,18 @@ func (md ManifestDiff) String() string { // Diff gathers a given release's manifest and templates and outputs a string of diffs if there are any or reports that there are no diffs. func (c *Client) Diff() error { - for _, r := range c.CourseFile.Releases { - fmt.Printf("\nRunning 'diff' on %s in %s\n", r.Name, r.Namespace) - thisReleaseDiff, err := c.diffRelease(r.Name, r.Namespace) + for _, release := range c.CourseFile.Releases { + fmt.Printf("\nRunning 'diff' on %s in %s\n", release.Name, release.Namespace) + thisReleaseDiff, err := c.diffRelease(release.Name, release.Namespace) if err != nil { + if c.Continue() { + color.Red("error with release %s: %s, continuing.", release.Name, err.Error()) + continue + } return err } if thisReleaseDiff == "" { - color.Green("There are no differences in release %s", r.Name) + color.Green("There are no differences in release %s", release.Name) } else { fmt.Print(thisReleaseDiff) } diff --git a/pkg/reckoner/git.go b/pkg/reckoner/git.go index 3b926a3b..bfaff788 100644 --- a/pkg/reckoner/git.go +++ b/pkg/reckoner/git.go @@ -51,7 +51,8 @@ func (c Client) cloneGitRepository(release *course.Release) error { } err = worktree.Checkout(&git.CheckoutOptions{ - Hash: *hash, + Hash: *hash, + Force: true, }) if err != nil { return fmt.Errorf("Error checking out git repository %s - %s", *release.GitClonePath, err) diff --git a/pkg/reckoner/hook.go b/pkg/reckoner/hook.go index b05c7353..8009d472 100644 --- a/pkg/reckoner/hook.go +++ b/pkg/reckoner/hook.go @@ -19,10 +19,9 @@ import ( "strings" "github.com/fatih/color" - "k8s.io/klog/v2" ) -func (c Client) execHook(hooks []string) error { +func (c Client) execHook(hooks []string, kind string) error { if c.DryRun { color.Yellow("hook not run due to --dry-run: %v", c.DryRun) return nil @@ -33,7 +32,7 @@ func (c Client) execHook(hooks []string) error { } for _, hook := range hooks { - color.Green("Running hook %s", hook) + color.Green("Running %s hook: %s", kind, hook) commands := strings.Split(hook, " ") args := commands[1:] @@ -41,7 +40,7 @@ func (c Client) execHook(hooks []string) error { command.Dir = c.BaseDirectory data, runError := command.CombinedOutput() - klog.V(3).Infof("command %s output: %s", command.String(), string(data)) + color.Green("Hook '%s' output: %s", command.String(), string(data)) if runError != nil { return runError } diff --git a/pkg/reckoner/namespace.go b/pkg/reckoner/namespace.go index 334a4922..9bcde5ba 100644 --- a/pkg/reckoner/namespace.go +++ b/pkg/reckoner/namespace.go @@ -27,7 +27,7 @@ import ( ) // CreateNamespace creates a kubernetes namespace with the given annotations and labels -func (c *Client) CreateNamespace(namespace string, annotations, labels map[string]string) error { +func (c *Client) CreateNamespace(namespace string, annotations, labels map[string]string, runningNamespaceList *v1.NamespaceList) error { ns := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, @@ -35,10 +35,11 @@ func (c *Client) CreateNamespace(namespace string, annotations, labels map[strin Labels: labels, }, } - _, err := c.KubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) + returnedNS, err := c.KubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) if err != nil { return err } + runningNamespaceList.Items = append(runningNamespaceList.Items, *returnedNS) return nil } @@ -78,7 +79,7 @@ func (c *Client) NamespaceManagement() error { return err } for _, release := range c.CourseFile.Releases { - err := c.CreateOrPatchNamespace(release.NamespaceMgmt.Settings.Overwrite, release.Namespace, release.NamespaceMgmt, namespaces) + err := c.CreateOrPatchNamespace(release.NamespaceMgmt.Settings.Overwrite, release.Namespace, *release.NamespaceMgmt, namespaces) if err != nil { return err } @@ -95,7 +96,7 @@ func (c *Client) CreateOrPatchNamespace(overWrite bool, namespaceName string, na annotations, labels := labelsAndAnnotationsToUpdate(overWrite, namespaceMgmt.Metadata.Annotations, namespaceMgmt.Metadata.Labels, ns) err = c.PatchNamespace(namespaceName, annotations, labels) } else { - err = c.CreateNamespace(namespaceName, namespaceMgmt.Metadata.Annotations, namespaceMgmt.Metadata.Labels) + err = c.CreateNamespace(namespaceName, namespaceMgmt.Metadata.Annotations, namespaceMgmt.Metadata.Labels, namespaces) } return err } diff --git a/pkg/reckoner/namespace_test.go b/pkg/reckoner/namespace_test.go index b21eb7f0..f29ad89a 100644 --- a/pkg/reckoner/namespace_test.go +++ b/pkg/reckoner/namespace_test.go @@ -33,7 +33,7 @@ func TestCreateNamespace(t *testing.T) { name := "reckoner" annotations := map[string]string{"nginx.ingress.kubernetes.io/http2-push-preload": "true"} labels := map[string]string{"app.kubernetes.io/name": "ingress-nginx"} - err := fakeClient.CreateNamespace(name, annotations, labels) + err := fakeClient.CreateNamespace(name, annotations, labels, &v1.NamespaceList{}) assert.NoError(t, err) namespace, err := fakeKubeClinet.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{}) @@ -54,7 +54,7 @@ func TestPatchNamespace(t *testing.T) { name := "reckoner" annotations := map[string]string{"nginx.ingress.kubernetes.io/http2-push-preload": "true"} labels := map[string]string{"app.kubernetes.io/name": "ingress-nginx"} - err := fakeClient.CreateNamespace(name, annotations, labels) + err := fakeClient.CreateNamespace(name, annotations, labels, &v1.NamespaceList{}) assert.NoError(t, err) newAnnotations := map[string]string{"service.beta.kubernetes.io/aws-load-balancer-internal": "0.0.0.0/0"} @@ -81,7 +81,7 @@ func TestCheckIfNamespaceExist(t *testing.T) { name := "reckoner" annotations := map[string]string{"nginx.ingress.kubernetes.io/http2-push-preload": "true"} labels := map[string]string{"app.kubernetes.io/name": "ingress-nginx"} - err := fakeClient.CreateNamespace(name, annotations, labels) + err := fakeClient.CreateNamespace(name, annotations, labels, &v1.NamespaceList{}) assert.NoError(t, err) nsList, err := fakeClient.KubeClient.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{}) @@ -104,7 +104,7 @@ func TestLabelsAndAnnotationsToUpdate(t *testing.T) { name := "reckoner" annotations := map[string]string{"nginx.ingress.kubernetes.io/http2-push-preload": "true"} labels := map[string]string{"app.kubernetes.io/name": "ingress-nginx"} - err := fakeClient.CreateNamespace(name, annotations, labels) + err := fakeClient.CreateNamespace(name, annotations, labels, &v1.NamespaceList{}) assert.NoError(t, err) newLabels := map[string]string{"app.kubernetes.io/name": "ingress-nginx-new", "label-key-1": "label-value-1"} diff --git a/pkg/reckoner/plot.go b/pkg/reckoner/plot.go index 67d36aa1..b75621a4 100644 --- a/pkg/reckoner/plot.go +++ b/pkg/reckoner/plot.go @@ -20,8 +20,7 @@ import ( "os" "strings" - "gopkg.in/yaml.v3" - "k8s.io/klog/v2" + "gopkg.in/yaml.v2" "github.com/fairwindsops/reckoner/pkg/course" "github.com/fatih/color" @@ -29,7 +28,7 @@ import ( ) // Plot actually plots the releases -func (c Client) Plot() error { +func (c *Client) Plot() error { err := c.NamespaceManagement() if err != nil { return err @@ -40,27 +39,27 @@ func (c Client) Plot() error { return err } - err = c.execHook(c.CourseFile.Hooks.PreInstall) + err = c.execHook(c.CourseFile.Hooks.PreInstall, "course pre") if err != nil { - return err + if !c.Continue() { + return err + } } for _, release := range c.CourseFile.Releases { - err = c.execHook(release.Hooks.PreInstall) + err = c.execHook(release.Hooks.PreInstall, "release pre") if err != nil { - return err - } - - if release.GitClonePath != nil { - if err := c.cloneGitRepository(release); err != nil { - return err + if c.Continue() { + color.Red("error with release %s: %s, continuing.", release.Name, err.Error()) + continue } + return err } args, tmpFile, err := buildHelmArgs("upgrade", *release) if err != nil { - klog.Error(err) + color.Red(err.Error()) continue } if tmpFile != nil { @@ -68,8 +67,28 @@ func (c Client) Plot() error { } if !c.DryRun { + if release.GitClonePath != nil { + if err := c.cloneGitRepository(release); err != nil { + if c.Continue() { + color.Red("error with release %s: %s, continuing.", release.Name, err.Error()) + continue + } + return err + } + if err := c.Helm.UpdateDependencies(fmt.Sprintf("%s/%s", *release.GitClonePath, *release.GitChartSubPath)); err != nil { + if c.Continue() { + color.Red("error with release %s: %s, continuing.", release.Name, err.Error()) + continue + } + return err + } + } out, stdErr, err := c.Helm.Exec(args...) if err != nil { + if c.Continue() { + color.Red("error with release %s: %s, continuing.", release.Name, err.Error()) + continue + } return fmt.Errorf("error plotting release %s: %s", release.Name, stdErr) } fmt.Println(out) @@ -78,15 +97,21 @@ func (c Client) Plot() error { color.Yellow("would have run: helm %s", strings.Join(args, " ")) } - err = c.execHook(release.Hooks.PostInstall) + err = c.execHook(release.Hooks.PostInstall, "release post") if err != nil { + if c.Continue() { + color.Red("error with release %s: %s, continuing.", release.Name, err.Error()) + continue + } return err } } - err = c.execHook(c.CourseFile.Hooks.PostInstall) + err = c.execHook(c.CourseFile.Hooks.PostInstall, "course post") if err != nil { - return err + if !c.Continue() { + return err + } } return nil @@ -103,7 +128,7 @@ func (c Client) TemplateAll() (string, error) { for _, release := range c.CourseFile.Releases { out, err := c.TemplateRelease(release.Name) if err != nil { - klog.Error(err) + color.Red(err.Error()) continue } fullOutput = fullOutput + out diff --git a/pkg/reckoner/plot_test.go b/pkg/reckoner/plot_test.go index 0cf2bb93..97f30598 100644 --- a/pkg/reckoner/plot_test.go +++ b/pkg/reckoner/plot_test.go @@ -23,9 +23,8 @@ import ( func Test_buildHelmArgs(t *testing.T) { type args struct { - releaseName string - command string - release course.Release + command string + release course.Release } tests := []struct { name string diff --git a/pkg/reckoner/update.go b/pkg/reckoner/update.go index fd635629..30186d20 100644 --- a/pkg/reckoner/update.go +++ b/pkg/reckoner/update.go @@ -21,17 +21,21 @@ import ( func (c *Client) Update() error { updatedReleases := []*course.Release{} - for i, r := range c.CourseFile.Releases { - thisReleaseDiff, err := c.diffRelease(r.Name, r.Namespace) + for i, release := range c.CourseFile.Releases { + thisReleaseDiff, err := c.diffRelease(release.Name, release.Namespace) if err != nil { + if c.Continue() { + color.Red("error with release %s: %s, continuing.", release.Name, err.Error()) + continue + } return err } if thisReleaseDiff != "" { - color.Yellow("Update available for %s in namespace %s. Added to plot list.", r.Name, r.Namespace) + color.Yellow("Update available for %s in namespace %s. Added to plot list.", release.Name, release.Namespace) updatedReleases = append(updatedReleases, c.CourseFile.Releases[i]) continue } - color.Green("No update necessary for %s in namespace %s.", r.Name, r.Namespace) + color.Green("No update necessary for %s in namespace %s.", release.Name, release.Namespace) } c.CourseFile.Releases = updatedReleases err := c.Plot() From 874a533764cf380a2d7978c98fb6b7c9ce16cff7 Mon Sep 17 00:00:00 2001 From: Luke Reed Date: Mon, 28 Feb 2022 15:41:20 -0500 Subject: [PATCH 2/8] circleci - remove python e2e tests and comment out release workflow temporarily --- .circleci/config.yml | 199 +++++++------------------------------------ 1 file changed, 32 insertions(+), 167 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b6bd4f91..76d4a0b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,35 +17,8 @@ version: 2.1 orbs: rok8s: fairwinds/rok8s-scripts@11 -executors: - python-3-7: - docker: - - image: circleci/python:3.7 - python-3-8: - docker: - - image: circleci/python:3.8 - python-3-9: - docker: - - image: cimg/python:3.9 - references: - e2e_configuration_python: &e2e_configuration_python - pre_script: end_to_end_testing/pre_python.sh - script: end_to_end_testing/run_python.sh - command_runner_image: quay.io/reactiveops/ci-images:v11-buster - enable_docker_layer_caching: false - store-test-results: /tmp/test-results - kind_version: 0.10.0 - requires: - - build-3-7 - - build-3-8 - - build-3-9 - filters: - branches: - only: /.*/ - tags: - ignore: /.*/ - e2e_configuration_go: &e2e_configuration_go + e2e_configuration: &e2e_configuration attach-workspace: true workspace-location: / pre_script: end_to_end_testing/pre_go.sh @@ -55,75 +28,15 @@ references: store-test-results: /tmp/test-results kind_version: 0.10.0 requires: - - test-go - - snapshot-go + - test + - snapshot filters: branches: only: /.*/ tags: ignore: /.*/ - install_goreleaser: &install_goreleaser - run: - name: Install Goreleaser - command: | - curl -fsSLo goreleaser_amd64.deb https://github.com/goreleaser/goreleaser/releases/download/v0.164.0/goreleaser_amd64.deb - echo "577c88019cca787161ee1ff853d45be7f635ad7d8a204bd7a2735ab04abcc255 goreleaser_amd64.deb" | sha256sum -c - - sudo dpkg -i goreleaser_amd64.deb - rm goreleaser_amd64.deb - install_go: &install_go - run: - name: Install Golang - command: | - curl -LO https://golang.org/dl/go1.17.7.linux-amd64.tar.gz - tar -C /usr/local -xzf go1.17.7.linux-amd64.tar.gz - echo "PATH=$PATH:/usr/local/go/bin" >> ${BASH_ENV} jobs: - build-3-7: - executor: python-3-7 - working_directory: ~/reckoner - steps: - - run: - name: Setup PATH to support pip user installs - command: echo 'export PATH=$PATH:/home/circleci/.local/bin' >> $BASH_ENV - - checkout - - run: - name: Unit Tests - command: | - pip install --user -r development-requirements.txt - pip install --user -e . - reckoner --version - pytest --cov ./ - build-3-8: - executor: python-3-8 - working_directory: ~/reckoner - steps: - - run: - name: Setup PATH to support pip user installs - command: echo 'export PATH=$PATH:/home/circleci/.local/bin' >> $BASH_ENV - - checkout - - run: - name: Unit Tests - command: | - pip install --user -r development-requirements.txt - pip install --user -e . - reckoner --version - pytest --cov ./ - build-3-9: - executor: python-3-9 - working_directory: ~/reckoner - steps: - - run: - name: Setup PATH to support pip user installs - command: echo 'export PATH=$PATH:/home/circleci/.local/bin' >> $BASH_ENV - - checkout - - run: - name: Unit Tests - command: | - pip install --user -r development-requirements.txt - pip install --user -e . - reckoner --version - pytest --cov ./ - test-go: + test: working_directory: /go/src/github.com/fairwindsops/reckoner docker: - image: circleci/golang:1.17 @@ -132,7 +45,7 @@ jobs: steps: - checkout - run: make test - snapshot-go: + snapshot: working_directory: /go/src/github.com/fairwindsops/reckoner docker: - image: goreleaser/goreleaser:v1.5.0 @@ -149,26 +62,6 @@ jobs: root: /go/src/github.com/fairwindsops/reckoner paths: - dist - release: - executor: python-3-9 - environment: - GITHUB_ORGANIZATION: $CIRCLE_PROJECT_USERNAME - GITHUB_REPOSITORY: $CIRCLE_PROJECT_REPONAME - working_directory: ~/reckoner - steps: - - checkout - - run: - name: init .pypirc - command: | - echo -e "[pypi]" >> ~/.pypirc - echo -e "username = $PYPI_USERNAME" >> ~/.pypirc - echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc - - run: - name: package and upload - command: | - pip install twine - python setup.py sdist bdist_wheel - twine upload dist/* publish_docs: docker: - image: cimg/node:15.5.1 @@ -213,73 +106,45 @@ workflows: version: 2 build_and_test: jobs: - - test-go: + - test: filters: tags: ignore: /.*/ branches: only: /.*/ - - build-3-7: - filters: - tags: - ignore: /.*/ - branches: - only: /.*/ - - build-3-8: - filters: - tags: - ignore: /.*/ - branches: - only: /.*/ - - build-3-9: - filters: - tags: - ignore: /.*/ - branches: - only: /.*/ - - snapshot-go: + - snapshot: requires: - - test-go + - test filters: tags: ignore: /.*/ branches: only: /.*/ - rok8s/kubernetes_e2e_tests: - name: "End-To-End Kubernetes 1.19 - Python" + name: "End-To-End Kubernetes 1.19.7" kind_node_image: "kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca" - <<: *e2e_configuration_python + <<: *e2e_configuration - rok8s/kubernetes_e2e_tests: - name: "End-To-End Kubernetes 1.20 - Python" - kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab" - <<: *e2e_configuration_python - - rok8s/kubernetes_e2e_tests: - name: "End-To-End Kubernetes 1.19 - Go" - kind_node_image: "kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca" - <<: *e2e_configuration_go + name: "End-To-End Kubernetes 1.20.2" + kind_node_image: "kindest/node:v1.20.2@sha256:15d3b5c4f521a84896ed1ead1b14e4774d02202d5c65ab68f30eeaf310a3b1a7" + <<: *e2e_configuration - rok8s/kubernetes_e2e_tests: - name: "End-To-End Kubernetes 1.20 - Go" - kind_node_image: "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab" - <<: *e2e_configuration_go - release: - jobs: - - release: - filters: - tags: - only: /.*/ - branches: - ignore: /.*/ - - publish_docs: - filters: - branches: - ignore: /.*/ - tags: - only: /.*/ - - rok8s/github_release: - requires: - - release - filters: - branches: - ignore: /.*/ - tags: - only: /.*/ + name: "End-To-End Kubernetes 1.21.1" + kind_node_image: "kindest/node:v1.21.2@sha256:9d07ff05e4afefbba983fac311807b3c17a5f36e7061f6cb7e2ba756255b2be4" + <<: *e2e_configuration + # release: + # jobs: + # - publish_docs: + # filters: + # branches: + # ignore: /.*/ + # tags: + # only: /.*/ + # - rok8s/github_release: + # requires: + # - release + # filters: + # branches: + # ignore: /.*/ + # tags: + # only: /.*/ From b565716b9298a77aa01532c56717119aeac87675 Mon Sep 17 00:00:00 2001 From: Luke Reed Date: Mon, 28 Feb 2022 15:55:11 -0500 Subject: [PATCH 3/8] try updating kind version in e2e tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 76d4a0b3..cb0571d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ references: command_runner_image: quay.io/reactiveops/ci-images:v11-buster enable_docker_layer_caching: false store-test-results: /tmp/test-results - kind_version: 0.10.0 + kind_version: 0.11.1 requires: - test - snapshot From db6f7981930ef1264e4532448b7cb132bcf00fca Mon Sep 17 00:00:00 2001 From: Luke Reed Date: Mon, 28 Feb 2022 16:02:42 -0500 Subject: [PATCH 4/8] circleci fixes --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb0571d6..acbddd9e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,12 +96,12 @@ jobs: sudo chmod +x /usr/bin/yq yq --version - rok8s/get_vault_env: - vault_path: repo/pluto/env + vault_path: repo/reckoner/env - run: name: Publish Docs Site to S3 command: | cd ./dist - aws s3 sync ./ s3://pluto.docs.fairwinds.com --delete + aws s3 sync ./ s3://reckoner.docs.fairwinds.com --delete workflows: version: 2 build_and_test: From 56727537182f6253e8bfe086018ddcb956d8592f Mon Sep 17 00:00:00 2001 From: Luke Reed Date: Mon, 28 Feb 2022 16:11:53 -0500 Subject: [PATCH 5/8] update based on feedback --- .circleci/config.yml | 8 ++++---- .../course_files/05_test_exit_on_post_install_hook.yml | 2 +- pkg/course/course.go | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index acbddd9e..45ba22f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,10 +120,6 @@ workflows: ignore: /.*/ branches: only: /.*/ - - rok8s/kubernetes_e2e_tests: - name: "End-To-End Kubernetes 1.19.7" - kind_node_image: "kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca" - <<: *e2e_configuration - rok8s/kubernetes_e2e_tests: name: "End-To-End Kubernetes 1.20.2" kind_node_image: "kindest/node:v1.20.2@sha256:15d3b5c4f521a84896ed1ead1b14e4774d02202d5c65ab68f30eeaf310a3b1a7" @@ -132,6 +128,10 @@ workflows: name: "End-To-End Kubernetes 1.21.1" kind_node_image: "kindest/node:v1.21.2@sha256:9d07ff05e4afefbba983fac311807b3c17a5f36e7061f6cb7e2ba756255b2be4" <<: *e2e_configuration + - rok8s/kubernetes_e2e_tests: + name: "End-To-End Kubernetes 1.22.0" + kind_node_image: "kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047" + <<: *e2e_configuration # release: # jobs: # - publish_docs: diff --git a/end_to_end_testing/course_files/05_test_exit_on_post_install_hook.yml b/end_to_end_testing/course_files/05_test_exit_on_post_install_hook.yml index 0a448675..98e70cd5 100644 --- a/end_to_end_testing/course_files/05_test_exit_on_post_install_hook.yml +++ b/end_to_end_testing/course_files/05_test_exit_on_post_install_hook.yml @@ -13,4 +13,4 @@ charts: pre_install: - whoami post_install: - - exit 1 + - exit 1 diff --git a/pkg/course/course.go b/pkg/course/course.go index f86c44e4..ca2234b3 100644 --- a/pkg/course/course.go +++ b/pkg/course/course.go @@ -27,7 +27,6 @@ import ( "github.com/thoas/go-funk" "github.com/xeipuuv/gojsonschema" - // "gopkg.in/yaml.v2" "gopkg.in/yaml.v3" "k8s.io/klog/v2" ) From aec117d195ead782a827b4be30c20e4ab227f5a9 Mon Sep 17 00:00:00 2001 From: Luke Reed Date: Mon, 28 Feb 2022 16:13:56 -0500 Subject: [PATCH 6/8] Update .circleci/config.yml Co-authored-by: Andrew Suderman --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 45ba22f9..065ecc0d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,6 @@ references: command_runner_image: quay.io/reactiveops/ci-images:v11-buster enable_docker_layer_caching: false store-test-results: /tmp/test-results - kind_version: 0.11.1 requires: - test - snapshot From ebb42c2a4dabd09bf5a8c244270a2486731f3446 Mon Sep 17 00:00:00 2001 From: Luke Reed Date: Mon, 28 Feb 2022 17:05:33 -0500 Subject: [PATCH 7/8] change git hash for 10-test --- end_to_end_testing/course_files/10_test_git_chart.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end_to_end_testing/course_files/10_test_git_chart.yml b/end_to_end_testing/course_files/10_test_git_chart.yml index 74c02190..b3dfead9 100644 --- a/end_to_end_testing/course_files/10_test_git_chart.yml +++ b/end_to_end_testing/course_files/10_test_git_chart.yml @@ -18,7 +18,7 @@ charts: path: stable goldilocks-10: chart: goldilocks - version: e3ff9b07be83cad790365402166574977a47491d + version: 125e58c33b29fb28e419732997aff1325578ddfa repository: git: https://github.com/FairwindsOps/charts path: stable From 9d47e199f37defa3a7e1f3da7ec3b56b27698131 Mon Sep 17 00:00:00 2001 From: Luke Reed Date: Tue, 1 Mar 2022 14:37:41 -0500 Subject: [PATCH 8/8] better namespace_management merging and test for that; also changes asked for in review --- cmd/root.go | 6 +-- ...efault_namespace_annotation_and_labels.yml | 10 ++++ .../tests/15_namespace_management.yaml | 19 +++++++- go.mod | 2 +- pkg/course/course.go | 47 ++++++++++++++++++- pkg/reckoner/namespace.go | 2 +- pkg/reckoner/plot.go | 2 +- pkg/reckoner/update.go | 4 +- 8 files changed, 80 insertions(+), 12 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 8db3d03a..bd58160d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -120,7 +120,7 @@ var templateCmd = &cobra.Command{ Long: "Templates a helm chart for a release or several releases. Automatically sets --create-namespaces=false --dry-run=true", PreRunE: validateArgs, Run: func(cmd *cobra.Command, args []string) { - client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, continueOnError) + client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, false) if err != nil { color.Red(err.Error()) os.Exit(1) @@ -140,7 +140,7 @@ var getManifestsCmd = &cobra.Command{ Long: "Gets the manifests currently in the cluster.", PreRunE: validateArgs, Run: func(cmd *cobra.Command, args []string) { - client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, continueOnError) + client, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, false) if err != nil { color.Red(err.Error()) os.Exit(1) @@ -189,7 +189,7 @@ var lintCmd = &cobra.Command{ return validateArgs(cmd, args) }, Run: func(cmd *cobra.Command, args []string) { - _, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, continueOnError) + _, err := reckoner.NewClient(courseFile, version, runAll, onlyRun, false, true, false, courseSchema, false) if err != nil { color.Red(err.Error()) os.Exit(1) diff --git a/end_to_end_testing/course_files/15_test_default_namespace_annotation_and_labels.yml b/end_to_end_testing/course_files/15_test_default_namespace_annotation_and_labels.yml index 9aa5fc1f..b8b6ca2b 100644 --- a/end_to_end_testing/course_files/15_test_default_namespace_annotation_and_labels.yml +++ b/end_to_end_testing/course_files/15_test_default_namespace_annotation_and_labels.yml @@ -18,3 +18,13 @@ charts: namespace-test: repository: fairwinds-incubator chart: basic-demo + namespace-test-merge: + namespace: 15-annotatednamespace-2 + namespace_management: + metadata: + annotations: + this: exists + labels: + this: alsoexists + repository: fairwinds-incubator + chart: basic-demo diff --git a/end_to_end_testing/tests/15_namespace_management.yaml b/end_to_end_testing/tests/15_namespace_management.yaml index 1f8b9521..81a4b5d4 100644 --- a/end_to_end_testing/tests/15_namespace_management.yaml +++ b/end_to_end_testing/tests/15_namespace_management.yaml @@ -23,8 +23,23 @@ testcases: kubectl get ns {{.namespace}} -ojson | jq -e ".metadata.labels[\"rocks\"] == \"reckoner\"" assertions: - result.code ShouldEqual 0 + - script: | + kubectl get ns {{.namespace}}-2 -ojson | jq -e ".metadata.annotations[\"this\"] == \"exists\"" + assertions: + - result.code ShouldEqual 0 + - script: | + kubectl get ns {{.namespace}}-2 -ojson | jq -e ".metadata.labels[\"this\"] == \"alsoexists\"" + assertions: + - result.code ShouldEqual 0 + - script: | + kubectl get ns {{.namespace}}-2 -ojson | jq -e ".metadata.annotations[\"reckoner\"] == \"rocks\"" + assertions: + - result.code ShouldEqual 0 + - script: | + kubectl get ns {{.namespace}}-2 -ojson | jq -e ".metadata.labels[\"rocks\"] == \"reckoner\"" + assertions: + - result.code ShouldEqual 0 - name: 15 - cleanup steps: - script: | - helm -n {{.namespace}} delete {{.release}} - kubectl delete ns {{.namespace}} + kubectl delete ns {{.namespace}} {{.namespace}}-2 diff --git a/go.mod b/go.mod index 213da0a1..d2bc1ef5 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.30.0 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect diff --git a/pkg/course/course.go b/pkg/course/course.go index ca2234b3..ebbd938d 100644 --- a/pkg/course/course.go +++ b/pkg/course/course.go @@ -97,7 +97,7 @@ type NamespaceConfig struct { } `yaml:"metadata,omitempty" json:"metadata,omitempty"` Settings struct { // Overwrite specifies if these annotations and labels should be overwritten in the event that they already exist. - Overwrite bool `yaml:"overwrite,omitempty" json:"overwrite,omitempty"` + Overwrite *bool `yaml:"overwrite,omitempty" json:"overwrite,omitempty"` } `yaml:"settings" json:"settings"` } @@ -321,17 +321,36 @@ func OpenCourseV1(fileName string) (*FileV1, error) { return courseFile, nil } +// UnmarshalYAML implements the yaml.Unmarshaler interface for FileV1. This allows us to do environment variable parsing +// and changing behavior for boolean parsing such that non-quoted `yes`, `no`, `on`, `off` become booleans. func (f *FileV1) UnmarshalYAML(value *yaml.Node) error { err := decodeYamlWithEnv(value) if err != nil { return err } + // This little monster allows us to run Decode on the whole FileV1 struct type without causing an infinite loop + // because Decode will call this UnmarshalYAML method again and again if we didn't have the intermediate FileV1Unmarshal struct. if err := value.Decode((*FileV1Unmarshal)(f)); err != nil { return err } return nil } +// UnmarshalYAML implements the yaml.Unmarshaler interface for FileV2. This allows us to do environment variable parsing +// and changing behavior for boolean parsing such that non-quoted `yes`, `no`, `on`, `off` become booleans. +func (f *FileV2) UnmarshalYAML(value *yaml.Node) error { + err := decodeYamlWithEnv(value) + if err != nil { + return err + } + // This little monster allows us to run Decode on the whole FileV2 struct type without causing an infinite loop + // because Decode will call this UnmarshalYAML method again and again if we didn't have the intermediate FileV2Unmarshal struct. + if err := value.Decode((*FileV2Unmarshal)(f)); err != nil { + return err + } + return nil +} + func decodeYamlWithEnv(value *yaml.Node) error { v := *value for i := 0; i < len(v.Content); i += 2 { @@ -441,16 +460,36 @@ func (f *FileV2) populateNamespaceManagement() { var emptyNamespaceMgmt NamespaceConfig if f.NamespaceMgmt.Default == nil { f.NamespaceMgmt.Default = &emptyNamespaceMgmt + f.NamespaceMgmt.Default.Settings.Overwrite = boolPtr(false) + } else if f.NamespaceMgmt.Default.Settings.Overwrite == nil { + f.NamespaceMgmt.Default.Settings.Overwrite = boolPtr(false) } for releaseIndex, release := range f.Releases { if release.NamespaceMgmt == nil { klog.V(5).Infof("using default namespace management for release: %s", release.Name) release.NamespaceMgmt = f.NamespaceMgmt.Default f.Releases[releaseIndex] = release + } else { + release.NamespaceMgmt = mergeNamespaceManagement(f.NamespaceMgmt.Default, release.NamespaceMgmt) } } } +func mergeNamespaceManagement(defaults *NamespaceConfig, mergeInto *NamespaceConfig) *NamespaceConfig { + d := defaults + for k, v := range mergeInto.Metadata.Annotations { + d.Metadata.Annotations[k] = v + } + for k, v := range mergeInto.Metadata.Labels { + d.Metadata.Labels[k] = v + } + if mergeInto.Settings.Overwrite != nil { + d.Settings.Overwrite = mergeInto.Settings.Overwrite + } + mergeInto = d + return mergeInto +} + func (f *FileV2) validateJsonSchema(schemaData []byte) error { klog.V(10).Infof("validating course file against schema: \n%s", string(schemaData)) schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaData)) @@ -487,7 +526,7 @@ func (r *Release) SetGitPaths(clonePath, subPath string) error { } func parseEnv(data string) (string, error) { - dataWithEnv := os.Expand(string(data), envMapper) + dataWithEnv := os.Expand(data, envMapper) if strings.Contains(dataWithEnv, "_ENV_NOT_SET_") { return data, fmt.Errorf("course has env variables that are not properly set") } @@ -501,3 +540,7 @@ func envMapper(key string) string { color.Red("ERROR: environment variable %s is not set", key) return "_ENV_NOT_SET_" } + +func boolPtr(b bool) *bool { + return &b +} diff --git a/pkg/reckoner/namespace.go b/pkg/reckoner/namespace.go index 9bcde5ba..197f1e68 100644 --- a/pkg/reckoner/namespace.go +++ b/pkg/reckoner/namespace.go @@ -79,7 +79,7 @@ func (c *Client) NamespaceManagement() error { return err } for _, release := range c.CourseFile.Releases { - err := c.CreateOrPatchNamespace(release.NamespaceMgmt.Settings.Overwrite, release.Namespace, *release.NamespaceMgmt, namespaces) + err := c.CreateOrPatchNamespace(*release.NamespaceMgmt.Settings.Overwrite, release.Namespace, *release.NamespaceMgmt, namespaces) if err != nil { return err } diff --git a/pkg/reckoner/plot.go b/pkg/reckoner/plot.go index b75621a4..71641e50 100644 --- a/pkg/reckoner/plot.go +++ b/pkg/reckoner/plot.go @@ -20,7 +20,7 @@ import ( "os" "strings" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/fairwindsops/reckoner/pkg/course" "github.com/fatih/color" diff --git a/pkg/reckoner/update.go b/pkg/reckoner/update.go index 30186d20..de85cf29 100644 --- a/pkg/reckoner/update.go +++ b/pkg/reckoner/update.go @@ -21,7 +21,7 @@ import ( func (c *Client) Update() error { updatedReleases := []*course.Release{} - for i, release := range c.CourseFile.Releases { + for idx, release := range c.CourseFile.Releases { thisReleaseDiff, err := c.diffRelease(release.Name, release.Namespace) if err != nil { if c.Continue() { @@ -32,7 +32,7 @@ func (c *Client) Update() error { } if thisReleaseDiff != "" { color.Yellow("Update available for %s in namespace %s. Added to plot list.", release.Name, release.Namespace) - updatedReleases = append(updatedReleases, c.CourseFile.Releases[i]) + updatedReleases = append(updatedReleases, c.CourseFile.Releases[idx]) continue } color.Green("No update necessary for %s in namespace %s.", release.Name, release.Namespace)