Skip to content

Commit

Permalink
chore(addons): merge two cfn template's Metadata section (#995)
Browse files Browse the repository at this point in the history
Merges the ["Metadata"](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/metadata-section-structure.html) section of two cfn templates under "addons/".

Related to #994 

_By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._
  • Loading branch information
efekarakus committed Jun 8, 2020
1 parent e77120a commit 576179d
Show file tree
Hide file tree
Showing 11 changed files with 378 additions and 98 deletions.
12 changes: 1 addition & 11 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,17 @@ go 1.13

require (
github.com/AlecAivazis/survey/v2 v2.0.7
github.com/Microsoft/hcsshim v0.8.9 // indirect
github.com/Netflix/go-expect v0.0.0-20190729225929-0e00d9168667 // indirect
github.com/aws/aws-sdk-go v1.31.12
github.com/awslabs/goformation/v4 v4.8.0
github.com/briandowns/spinner v1.11.1
github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect
github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b // indirect
github.com/containerd/ttrpc v1.0.1 // indirect
github.com/containerd/typeurl v1.0.1 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/dustin/go-humanize v1.0.0
github.com/fatih/color v1.9.0
github.com/fatih/structs v1.1.0
github.com/gobuffalo/packd v1.0.0
github.com/gobuffalo/packr/v2 v2.8.0
github.com/gogo/googleapis v1.4.0 // indirect
github.com/golang/mock v1.4.3
github.com/google/uuid v1.1.1
github.com/hashicorp/hcl v1.0.0
github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c // indirect
github.com/imdario/mergo v0.3.9
github.com/karrick/godirwalk v1.15.6 // indirect
Expand All @@ -31,8 +23,6 @@ require (
github.com/moby/buildkit v0.7.1
github.com/onsi/ginkgo v1.12.3
github.com/onsi/gomega v1.10.1
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/pelletier/go-toml v1.8.0 // indirect
github.com/sirupsen/logrus v1.6.0 // indirect
github.com/spf13/afero v1.2.2
Expand All @@ -44,7 +34,7 @@ require (
github.com/stretchr/testify v1.6.1
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 // indirect
gopkg.in/ini.v1 v1.57.0
gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c
)

replace github.com/containerd/containerd => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55
Expand Down
92 changes: 6 additions & 86 deletions go.sum

Large diffs are not rendered by default.

36 changes: 36 additions & 0 deletions internal/pkg/addons/addons.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/aws/amazon-ecs-cli-v2/internal/pkg/template"
"github.com/aws/amazon-ecs-cli-v2/internal/pkg/workspace"
"gopkg.in/yaml.v3"
)

const (
Expand Down Expand Up @@ -98,6 +99,41 @@ func (a *Addons) Template() (string, error) {
return content.String(), nil
}

// template will replace Template.
// template merges CloudFormation templates under the "addons/" directory of a service
// into a single CloudFormation template and returns it.
//
// If the addons directory doesn't exist, it returns the empty string and ErrDirNotExist.
func (a *Addons) template() (string, error) {
fnames, err := a.ws.ReadAddonsDir(a.svcName)
if err != nil {
return "", &ErrDirNotExist{
SvcName: a.svcName,
ParentErr: err,
}
}

var mergedTemplate cfnTemplate
for _, fname := range filterYAMLfiles(fnames) {
out, err := a.ws.ReadAddon(a.svcName, fname)
if err != nil {
return "", fmt.Errorf("read addon %s under service %s: %w", fname, a.svcName, err)
}
var tpl cfnTemplate
if err := yaml.Unmarshal(out, &tpl); err != nil {
return "", fmt.Errorf("unmarshal addon %s under service %s: %w", fname, a.svcName, err)
}
if err := mergedTemplate.merge(tpl); err != nil {
return "", fmt.Errorf("merge addon %s under service %s: %w", fname, a.svcName, err)
}
}
out, err := yaml.Marshal(&mergedTemplate)
if err != nil {
return "", fmt.Errorf("marshal merged addons template: %w", err)
}
return string(out), nil
}

func filterYAMLfiles(files []string) []string {
yamlExtensions := []string{".yaml", ".yml"}

Expand Down
86 changes: 86 additions & 0 deletions internal/pkg/addons/addons_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ package addons
import (
"bytes"
"errors"
"io/ioutil"
"path/filepath"
"testing"

"github.com/aws/amazon-ecs-cli-v2/internal/pkg/addons/mocks"
Expand Down Expand Up @@ -106,3 +108,87 @@ func TestAddons_Template(t *testing.T) {
})
}
}

func TestAddons_template(t *testing.T) {
const testSvcName = "mysvc"
testErr := errors.New("some error")
testCases := map[string]struct {
mockAddons func(ctrl *gomock.Controller) *Addons

wantedTemplate string
wantedErr error
}{
"return ErrDirNotExist if addons doesn't exist in a service": {
mockAddons: func(ctrl *gomock.Controller) *Addons {
ws := mocks.NewMockworkspaceReader(ctrl)
ws.EXPECT().ReadAddonsDir(testSvcName).
Return(nil, testErr)
return &Addons{
svcName: testSvcName,
ws: ws,
}
},
wantedErr: &ErrDirNotExist{
SvcName: testSvcName,
ParentErr: testErr,
},
},
"merge invalid Metadata fields": {
mockAddons: func(ctrl *gomock.Controller) *Addons {
ws := mocks.NewMockworkspaceReader(ctrl)
ws.EXPECT().ReadAddonsDir(testSvcName).Return([]string{"first.yaml", "invalid-second.yaml"}, nil)

first, _ := ioutil.ReadFile(filepath.Join("testdata", "metadata", "first.yaml"))
ws.EXPECT().ReadAddon(testSvcName, "first.yaml").Return(first, nil)

second, _ := ioutil.ReadFile(filepath.Join("testdata", "metadata", "invalid-second.yaml"))
ws.EXPECT().ReadAddon(testSvcName, "invalid-second.yaml").Return(second, nil)
return &Addons{
svcName: testSvcName,
ws: ws,
}
},
wantedErr: errors.New("merge addon invalid-second.yaml under service mysvc: metadata key Services already exists with a different definition"),
},
"merge Metadata fields successfully": {
mockAddons: func(ctrl *gomock.Controller) *Addons {
ws := mocks.NewMockworkspaceReader(ctrl)
ws.EXPECT().ReadAddonsDir(testSvcName).Return([]string{"first.yaml", "valid-second.yaml"}, nil)

first, _ := ioutil.ReadFile(filepath.Join("testdata", "metadata", "first.yaml"))
ws.EXPECT().ReadAddon(testSvcName, "first.yaml").Return(first, nil)

second, _ := ioutil.ReadFile(filepath.Join("testdata", "metadata", "valid-second.yaml"))
ws.EXPECT().ReadAddon(testSvcName, "valid-second.yaml").Return(second, nil)
return &Addons{
svcName: testSvcName,
ws: ws,
}
},
wantedTemplate: func() string {
wanted, _ := ioutil.ReadFile(filepath.Join("testdata", "metadata", "wanted.yaml"))
return string(wanted)
}(),
},
}

for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
// GIVEN
ctrl := gomock.NewController(t)
defer ctrl.Finish()
addons := tc.mockAddons(ctrl)

// WHEN
actualTemplate, actualErr := addons.template()

// THEN
if tc.wantedErr != nil {
require.EqualError(t, tc.wantedErr, actualErr.Error())
} else {
require.NoError(t, actualErr)
require.Equal(t, tc.wantedTemplate, actualTemplate)
}
})
}
}
115 changes: 115 additions & 0 deletions internal/pkg/addons/cloudformation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package addons

import (
"gopkg.in/yaml.v3"
)

type cfnSection int

const (
metadataSection cfnSection = iota + 1
)

// cfnTemplate represents a parsed YAML AWS CloudFormation template.
type cfnTemplate struct {
Metadata yaml.Node `yaml:"Metadata,omitempty"`
}

// merge combines non-empty fields of other with cf's fields.
func (t *cfnTemplate) merge(other cfnTemplate) error {
if err := t.mergeMetadata(other.Metadata); err != nil {
return err
}
return nil
}

// mergeMetadata updates cf's Metadata with additional metadata.
// If the key already exists in Metadata but with a different definition, returns errMetadataKeyAlreadyExists.
func (t *cfnTemplate) mergeMetadata(metadata yaml.Node) error {
if err := mergeMapNodes(&t.Metadata, &metadata); err != nil {
return wrapKeyAlreadyExistsErr(metadataSection, err)
}
return nil
}

// mergeMapNodes merges the src node to dst.
// It assumes that both nodes have a "mapping" type. See https://yaml.org/spec/1.2/spec.html#id2802432
//
// If a key in src already exists in dst and the values are different, then returns a errKeyAlreadyExists.
// If a key in src already exists in dst and the values are equal, then do nothing.
// If a key in src doesn't exist in dst, then add the key and its value to dst.
func mergeMapNodes(dst, src *yaml.Node) error {
if src.IsZero() {
return nil
}

if dst.IsZero() {
*dst = *src
return nil
}

dstMap := mappingNode(dst)
var newContent []*yaml.Node
for i := 0; i < len(src.Content); i += 2 {
// The content of a map always come in pairs.
// The first element represents a key, ex: {Value: "ELBIngressGroup", Kind: ScalarNode, Tag: "!!str", Content: nil}
// The second element holds the value, ex: {Value: "", Kind: MappingNode, Tag:"!!map", Content:[...]}
key := src.Content[i].Value
srcValue := src.Content[i+1]

dstValue, ok := dstMap[key]
if !ok {
// The key doesn't exist in dst, we want to retain the two src nodes.
newContent = append(newContent, src.Content[i], src.Content[i+1])
continue
}

if !isEqual(dstValue, srcValue) {
return &errKeyAlreadyExists{
Key: key,
First: dstValue,
Second: srcValue,
}
}
}
dst.Content = append(dst.Content, newContent...)
return nil
}

// mappingNode transforms a flat "mapping" yaml.Node to a hashmap.
func mappingNode(n *yaml.Node) map[string]*yaml.Node {
m := make(map[string]*yaml.Node)
for i := 0; i < len(n.Content); i += 2 {
m[n.Content[i].Value] = n.Content[i+1]
}
return m
}

// isEqual returns true if the first and second nodes are deeply equal in all of their values except stylistic ones.
//
// We ignore the style (ex: single quote vs. double) in which the nodes are defined, the comments associated with
// the nodes, and the indentation and position of the nodes as they're only visual properties and don't matter.
func isEqual(first *yaml.Node, second *yaml.Node) bool {
if first == nil {
return second == nil
}
if second == nil {
return false
}
if len(first.Content) != len(second.Content) {
return false
}
hasSameContent := true
for i := 0; i < len(first.Content); i += 1 {
hasSameContent = hasSameContent && isEqual(first.Content[i], second.Content[i])
}
return first.Kind == second.Kind &&
first.Tag == second.Tag &&
first.Value == second.Value &&
first.Anchor == second.Anchor &&
isEqual(first.Alias, second.Alias) &&
hasSameContent
}
58 changes: 57 additions & 1 deletion internal/pkg/addons/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,14 @@

package addons

import "fmt"
import (
"errors"
"fmt"
"strings"

"github.com/aws/amazon-ecs-cli-v2/internal/pkg/term/color"
"gopkg.in/yaml.v3"
)

// ErrDirNotExist occurs when an addons directory for a service does not exist.
type ErrDirNotExist struct {
Expand All @@ -14,3 +21,52 @@ type ErrDirNotExist struct {
func (e *ErrDirNotExist) Error() string {
return fmt.Sprintf("read addons directory for service %s: %v", e.SvcName, e.ParentErr)
}

type errKeyAlreadyExists struct {
Key string
First *yaml.Node
Second *yaml.Node
}

func (e *errKeyAlreadyExists) Error() string {
return fmt.Sprintf("key %s already exists with a different definition", e.Key)
}

// errMetadataKeyAlreadyExists occurs if two addons have the same key in their "Metadata" section but with different values.
type errMetadataKeyAlreadyExists struct {
*errKeyAlreadyExists
}

func (e *errMetadataKeyAlreadyExists) Error() string {
return fmt.Sprintf("metadata key %s already exists with a different definition", e.Key)
}

// HumanError returns a string that explains the error with human-friendly details.
func (e *errMetadataKeyAlreadyExists) HumanError() string {
fout, _ := yaml.Marshal(e.First)
sout, _ := yaml.Marshal(e.Second)
return fmt.Sprintf(`The "Metadata" key %s exists with two different values under addons:
%s
and
%s`,
color.HighlightCode(e.Key),
color.HighlightCode(strings.TrimSpace(string(fout))),
color.HighlightCode(strings.TrimSpace(string(sout))))
}

// wrapKeyAlreadyExistsErr wraps the err if its an errKeyAlreadyExists error with additional cfn section metadata.
// If the error is not an errKeyAlreadyExists, then return it as is.
func wrapKeyAlreadyExistsErr(section cfnSection, err error) error {
var keyExistsErr *errKeyAlreadyExists
if !errors.As(err, &keyExistsErr) {
return err
}
switch section {
case metadataSection:
return &errMetadataKeyAlreadyExists{
errKeyAlreadyExists: keyExistsErr,
}
default:
return err
}
}

0 comments on commit 576179d

Please sign in to comment.