IAM refactor

Makefile

@@ -24,8 +24,9 @@ deps:
 	go get "github.com/golang/lint/golint"
 	go get "github.com/jstemmer/go-junit-report"
 	go get "github.com/aktau/github-release"
-	#go get -t -d -v $(SRC_FILES)
 	glide install
+	patch -p1 < go-git.v4.patch
+	gem install cfn-nag
 
 gen:
 	go generate $(SRC_FILES)
@@ -37,18 +38,31 @@ lint: fmt
 
 nag:
 	@echo "=== cfn_nag ==="
-	grep -l AWSTemplateFormatVersion: templates/assets/*.yml |xargs -t -n 1 cfn_nag
-
-
-test: lint gen
+	@mkdir -p $(BUILD_DIR)/cfn_nag
+	@grep -l AWSTemplateFormatVersion: templates/assets/*.yml | while read -r line; do \
+		filename=`basename $$line` ;\
+		grep -v '{{' $$line > $(BUILD_DIR)/cfn_nag/$$filename ;\
+		output=`cfn_nag_scan --input-path $(BUILD_DIR)/cfn_nag/$$filename 2>&1` ;\
+		if [ $$? -ne 0 ]; then \
+			echo "$$output\n" ;\
+		fi ;\
+	done | grep ".*" ;\
+    if [ $$? -eq 0 ]; then \
+    	exit 1 ;\
+    fi
+
+test: lint gen nag
 	@echo "=== testing ==="
 ifneq ($(CIRCLE_WORKING_DIRECTORY),)
 	mkdir -p $(CIRCLE_WORKING_DIRECTORY)/test-results/unit
-	go test -v -cover $(SRC_FILES) | go-junit-report > $(CIRCLE_WORKING_DIRECTORY)/test-results/unit/report.xml
+	go test -v -cover $(SRC_FILES) -short | go-junit-report > $(CIRCLE_WORKING_DIRECTORY)/test-results/unit/report.xml
 else
-	go test -cover $(SRC_FILES)
+	go test -cover $(SRC_FILES) -short
 endif
 
+e2e: gen stage keypair
+	@echo "=== e2e testing ==="
+	MU_VERSION=$(VERSION) MU_BASEURL=https://mu-staging-$$(aws sts get-caller-identity --output text --query 'Account').s3.amazonaws.com go test -v ./e2e -timeout 60m
 
 build: gen $(BUILD_FILES)
 
@@ -57,11 +71,27 @@ $(BUILD_FILES):
 	mkdir -p $(BUILD_DIR)
 	GOOS=$(word 2,$(subst -, ,$(notdir $@))) GOARCH=$(word 3,$(subst -, ,$(notdir $@))) go build -ldflags=$(GOLDFLAGS) -o '$@'
 
-install: build
-	@echo "=== building $(VERSION) - $(PACKAGE)-$(OS)-$(ARCH) ==="
+install: $(BUILD_DIR)/$(PACKAGE)-$(OS)-$(ARCH)
+	@echo "=== installing $(VERSION) - $(PACKAGE)-$(OS)-$(ARCH) ==="
 	cp $(BUILD_DIR)/$(PACKAGE)-$(OS)-$(ARCH) /usr/local/bin/mu
 	chmod 755 /usr/local/bin/mu
 
+keypair:
+	@aws ec2 describe-key-pairs --key-names mu-e2e > /dev/null 2>&1; \
+	if [ $$? -ne 0 ]; then \
+		echo "=== creating keypair ==="; \
+		aws ec2 create-key-pair --key-name mu-e2e --query "KeyMaterial" --output text > ~/.ssh/mu-e2e-$$(aws sts get-caller-identity --output text --query 'Account').pem; \
+		chmod 600 ~/.ssh/mu-e2e-$$(aws sts get-caller-identity --output text --query 'Account').pem; \
+	fi;
+
+stage: fmt $(BUILD_DIR)/$(PACKAGE)-linux-$(ARCH)
+	@echo "=== staging to S3 bucket ==="
+	@export BUCKET_NAME=mu-staging-$$(aws sts get-caller-identity --output text --query 'Account') ;\
+	aws s3 mb s3://$$BUCKET_NAME || echo "bucket exists" ;\
+	aws s3 website --index-document index.html s3://$$BUCKET_NAME ;\
+	aws s3 sync $(BUILD_DIR) s3://$$BUCKET_NAME/v$(VERSION)/ --acl public-read --exclude "*" --include "$(PACKAGE)-linux-*" ;\
+	echo https://$$BUCKET_NAME.s3.amazonaws.com
+
 release-clean:
 ifeq ($(IS_MASTER),)
 	@echo "=== clearing old release $(VERSION) ==="
@@ -103,4 +133,4 @@ fmt:
 	go fmt $(SRC_FILES)
 
 
-.PHONY: default all lint test build deps gen clean release-clean release-create dev-release release install $(UPLOAD_FILES) $(TARGET_OS)
+.PHONY: default all lint test e2e build deps gen clean release-clean release-create dev-release release install $(UPLOAD_FILES) $(BUILD_FILES) $(TARGET_OS) keypair stage

README.md

@@ -48,6 +48,7 @@ Refer to the wiki for complete details on the configuration of `mu.yml` and the
 * **[Services](https://github.com/stelligent/mu/wiki/Services)** - managing ECS service configuration
 * **[Databases](https://github.com/stelligent/mu/wiki/Databases)** - managing database configuration
 * **[Pipelines](https://github.com/stelligent/mu/wiki/Pipelines)** - managing continuous delivery pipelines
+* **[IAM](https://github.com/stelligent/mu/wiki/IAM)** - managing IAM roles that mu uses
 * **[CLI](https://github.com/stelligent/mu/wiki/CLI-Usage)** - details about using the CLI
 * **[Custom CloudFormation](https://github.com/stelligent/mu/wiki/Custom-CloudFormation)** - details about customizing the CloudFormation that is generated by mu.
 * **[Service Discovery](https://github.com/stelligent/mu/wiki/Service-Discovery)** - details about configuring and using service discovery
@@ -69,3 +70,6 @@ Want to contribute to Mu?  Awesome!  Check out the [contributing guidelines](CON
 * Go to src `cd $GOPATH/src/github.com/stelligent/mu`
 * Ensure [AWS CLI](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html) is configured with an access key, secret access key, and region.
 * Build with `make`
+* Run unit tests with `make test`
+* Run end-to-end tests with `make e2e`...takes about 30 minutes and will incur charges in your AWS account.
+

VERSION

@@ -1 +1 @@
-0.2.6
+1.0.1

cli/app.go

@@ -44,17 +44,11 @@ func NewApp() *cli.App {
 		}
 
 		// TODO: support initializing context from other cloud providers?
-		err = aws.InitializeContext(context, c.String("profile"), c.String("region"), c.Bool("dryrun"))
+		err = aws.InitializeContext(context, c.String("profile"), c.String("assume-role"), c.String("region"), c.Bool("dryrun"), c.Bool("skip-version-check"))
 		if err != nil {
 			return err
 		}
 
-		if c.Bool("silent") {
-			context.DockerOut = ioutil.Discard
-		} else {
-			context.DockerOut = os.Stdout
-		}
-
 		err = context.InitializeConfigFromFile(c.String("config"))
 		if err != nil {
 			// ignore errors for init command
@@ -63,6 +57,17 @@ func NewApp() *cli.App {
 			}
 		}
 
+		// Allow overriding the `DisableIAM` in config via `--disable-iam` or `-I`
+		if c.Bool("disable-iam") {
+			context.Config.DisableIAM = true
+		}
+
+		if c.Bool("silent") {
+			context.DockerOut = ioutil.Discard
+		} else {
+			context.DockerOut = os.Stdout
+		}
+
 		// Get the namespace for the stack creation.  This will prefix the stack names
 		// The order of precedence is command-line arg, env variable then config file
 		nameSpace := c.String("namespace")
@@ -95,6 +100,10 @@ func NewApp() *cli.App {
 			Name:  "region, r",
 			Usage: "AWS Region to use",
 		},
+		cli.StringFlag{
+			Name:  "assume-role, a",
+			Usage: "ARN of IAM role to assume",
+		},
 		cli.StringFlag{
 			Name:  "profile, p",
 			Usage: "AWS config profile to use",
@@ -115,6 +124,14 @@ func NewApp() *cli.App {
 			Name:  "dryrun, d",
 			Usage: "generate the cloudformation templates without upserting stacks",
 		},
+		cli.BoolFlag{
+			Name:  "disable-iam, I",
+			Usage: "disable the automatic creation of IAM resources",
+		},
+		cli.BoolFlag{
+			Name:  "skip-version-check, F",
+			Usage: "disable the checking of stack major numbers before updating",
+		},
 	}
 
 	return app

cli/app_test.go

@@ -11,17 +11,20 @@ func TestNewApp(t *testing.T) {
 
 	assert.NotNil(app)
 	assert.Equal("mu", app.Name, "Name should match")
-	assert.Equal("0.0.0-local", app.Version, "Version should match")
+	assert.Equal("1.0.0-local", app.Version, "Version should match")
 	assert.Equal("Microservice Platform on AWS", app.Usage, "usage should match")
 	assert.Equal(true, app.EnableBashCompletion, "bash completion should match")
-	assert.Equal(7, len(app.Flags), "Flags len should match")
+	assert.Equal(10, len(app.Flags), "Flags len should match")
 	assert.Equal("config, c", app.Flags[0].GetName(), "Flags name should match")
 	assert.Equal("region, r", app.Flags[1].GetName(), "Flags name should match")
-	assert.Equal("profile, p", app.Flags[2].GetName(), "Flags name should match")
-	assert.Equal("namespace, n", app.Flags[3].GetName(), "Flags name should match")
-	assert.Equal("silent, s", app.Flags[4].GetName(), "Flags name should match")
-	assert.Equal("verbose, V", app.Flags[5].GetName(), "Flags name should match")
-	assert.Equal("dryrun, d", app.Flags[6].GetName(), "Flags name should match")
+	assert.Equal("assume-role, a", app.Flags[2].GetName(), "Flags name should match")
+	assert.Equal("profile, p", app.Flags[3].GetName(), "Flags name should match")
+	assert.Equal("namespace, n", app.Flags[4].GetName(), "Flags name should match")
+	assert.Equal("silent, s", app.Flags[5].GetName(), "Flags name should match")
+	assert.Equal("verbose, V", app.Flags[6].GetName(), "Flags name should match")
+	assert.Equal("dryrun, d", app.Flags[7].GetName(), "Flags name should match")
+	assert.Equal("disable-iam, I", app.Flags[8].GetName(), "Flags name should match")
+	assert.Equal("skip-version-check, F", app.Flags[9].GetName(), "Flags name should match")
 	assert.Equal(5, len(app.Commands), "Commands len should match")
 	assert.Equal("init", app.Commands[0].Name, "Command[0].name should match")
 	assert.Equal("environment", app.Commands[1].Name, "Command[1].name should match")

common/const.go

@@ -9,5 +9,5 @@ var log = logging.MustGetLogger("common")
 // Constants used by common package
 const (
 	Empty          = ""
-	DefaultVersion = "0.0.0-local"
+	DefaultVersion = "1.0.0-local"
 )

common/context.go

@@ -83,15 +83,22 @@ func (ctx *Context) InitializeConfigFromFile(muFile string) error {
 			// See if the build was initiated by CodePipeline
 			if parts[0] == "codepipeline" {
 				// Try retrieving the revision from the CodePipeline status
-				gitInfo, err := ctx.PipelineManager.GetGitInfo(parts[1])
+				gitInfo, err := ctx.LocalPipelineManager.GetGitInfo(parts[1])
 				if err != nil {
-					log.Warningf("Unable to determine git information from CodeBuild initiator: %s", initiator)
+					log.Warningf("Unable to determine git information from CodeBuild initiator '%s': %s", initiator, err)
+				}
+
+				sourceVersion := os.Getenv("CODEBUILD_RESOLVED_SOURCE_VERSION")
+				if sourceVersion == "" {
+					sourceVersion = gitInfo.Revision
+				}
+				if len(sourceVersion) > 7 {
+					ctx.Config.Repo.Revision = string(sourceVersion[:7])
 				}
 
-				ctx.Config.Repo.Provider = gitInfo.Provider
-				ctx.Config.Repo.Revision = string(gitInfo.Revision[:7])
 				ctx.Config.Repo.Name = gitInfo.RepoName
 				ctx.Config.Repo.Slug = gitInfo.Slug
+				ctx.Config.Repo.Provider = gitInfo.Provider
 			} else {
 				log.Warningf("Unable to process CodeBuild initiator: %s", initiator)
 			}

common/roleset.go

@@ -0,0 +1,35 @@
+package common
+
+// Roleset is a map of Role ARNs keyed by role type
+type Roleset map[string]string
+
+// RolesetUpserter for managing a roleset
+type RolesetUpserter interface {
+	UpsertCommonRoleset() error
+	UpsertEnvironmentRoleset(environmentName string) error
+	UpsertServiceRoleset(environmentName string, serviceName string) error
+	UpsertPipelineRoleset(serviceName string) error
+}
+
+// RolesetGetter for getting a roleset
+type RolesetGetter interface {
+	GetCommonRoleset() (Roleset, error)
+	GetEnvironmentRoleset(environmentName string) (Roleset, error)
+	GetServiceRoleset(environmentName string, serviceName string) (Roleset, error)
+	GetPipelineRoleset(serviceName string) (Roleset, error)
+}
+
+// RolesetDeleter for deleting a roleset
+type RolesetDeleter interface {
+	DeleteCommonRoleset() error
+	DeleteEnvironmentRoleset(environmentName string) error
+	DeleteServiceRoleset(environmentName string, serviceName string) error
+	DeletePipelineRoleset(serviceName string) error
+}
+
+// RolesetManager composite of all roleset capabilities
+type RolesetManager interface {
+	RolesetGetter
+	RolesetUpserter
+	RolesetDeleter
+}

common/stack.go

@@ -33,7 +33,7 @@ type StackWaiter interface {
 
 // StackUpserter for applying changes to a stack
 type StackUpserter interface {
-	UpsertStack(stackName string, templateBodyReader io.Reader, parameters map[string]string, tags map[string]string) error
+	UpsertStack(stackName string, templateBodyReader io.Reader, parameters map[string]string, tags map[string]string, roleArn string) error
 }
 
 // StackLister for listing stacks

common/types.go

@@ -7,19 +7,21 @@ import (
 
 // Context defines the context object passed around
 type Context struct {
-	Config          Config
-	StackManager    StackManager
-	ClusterManager  ClusterManager
-	InstanceManager InstanceManager
-	ElbManager      ElbManager
-	RdsManager      RdsManager
-	ParamManager    ParamManager
-	PipelineManager PipelineManager
-	LogsManager     LogsManager
-	DockerManager   DockerManager
-	DockerOut       io.Writer
-	TaskManager     TaskManager
-	ArtifactManager ArtifactManager
+	Config               Config
+	StackManager         StackManager
+	ClusterManager       ClusterManager
+	InstanceManager      InstanceManager
+	ElbManager           ElbManager
+	RdsManager           RdsManager
+	ParamManager         ParamManager
+	LocalPipelineManager PipelineManager // instance that ignores region/profile/role
+	PipelineManager      PipelineManager
+	LogsManager          LogsManager
+	DockerManager        DockerManager
+	DockerOut            io.Writer
+	TaskManager          TaskManager
+	ArtifactManager      ArtifactManager
+	RolesetManager       RolesetManager
 }
 
 // Config defines the structure of the yml file for the mu config
@@ -36,7 +38,11 @@ type Config struct {
 		Branch   string
 		Provider string
 	} `yaml:"-"`
-	Templates map[string]interface{} `yaml:"templates,omitempty"`
+	Templates  map[string]interface{} `yaml:"templates,omitempty"`
+	DisableIAM bool                   `yaml:"disableIAM,omitempty"`
+	Roles      struct {
+		CloudFormation string `yaml:"cloudFormation,omitempty"`
+	} `yaml:"roles,omitempty"`
 }
 
 // Environment defines the structure of the yml file for an environment
@@ -71,6 +77,12 @@ type Environment struct {
 		InstanceSubnetIds []string `yaml:"instanceSubnetIds,omitempty"`
 		ElbSubnetIds      []string `yaml:"elbSubnetIds,omitempty"`
 	} `yaml:"vpcTarget,omitempty"`
+	Roles struct {
+		EcsInstance      string `yaml:"ecsInstance,omitempty"`
+		ConsulClientTask string `yaml:"consulClientTask,omitempty"`
+		ConsulInstance   string `yaml:"consulInstance,omitempty"`
+		ConsulServerTask string `yaml:"consulServerTask,omitempty"`
+	} `yaml:"roles,omitempty"`
 }
 
 // Service defines the structure of the yml file for a service
@@ -91,6 +103,12 @@ type Service struct {
 	Priority        int                    `yaml:"priority,omitempty"`
 	Pipeline        Pipeline               `yaml:"pipeline,omitempty"`
 	Database        Database               `yaml:"database,omitempty"`
+	Roles           struct {
+		Ec2Instance string `yaml:"ec2Instance,omitempty"`
+		CodeDeploy  string `yaml:"codeDeploy,omitempty"`
+		EcsService  string `yaml:"ecsService,omitempty"`
+		EcsTask     string `yaml:"ecsTask,omitempty"`
+	} `yaml:"roles,omitempty"`
 }
 
 // Database definition
@@ -124,13 +142,25 @@ type Pipeline struct {
 		Type        string `yaml:"type,omitempty"`
 		ComputeType string `yaml:"computeType,omitempty"`
 		Image       string `yaml:"image,omitempty"`
+		Roles       struct {
+			CodeBuild string `yaml:"codeBuild,omitempty"`
+			Mu        string `yaml:"mu,omitempty"`
+		} `yaml:"roles,omitempty"`
 	} `yaml:"acceptance,omitempty"`
 	Production struct {
 		Disabled    bool   `yaml:"disabled,omitempty"`
 		Environment string `yaml:"environment,omitempty"`
+		Roles       struct {
+			CodeBuild string `yaml:"codeBuild,omitempty"`
+			Mu        string `yaml:"mu,omitempty"`
+		} `yaml:"roles,omitempty"`
 	} `yaml:"production,omitempty"`
 	MuBaseurl string `yaml:"muBaseurl,omitempty"`
 	MuVersion string `yaml:"muVersion,omitempty"`
+	Roles     struct {
+		Pipeline string `yaml:"pipeline,omitempty"`
+		Build    string `yaml:"build,omitempty"`
+	} `yaml:"roles,omitempty"`
 }
 
 // Stack summary
@@ -205,6 +235,7 @@ type StackType string
 const (
 	StackTypeVpc          StackType = "vpc"
 	StackTypeTarget                 = "target"
+	StackTypeIam                    = "iam"
 	StackTypeEnv                    = "environment"
 	StackTypeLoadBalancer           = "loadbalancer"
 	StackTypeConsul                 = "consul"

e2e/e2e-basic/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.4-alpine
+WORKDIR /code
+ADD . /code
+RUN pip install -r requirements.txt
+ENV FLASK_APP app.py
+CMD ["flask", "run","--host=0.0.0.0"]