Merge pull request #180 from stelligent/develop

v1.0.0

.circleci/config.yml

@@ -0,0 +1,73 @@
+version: 2
+
+jobs:
+  build:
+    machine: true
+
+    ## Don't build when new release tags are pushed
+    branches:
+      ignore:
+        - /^v[0-9]/
+
+    steps:
+      - run:
+          name: Install Glide
+          command: |
+            sudo add-apt-repository ppa:masterminds/glide -y
+            sudo apt-get update
+            sudo apt-get install glide -y
+
+      - checkout
+
+      - run:
+          name: Setup Source
+          command: |
+            IMPORT_PATH=/home/circleci/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME
+            APP_PATH=$IMPORT_PATH/$CIRCLE_PROJECT_REPONAME
+            mkdir -p $IMPORT_PATH
+            ln -sf "$(pwd)" $APP_PATH
+            echo "export APP_PATH=$APP_PATH" >> $BASH_ENV
+
+      - run:
+          name: Dependencies
+          command: cd $APP_PATH && make deps
+
+      - run:
+          name: Compile
+          command: cd $APP_PATH && make build
+
+      - run:
+          name: Test
+          command: cd $APP_PATH && make test
+
+      - store_test_results:
+          path: test-results
+
+      - store_artifacts:
+          path: artifacts
+
+      - add_ssh_keys:
+          fingerprints:
+            - "53:1f:d0:6f:24:2a:bf:cb:92:0f:bc:1e:14:6e:1d:72"
+
+      ## In order for deploy to work, you need an environment variable set in CircleCI named "GITHUB_TOKEN" that contains
+      ##  a token created from https://github.com/settings/tokens
+      ##
+      ## Also, an read/write SSH key needs to be created and private key added to circle ci
+      ##    as per https://circleci.com/docs/adding-read-write-deployment-key/
+      ##
+      ## Suggest creating a machine user for the token and ssh key to limit access
+      ##
+      - deploy:
+          name: Deploy
+          command: |
+            cd $APP_PATH
+            if [ "${CIRCLE_BRANCH}" == "develop" ]; then
+              make dev-release
+            elif [ "${CIRCLE_BRANCH}" == "master" ]; then
+              make release
+            fi
+
+
+
+

.gitignore

@@ -2,3 +2,4 @@
 .idea
 .release/
 vendor/
+*~

Makefile

@@ -9,7 +9,8 @@ VERSION := $(shell cat VERSION)$(if $(IS_MASTER),,-$(BRANCH))
 SRC_FILES = $(shell glide nv)
 ARCH := $(shell go env GOARCH)
 OS := $(shell go env GOOS)
-BUILD_DIR = $(if $(CIRCLE_ARTIFACTS),$(CIRCLE_ARTIFACTS),.release)
+BUILD_DIR = .release
+BUILD_DIR = $(abspath $(if $(CIRCLE_WORKING_DIRECTORY),$(CIRCLE_WORKING_DIRECTORY)/artifacts,.release))
 BUILD_FILES = $(foreach os, $(TARGET_OS), $(BUILD_DIR)/$(PACKAGE)-$(os)-$(ARCH))
 UPLOAD_FILES = $(foreach os, $(TARGET_OS), $(PACKAGE)-$(os)-$(ARCH))
 GOLDFLAGS = "-X main.version=$(VERSION)"
@@ -23,8 +24,9 @@ deps:
 	go get "github.com/golang/lint/golint"
 	go get "github.com/jstemmer/go-junit-report"
 	go get "github.com/aktau/github-release"
-	#go get -t -d -v $(SRC_FILES)
 	glide install
+	patch -p1 < go-git.v4.patch
+	gem install cfn-nag
 
 gen:
 	go generate $(SRC_FILES)
@@ -36,18 +38,31 @@ lint: fmt
 
 nag:
 	@echo "=== cfn_nag ==="
-	grep -l AWSTemplateFormatVersion: templates/assets/*.yml |xargs -t -n 1 cfn_nag
-
-
-test: lint gen
+	@mkdir -p $(BUILD_DIR)/cfn_nag
+	@grep -l AWSTemplateFormatVersion: templates/assets/*.yml | while read -r line; do \
+		filename=`basename $$line` ;\
+		grep -v '{{' $$line > $(BUILD_DIR)/cfn_nag/$$filename ;\
+		output=`cfn_nag_scan --input-path $(BUILD_DIR)/cfn_nag/$$filename 2>&1` ;\
+		if [ $$? -ne 0 ]; then \
+			echo "$$output\n" ;\
+		fi ;\
+	done | grep ".*" ;\
+    if [ $$? -eq 0 ]; then \
+    	exit 1 ;\
+    fi
+
+test: lint gen nag
 	@echo "=== testing ==="
-ifneq ($(CIRCLE_TEST_REPORTS),)
-	mkdir -p $(CIRCLE_TEST_REPORTS)/unit
-	go test -v -cover $(SRC_FILES) | go-junit-report > $(CIRCLE_TEST_REPORTS)/unit/report.xml
+ifneq ($(CIRCLE_WORKING_DIRECTORY),)
+	mkdir -p $(CIRCLE_WORKING_DIRECTORY)/test-results/unit
+	go test -v -cover $(SRC_FILES) -short | go-junit-report > $(CIRCLE_WORKING_DIRECTORY)/test-results/unit/report.xml
 else
-	go test -cover $(SRC_FILES)
+	go test -cover $(SRC_FILES) -short
 endif
 
+e2e: gen stage keypair
+	@echo "=== e2e testing ==="
+	MU_VERSION=$(VERSION) MU_BASEURL=https://mu-staging-$$(aws sts get-caller-identity --output text --query 'Account').s3.amazonaws.com go test -v ./e2e -timeout 60m
 
 build: gen $(BUILD_FILES)
 
@@ -56,11 +71,27 @@ $(BUILD_FILES):
 	mkdir -p $(BUILD_DIR)
 	GOOS=$(word 2,$(subst -, ,$(notdir $@))) GOARCH=$(word 3,$(subst -, ,$(notdir $@))) go build -ldflags=$(GOLDFLAGS) -o '$@'
 
-install: build
-	@echo "=== building $(VERSION) - $(PACKAGE)-$(OS)-$(ARCH) ==="
+install: $(BUILD_DIR)/$(PACKAGE)-$(OS)-$(ARCH)
+	@echo "=== installing $(VERSION) - $(PACKAGE)-$(OS)-$(ARCH) ==="
 	cp $(BUILD_DIR)/$(PACKAGE)-$(OS)-$(ARCH) /usr/local/bin/mu
 	chmod 755 /usr/local/bin/mu
 
+keypair:
+	@aws ec2 describe-key-pairs --key-names mu-e2e > /dev/null 2>&1; \
+	if [ $$? -ne 0 ]; then \
+		echo "=== creating keypair ==="; \
+		aws ec2 create-key-pair --key-name mu-e2e --query "KeyMaterial" --output text > ~/.ssh/mu-e2e-$$(aws sts get-caller-identity --output text --query 'Account').pem; \
+		chmod 600 ~/.ssh/mu-e2e-$$(aws sts get-caller-identity --output text --query 'Account').pem; \
+	fi;
+
+stage: fmt $(BUILD_DIR)/$(PACKAGE)-linux-$(ARCH)
+	@echo "=== staging to S3 bucket ==="
+	@export BUCKET_NAME=mu-staging-$$(aws sts get-caller-identity --output text --query 'Account') ;\
+	aws s3 mb s3://$$BUCKET_NAME || echo "bucket exists" ;\
+	aws s3 website --index-document index.html s3://$$BUCKET_NAME ;\
+	aws s3 sync $(BUILD_DIR) s3://$$BUCKET_NAME/v$(VERSION)/ --acl public-read --exclude "*" --include "$(PACKAGE)-linux-*" ;\
+	echo https://$$BUCKET_NAME.s3.amazonaws.com
+
 release-clean:
 ifeq ($(IS_MASTER),)
 	@echo "=== clearing old release $(VERSION) ==="
@@ -102,4 +133,4 @@ fmt:
 	go fmt $(SRC_FILES)
 
 
-.PHONY: default all lint test build deps gen clean release-clean release-create dev-release release install $(UPLOAD_FILES) $(TARGET_OS)
+.PHONY: default all lint test e2e build deps gen clean release-clean release-create dev-release release install $(UPLOAD_FILES) $(BUILD_FILES) $(TARGET_OS) keypair stage

README.md

@@ -48,11 +48,16 @@ Refer to the wiki for complete details on the configuration of `mu.yml` and the
 * **[Services](https://github.com/stelligent/mu/wiki/Services)** - managing ECS service configuration
 * **[Databases](https://github.com/stelligent/mu/wiki/Databases)** - managing database configuration
 * **[Pipelines](https://github.com/stelligent/mu/wiki/Pipelines)** - managing continuous delivery pipelines
+* **[IAM](https://github.com/stelligent/mu/wiki/IAM)** - managing IAM roles that mu uses
 * **[CLI](https://github.com/stelligent/mu/wiki/CLI-Usage)** - details about using the CLI
 * **[Custom CloudFormation](https://github.com/stelligent/mu/wiki/Custom-CloudFormation)** - details about customizing the CloudFormation that is generated by mu.
 * **[Service Discovery](https://github.com/stelligent/mu/wiki/Service-Discovery)** - details about configuring and using service discovery
 * **[Traditional Infrastructure](https://github.com/stelligent/mu/wiki/Traditional-Infrastructure)** - details about using traditional infrastructure (EC2 instances) for running services, rather than ECS and Docker.
 
+# Support
+
+Need help?  Check out the [FAQ](https://github.com/stelligent/mu/wiki/FAQ) to try to find an answer to your question.  If you can't find an answer there, ask on [Gitter](https://gitter.im/stelligent/mu)!
+
 # Contributing
 
 Want to contribute to Mu?  Awesome!  Check out the [contributing guidelines](CONTRIBUTING.md) to get involved.
@@ -65,3 +70,6 @@ Want to contribute to Mu?  Awesome!  Check out the [contributing guidelines](CON
 * Go to src `cd $GOPATH/src/github.com/stelligent/mu`
 * Ensure [AWS CLI](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html) is configured with an access key, secret access key, and region.
 * Build with `make`
+* Run unit tests with `make test`
+* Run end-to-end tests with `make e2e`...takes about 30 minutes and will incur charges in your AWS account.
+

VERSION

@@ -1 +1 @@
-0.2.5
+1.0.1

cli/app.go

@@ -44,24 +44,47 @@ func NewApp() *cli.App {
 		}
 
 		// TODO: support initializing context from other cloud providers?
-		err = aws.InitializeContext(context, c.String("profile"), c.String("region"), c.Bool("dryrun"))
+		err = aws.InitializeContext(context, c.String("profile"), c.String("assume-role"), c.String("region"), c.Bool("dryrun"), c.Bool("skip-version-check"))
 		if err != nil {
 			return err
 		}
 
+		err = context.InitializeConfigFromFile(c.String("config"))
+		if err != nil {
+			// ignore errors for init command
+			if c.Args().First() != "init" {
+				log.Warningf("Unable to load mu config: %v", err)
+			}
+		}
+
+		// Allow overriding the `DisableIAM` in config via `--disable-iam` or `-I`
+		if c.Bool("disable-iam") {
+			context.Config.DisableIAM = true
+		}
+
 		if c.Bool("silent") {
 			context.DockerOut = ioutil.Discard
 		} else {
 			context.DockerOut = os.Stdout
 		}
 
-		err = context.InitializeConfigFromFile(c.String("config"))
-		if err != nil {
-			// ignore errors for init command
-			if c.Args().First() != "init" {
-				log.Warningf("Unable to load mu config: %v", err)
+		// Get the namespace for the stack creation.  This will prefix the stack names
+		// The order of precedence is command-line arg, env variable then config file
+		nameSpace := c.String("namespace")
+		if nameSpace != "" {
+			log.Debug("Using namespace \"%s\"", nameSpace)
+			context.Config.Namespace = nameSpace
+		} else {
+			nameSpace = os.Getenv("MU_NAMESPACE")
+			if nameSpace != "" {
+				log.Debug("Using namespace \"%s\"", nameSpace)
+				context.Config.Namespace = nameSpace
 			}
 		}
+		if context.Config.Namespace == "" {
+			log.Debug("Using namespace \"mu\"")
+			context.Config.Namespace = "mu"
+		}
 
 		return nil
 
@@ -77,10 +100,18 @@ func NewApp() *cli.App {
 			Name:  "region, r",
 			Usage: "AWS Region to use",
 		},
+		cli.StringFlag{
+			Name:  "assume-role, a",
+			Usage: "ARN of IAM role to assume",
+		},
 		cli.StringFlag{
 			Name:  "profile, p",
 			Usage: "AWS config profile to use",
 		},
+		cli.StringFlag{
+			Name:  "namespace, n",
+			Usage: "Namespace to use as a prefix for stacks",
+		},
 		cli.BoolFlag{
 			Name:  "silent, s",
 			Usage: "silent mode, errors only",
@@ -93,6 +124,14 @@ func NewApp() *cli.App {
 			Name:  "dryrun, d",
 			Usage: "generate the cloudformation templates without upserting stacks",
 		},
+		cli.BoolFlag{
+			Name:  "disable-iam, I",
+			Usage: "disable the automatic creation of IAM resources",
+		},
+		cli.BoolFlag{
+			Name:  "skip-version-check, F",
+			Usage: "disable the checking of stack major numbers before updating",
+		},
 	}
 
 	return app

cli/app_test.go

@@ -11,16 +11,20 @@ func TestNewApp(t *testing.T) {
 
 	assert.NotNil(app)
 	assert.Equal("mu", app.Name, "Name should match")
-	assert.Equal("0.0.0-local", app.Version, "Version should match")
+	assert.Equal("1.0.0-local", app.Version, "Version should match")
 	assert.Equal("Microservice Platform on AWS", app.Usage, "usage should match")
 	assert.Equal(true, app.EnableBashCompletion, "bash completion should match")
-	assert.Equal(6, len(app.Flags), "Flags len should match")
+	assert.Equal(10, len(app.Flags), "Flags len should match")
 	assert.Equal("config, c", app.Flags[0].GetName(), "Flags name should match")
 	assert.Equal("region, r", app.Flags[1].GetName(), "Flags name should match")
-	assert.Equal("profile, p", app.Flags[2].GetName(), "Flags name should match")
-	assert.Equal("silent, s", app.Flags[3].GetName(), "Flags name should match")
-	assert.Equal("verbose, V", app.Flags[4].GetName(), "Flags name should match")
-	assert.Equal("dryrun, d", app.Flags[5].GetName(), "Flags name should match")
+	assert.Equal("assume-role, a", app.Flags[2].GetName(), "Flags name should match")
+	assert.Equal("profile, p", app.Flags[3].GetName(), "Flags name should match")
+	assert.Equal("namespace, n", app.Flags[4].GetName(), "Flags name should match")
+	assert.Equal("silent, s", app.Flags[5].GetName(), "Flags name should match")
+	assert.Equal("verbose, V", app.Flags[6].GetName(), "Flags name should match")
+	assert.Equal("dryrun, d", app.Flags[7].GetName(), "Flags name should match")
+	assert.Equal("disable-iam, I", app.Flags[8].GetName(), "Flags name should match")
+	assert.Equal("skip-version-check, F", app.Flags[9].GetName(), "Flags name should match")
 	assert.Equal(5, len(app.Commands), "Commands len should match")
 	assert.Equal("init", app.Commands[0].Name, "Command[0].name should match")
 	assert.Equal("environment", app.Commands[1].Name, "Command[1].name should match")