Merge pull request #106 from stelligent/develop

Release 0.1.8
stelligent · Mar 8, 2017 · 358f42b · 358f42b
2 parents be7fa52 + bc54aa4
commit 358f42b
Show file tree

Hide file tree

Showing 60 changed files with 1,807 additions and 527 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -0,0 +1,4 @@
+# Override for Makefile
+[{Makefile, makefile, GNUmakefile}]
+indent_style = tab
+indent_size = 4
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "wiki"]
+	path = wiki
+	url = git@github.com:stelligent/mu.wiki.git
+	branch = master
diff --git a/Makefile b/Makefile
@@ -8,6 +8,7 @@ IS_MASTER := $(filter master, $(BRANCH))
 VERSION := $(shell cat VERSION)$(if $(IS_MASTER),,-$(BRANCH))
 SRC_FILES = $(shell glide nv)
 ARCH := $(shell go env GOARCH)
+OS := $(shell go env GOOS)
 BUILD_DIR = $(if $(CIRCLE_ARTIFACTS),$(CIRCLE_ARTIFACTS),.release)
 BUILD_FILES = $(foreach os, $(TARGET_OS), $(BUILD_DIR)/$(PACKAGE)-$(os)-$(ARCH))
 UPLOAD_FILES = $(foreach os, $(TARGET_OS), $(PACKAGE)-$(os)-$(ARCH))
@@ -24,14 +25,16 @@ deps:
 	go get "github.com/aktau/github-release"
 	#go get -t -d -v $(SRC_FILES)
 	glide install
+
+gen:
 	go generate $(SRC_FILES)
 
 lint: fmt
 	@echo "=== linting ==="
 	go vet $(SRC_FILES)
 	glide novendor | xargs -n1 golint -set_exit_status
 
-test: lint
+test: lint gen
 	@echo "=== testing ==="
 ifneq ($(CIRCLE_TEST_REPORTS),)
 	mkdir -p $(CIRCLE_TEST_REPORTS)/unit
@@ -41,13 +44,18 @@ else
 endif
 
 
-build: $(BUILD_FILES)
+build: gen $(BUILD_FILES)
 
 $(BUILD_FILES):
 	@echo "=== building $(VERSION) - $@ ==="
 	mkdir -p $(BUILD_DIR)
 	GOOS=$(word 2,$(subst -, ,$(notdir $@))) GOARCH=$(word 3,$(subst -, ,$(notdir $@))) go build -ldflags=$(GOLDFLAGS) -o '$@'
 
+install: build
+	@echo "=== building $(VERSION) - $(PACKAGE)-$(OS)-$(ARCH) ==="
+	cp $(BUILD_DIR)/$(PACKAGE)-$(OS)-$(ARCH) /usr/local/bin/mu
+	chmod 755 /usr/local/bin/mu
+
 release-clean:
 ifeq ($(IS_MASTER),)
 	@echo "=== clearing old release $(VERSION) ==="
@@ -89,4 +97,4 @@ fmt:
 	go fmt $(SRC_FILES)
 
 
-.PHONY: default all lint test build deps clean release-clean release-create dev-release release $(UPLOAD_FILES) $(TARGET_OS)
+.PHONY: default all lint test build deps gen clean release-clean release-create dev-release release install $(UPLOAD_FILES) $(TARGET_OS)
diff --git a/README.md b/README.md
@@ -1,209 +1,47 @@
 [![Build Status](https://circleci.com/gh/stelligent/mu.svg?style=shield)](https://circleci.com/gh/stelligent/mu) [![Join the chat at https://gitter.im/stelligent/mu](https://badges.gitter.im/stelligent/mu.svg)](https://gitter.im/stelligent/mu?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Go Report Card](https://goreportcard.com/badge/github.com/stelligent/mu)](https://goreportcard.com/report/github.com/stelligent/mu)
 
 
-
 # Why?
 Amazon ECS (EC2 Container Service) provides an excellent platform for deploying microservices as containers.  The challenge however is that there is a significant learning curve for microservice developers to deploy their applications in an efficient manner.  Specifically, they must learn to use CloudFormation to orchestrate the management of ECS, ECR, EC2, ELB, VPC, and IAM resources.  Additionally, tools like CodeBuild and CodePipeline must be mastered to create a continuous delivery pipeline for their microservices.
 
 To address these challenges, this tool was created to simplify the declaration and administration of the AWS resources necessary to support microservices.  Similar to how the [Serverless Framework](https://serverless.com/) improved the developer experience of Lambda and API Gateway, this tool makes it easier for developers to use ECS as a microservices platform.
 
 The `mu` tool uses CloudFormation stacks to manage all resources it creates.  Additionally, `mu` will not create any databases or other AWS resources to support itself.  It will only create resources (via CloudFormation) necessary to run your microservices.  This means at any point you can stop using `mu` and continue to manage the AWS resources that it created via AWS tools such as the CLI or the console.
 
-![Architecture Diagram](docs/ms-architecture-3.png)
-
-# Installation
-
-```bash
-# Install latest version to /usr/local/bin
-curl -s https://raw.githubusercontent.com/stelligent/mu/master/install.sh | sh
-
-# Install v0.1.0 version to ~/bin
-curl -s https://raw.githubusercontent.com/stelligent/mu/master/install.sh | INSTALL_VERSION=0.1.0 INSTALL_DIR=~/bin sh
-```
-
-# Environments
-Environments are defined to become a target for deploying services to.  Each environment is a CloudFormation stack consisting of the following resources:
-
-* **VPC** – To provide the network infrastructure to launch the ECS container instances into. Optionally, you can target an existing VPC.
-* **ECS Cluster** – The cluster that the services will be deployed into.
-* **Auto Scaling Group** – To manage the ECS container instances that contain the compute resources for running the containers.  Auto scaling policies will be defined based on memory entitlements in the cluster.
-* **Application Load Balancer** – To provide load balancing for the microservices running in containers.
-
-![Environment Diagram](docs/ms-architecture-1.png)
-
-## Configuration
-```
----
-
-### Define a list of environments
-environments:
-
-  # The unique name of the environment  (required)
-  - name: dev
-
-
-    ### Attributes for the ECS container instances
-    cluster:
-      imageId: ami-xxxxxx           # The AMI to use for the ECS container instances (default: latest ECS optimized AMI)
-      instanceTenancy: default      # Whether to use default or dedicated tenancy (default: default)
-      desiredCapacity: 1            # Desired number of ECS container instances (default 1)
-      maxSize: 2                    # Max size to scale the ECS ASG to (default: 2)
-      keyName: my-keypair           # name of EC2 keypair to associate with ECS container instances (default: none)
-      sshAllow: 0.0.0.0/0           # CIDR block to allow SSH access from (default: 0.0.0.0/0)
-      httpProxy: 10.0.0.43:8080     # Host and port to use for HTTP proxy for yum, docker images, and ECS (default: none)
-      scaleOutThreshold: 80         # Threshold for % memory utilization to scale out ECS container instances (default: 80)
-      scaleInThreshold: 30          # Threshold for % memory utilization to scale in ECS container instances (default: 30)
-
-    ### Attributes for the ELB
-    loadBalancer:
-      internal: true                # Whether to create an internal ELB or not (default: false)
-      hostedzone: mydomain.com      # HostedZone in Route53 to create ELB DNS for.  Leave blank to not create DNS (default: none)
-      name: api                     # Name to register in hostedzone for ELB DNS.  (default: environment name)
-      certificate: arn:aws:acm:...  # The ARN of a certificate in ACM.  If defined, will create HTTPS listener in ELB.  (default: none)
-
-    ### Attributes for the VPC to target.  If not defined, a VPC will be created. (default: none)
-    vpcTarget:
-        vpcId: vpc-xxxxx            # The id of the VPC to launch ECS container instances into
-        ecsSubnetIds:               # The list of subnets to use for ECS container instances
-          - subnet-xxxxx
-          - subnet-xxxxy
-          - subnet-xxxxz
-        elbSubnetIds:               # The list of subnets to use for ELBs
-          - subnet-xxxxx
-          - subnet-xxxxy
-          - subnet-xxxxz
-```
-
-## Commands
-```
-# List all environments
-> mu env list
-
-# Show details about a specific environment (ECS container instances, Running services, etc)
-> mu env show <environment_name>
-
-# Upsert an environment
-> mu env up <environment_name>
-
-# Terminate an environment
-> mu env terminate <environment_name>
-```
-
-# Services
-Services are first pushed to an ECR repository and then deployed to a specific environment.  Each service is a CloudFormation stack consisting of the following resources:
-
-* **Task Definition** – An ECS task definition referencing the image and tag in the ECR repo.
-* **Service** - An ECS service referencing the Task Definition.
-* **Target Group** - An ALB target group for the Service to reference and register containers in.
-* **Listener Rule** - A rule in the ALB listener from the environment to route specific URLs to the target group.
-
-![Service Diagram](docs/ms-architecture-2.png)
+![Architecture Diagram](https://github.com/stelligent/mu/wiki/img/ms-architecture-3.png)
 
-## Configuration
-```
----
-
-### Define the service for this repo
-service:
-  name: my-service                   # The unique name of the service (default: the name of the directory that mu.yml was in)
-  desiredCount: 4                    # The desired number of tasks to run for the service (default: 2)
-  dockerfile: ./Dockerfile           # The relative path to the Dockerfile to build images (default: ./Dockerfile)
-  imageRepository: tutum/hello-world # The repository to push images to and deploy services from.  Leave unset to have mu manage an ECR repository (default: none)
-  port: 80                           # The port to expose from the container (default: 8080)
-  healthEndpoint: /health            # The endpoint inside the container to determine if the task is healthy (default: /health)
-  cpu: 20                            # The number of CPU units to allocate to each task (default: 10)
-  memory: 400                        # The amount of memory in MiB to allocate to each task (default: 300)
-
-  # The paths to match on in the ALB and route to this service.  Leave blank to not create an ALB target group for this service (default: none)
-  pathPatterns:
-    - /bananas
-    - /apples
-
-  # The priority for resolving the pathPatterns from the ALB (between 1 and 99999)
-  priority: 25
-
-```
-
-## Commands
-```
-# Show details about a specific service (Which versions in which environments, pipeline status)
-> mu service show [<service_name>]
-
-# Build docker image and push to ECR
-> mu service push
-
-# Deploy the service to an environment
-> mu service deploy <environment_name>
-
-# Undeploy the service from an environment
-> mu service undeploy <environment_name> [<service_name>]
-```
-
-# Pipelines
-A pipeline can be created for each service that consists of the following steps:
-
-* **Source** - Retrieve source from GitHub for a specific branch.  Triggered on each commit.
-* **Build Artifact** - Compile the source code via CodeBuild and a `buildspec.yml`.
-* **Build Image** - Build the Docker image and push to ECR repository.
-* **Acceptance** - Deploy to acceptance environment and run automated tests.
-* **Production** - Wait for manual approval, then deploy to production environment.
+# Demo
+Watch the 90 second demo below to see mu in action!
 
-<a href="docs/ms-pipeline-1.png"><img src="docs/ms-pipeline-1.png" width="110"></a>
+![Demo](https://github.com/stelligent/mu/wiki/quickstart/mu-quickstart.gif)
 
+# Get Started!
+Install latest version to /usr/local/bin (or for additional options, see [wiki](https://github.com/stelligent/mu/wiki/Installation)):
 
-## Configuration
-```
----
-service:
-
-  name: my-service
-  # ... service config goes here ...
-
-  # Define the behavior of the pipeline
-  pipeline:
-      source:
-        repo: stelligent/microservice-exemplar  # The GitHub repo slug to build (default: none)
-        branch: mu                              # The branch to build from (default: master)
-      build:
-        image: aws/codebuild/java:openjdk-8     # The image to use for CodeBuild job (default: aws/codebuild/ubuntu-base:latest)
-        type: linuxContainer
-        computeType: BUILD_GENERAL1_SMALL       # The type of compute instance for builds (default: BUILD_GENERAL1_SMALL)
-      acceptance:
-        environment: dev                        # The environment name to deploy to for testing (default: dev)
-      production:
-        environment: production                 # The environment name to deploy to for production (default: production)
-```
-
-## Commands
-```
-# List the pipelines
-> mu pipeline list
-
-# Upsert the pipeline
-> mu pipeline up [-t <repo_token>]
-
-# Terminate the pipeline
-> mu pipeline terminate [<service_name>]
-```
-
-# Common flags
+```bash
+curl -s https://raw.githubusercontent.com/stelligent/mu/master/install.sh | sh
 ```
-# Path to mu config
-> mu -c path/to/mu.yml ...
 
-# AWS region
-> mu -r us-west-2 ...
+Assuming your project already has a Dockerfile, you can initialize your mu.yml file with: `mu init`.  More details available in the [quickstart](https://github.com/stelligent/mu/wiki/Quickstart).
 
-# or via environment variable
-> AWS_REGION=us-west-2 mu ...
+# What's next?
+Check out the [examples](examples) to see common `mu.yml` configuration use cases:
 
-# AWS profile
-> mu -p my-profile ...
+* **[Basic](examples/basic)** - Simple website with continuous delivery pipeline deploying to dev and prod environments
+* **[Test Automation](examples/pipeline-newman)** - Example of automating end-to-end testing via [Newman](https://github.com/postmanlabs/newman)
+* **[Env Variables](examples/service-env-vars)** - Defining environment variables for the service
+* **[HTTPS](examples/elb-https)** - Enable HTTPS on the ALB for an environment
+* **[DNS](examples/elb-dns)** - Associate Route53 resource record with ALB for an environment
+* **[VPC Target](examples/vpc-target)** - Targeting an existing VPC for an environment
+* **[Custom CloudFormation](examples/custom-cloudformation)** - Demonstration of adding custom AWS resources via CloudFormation
 
-# or via environment variable
-> AWS_PROFILE=my-profie mu ...
+Refer to the wiki for complete details on the configuration of `mu.yml` and the cli usage:
 
-```
+* **[Environments](https://github.com/stelligent/mu/wiki/Environments)** - managing VPCs, ECS clusters, container instances and ALBs
+* **[Services](https://github.com/stelligent/mu/wiki/Services)** - managing ECS service configuration
+* **[Pipelines](https://github.com/stelligent/mu/wiki/Pipelines)** - managing continuous delivery pipelines
+* **[CLI](https://github.com/stelligent/mu/wiki/CLI-Usage)** - details about using the CLI
+* **[Custom CloudFormation](https://github.com/stelligent/mu/wiki/Custom-CloudFormation)** - details about customizing the CloudFormation that is generated by mu.
 
 # Contributing
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.1.7
+0.1.8
diff --git a/cli/app.go b/cli/app.go
@@ -17,6 +17,7 @@ func NewApp() *cli.App {
 	app.EnableBashCompletion = true
 
 	app.Commands = []cli.Command{
+		*newInitCommand(context),
 		*newEnvironmentsCommand(context),
 		*newServicesCommand(context),
 		*newPipelinesCommand(context),
@@ -34,7 +35,7 @@ func NewApp() *cli.App {
 		}
 
 		// initialize context
-		err := context.InitializeContext(c.String("profile"), c.String("region"))
+		err := context.InitializeContext(c.String("profile"), c.String("region"), c.Bool("dryrun"))
 		if err != nil {
 			return err
 		}
@@ -45,8 +46,12 @@ func NewApp() *cli.App {
 
 		err = context.InitializeConfigFromFile(c.String("config"))
 		if err != nil {
-			log.Warningf("Unable to load mu config: %v", err)
+			// ignore errors for init command
+			if c.Args().First() != "init" {
+				log.Warningf("Unable to load mu config: %v", err)
+			}
 		}
+
 		return nil
 
 	}
@@ -73,6 +78,10 @@ func NewApp() *cli.App {
 			Name:  "verbose, V",
 			Usage: "increase level of log verbosity",
 		},
+		cli.BoolFlag{
+			Name:  "dryrun, d",
+			Usage: "generate the cloudformation templates without upserting stacks",
+		},
 	}
 
 	return app

diff --git a/cli/app_test.go b/cli/app_test.go
@@ -14,14 +14,16 @@ func TestNewApp(t *testing.T) {
 	assert.Equal("0.0.0-local", app.Version, "Version should match")
 	assert.Equal("Microservice Platform on AWS", app.Usage, "usage should match")
 	assert.Equal(true, app.EnableBashCompletion, "bash completion should match")
-	assert.Equal(5, len(app.Flags), "Flags len should match")
+	assert.Equal(6, len(app.Flags), "Flags len should match")
 	assert.Equal("config, c", app.Flags[0].GetName(), "Flags name should match")
 	assert.Equal("region, r", app.Flags[1].GetName(), "Flags name should match")
 	assert.Equal("profile, p", app.Flags[2].GetName(), "Flags name should match")
 	assert.Equal("silent, s", app.Flags[3].GetName(), "Flags name should match")
 	assert.Equal("verbose, V", app.Flags[4].GetName(), "Flags name should match")
-	assert.Equal(3, len(app.Commands), "Commands len should match")
-	assert.Equal("environment", app.Commands[0].Name, "Command[0].name should match")
-	assert.Equal("service", app.Commands[1].Name, "Command[1].name should match")
-	assert.Equal("pipeline", app.Commands[2].Name, "Command[2].name should match")
+	assert.Equal("dryrun, d", app.Flags[5].GetName(), "Flags name should match")
+	assert.Equal(4, len(app.Commands), "Commands len should match")
+	assert.Equal("init", app.Commands[0].Name, "Command[0].name should match")
+	assert.Equal("environment", app.Commands[1].Name, "Command[1].name should match")
+	assert.Equal("service", app.Commands[2].Name, "Command[2].name should match")
+	assert.Equal("pipeline", app.Commands[3].Name, "Command[3].name should match")
 }
diff --git a/cli/environments.go b/cli/environments.go
@@ -62,16 +62,23 @@ func newEnvironmentsListCommand(ctx *common.Context) *cli.Command {
 
 func newEnvironmentsShowCommand(ctx *common.Context) *cli.Command {
 	cmd := &cli.Command{
-		Name:      "show",
-		Usage:     "show environment details",
+		Name:  "show",
+		Usage: "show environment details",
+		Flags: []cli.Flag{
+			cli.StringFlag{
+				Name:  "format, f",
+				Usage: "output format, either 'json' or 'cli' (default: cli)",
+				Value: "cli",
+			},
+		},
 		ArgsUsage: "<environment>",
 		Action: func(c *cli.Context) error {
 			environmentName := c.Args().First()
 			if len(environmentName) == 0 {
 				cli.ShowCommandHelp(c, "show")
 				return errors.New("environment must be provided")
 			}
-			workflow := workflows.NewEnvironmentViewer(ctx, environmentName, os.Stdout)
+			workflow := workflows.NewEnvironmentViewer(ctx, c.String("format"), environmentName, os.Stdout)
 			return workflow()
 		},
 	}

diff --git a/cli/environments_test.go b/cli/environments_test.go
@@ -73,6 +73,8 @@ func TestNewEnvironmentsShowCommand(t *testing.T) {
 	assert.NotNil(command)
 	assert.Equal("show", command.Name, "Name should match")
 	assert.Equal("<environment>", command.ArgsUsage, "ArgsUsage should match")
+	assert.Equal(1, len(command.Flags), "Flag len should match")
+	assert.Equal("format, f", command.Flags[0].GetName(), "Flag should match")
 	assert.NotNil(command.Action)
 }
 func TestNewEnvironmentsTerminateCommand(t *testing.T) {