Move kuberenetes stuff out into separate runs, fix backend, fix some …

…issues with generated code

internal/config/config.go

@@ -87,6 +87,7 @@ type terraform struct {
 
 type eks struct {
 	ClusterName string `yaml:"clusterName"`
+	WorkerAMI   string `yaml:"workerAMI"`
 	Deploy      bool
 }
 

internal/generate/golang/generate.go

@@ -24,6 +24,7 @@ func Generate(t *templator.Templator, cfg *config.Commit0Config, service config.
 
 	util.TemplateFileIfDoesNotExist(basePath, "main.go", t.Go.GoMain, wg, data)
 	util.TemplateFileIfDoesNotExist(basePath, "go.mod", t.Go.GoMod, wg, data)
+	util.TemplateFileIfDoesNotExist(basePath, "server.go", t.Go.GoServer, wg, data)
 	util.TemplateFileIfDoesNotExist(healthPath, "health.go", t.Go.GoHealthServer, wg, data)
 
 	file := fmt.Sprintf("%s.go", service.Name)

internal/generate/http/generate.go

@@ -10,5 +10,5 @@ import (
 
 func GenerateGoHTTPGW(templator *templator.Templator, data templator.GolangTemplateData, basePath string, wg *sync.WaitGroup) {
 	path := filepath.Join(basePath, "http")
-	util.TemplateFileAndOverwrite(path, "main.go", templator.Go.GoHTTPGW, wg, data)
+	util.TemplateFileIfDoesNotExist(path, "main.go", templator.Go.GoHTTPGW, wg, data)
 }

internal/generate/kubernetes/generate.go

@@ -16,6 +16,8 @@ import (
 	"github.com/commitdev/commit0/internal/config"
 	"github.com/commitdev/commit0/internal/templator"
 	"github.com/commitdev/commit0/internal/util"
+	"github.com/kyokomi/emoji"
+	"github.com/logrusorgru/aurora"
 	"github.com/manifoldco/promptui"
 	"gopkg.in/yaml.v2"
 )
@@ -29,9 +31,26 @@ type Secrets struct {
 	}
 }
 
+// @TODO : These are specific to a k8s version. If we make the version a config option we will need to change this
+var amiLookup = map[string]string{
+	"us-east-1":    "ami-0392bafc801b7520f",
+	"us-east-2":    "ami-082bb518441d3954c",
+	"us-west-2":    "ami-05d586e6f773f6abf",
+	"eu-west-1":    "ami-059c6874350e63ca9",
+	"eu-central-1": "ami-0e21bc066a9dbabfa",
+}
+
 // Generate templates
 func Generate(t *templator.Templator, cfg *config.Commit0Config, wg *sync.WaitGroup, pathPrefix string) {
-	data := templator.GenericTemplateData{*cfg}
+	if cfg.Infrastructure.AWS.EKS.WorkerAMI == "" {
+		ami, found := amiLookup[cfg.Infrastructure.AWS.Region]
+		if !found {
+			log.Fatalln(aurora.Red(emoji.Sprintf(":exclamation: Unable to look up an AMI for the chosen region")))
+		}
+
+		cfg.Infrastructure.AWS.EKS.WorkerAMI = ami
+	}
+	data := templator.GenericTemplateData{Config: *cfg}
 	t.Kubernetes.TemplateFiles(data, false, wg, pathPrefix)
 }
 
@@ -50,17 +69,31 @@ func Execute(config *config.Commit0Config, pathPrefix string) {
 		}
 
 		envars := getAwsEnvars(readSecrets())
-		log.Println("Planning infrastructure...")
-		execute(exec.Command("terraform", "init"), pathPrefix, envars)
-		execute(exec.Command("terraform", "plan"), pathPrefix, envars)
+
+		pathPrefix = filepath.Join(pathPrefix, "kubernetes/terraform")
+
+		// @TODO : A check here would be nice to see if this stuff exists first, mostly for testing
+		log.Println(aurora.Cyan(emoji.Sprintf(":alarm_clock: Initializing remote backend...")))
+		execute(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "bootstrap/remote-state"), envars)
+		execute(exec.Command("terraform", "apply", "-auto-approve"), filepath.Join(pathPrefix, "bootstrap/remote-state"), envars)
+
+		log.Println(aurora.Cyan(":alarm_clock: Planning infrastructure..."))
+		execute(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging"), envars)
+		execute(exec.Command("terraform", "plan"), filepath.Join(pathPrefix, "environments/staging"), envars)
+
+		log.Println(aurora.Cyan(":alarm_clock: Applying infrastructure configuration..."))
+		execute(exec.Command("terraform", "apply"), filepath.Join(pathPrefix, "environments/staging"), envars)
+
+		log.Println(aurora.Cyan(":alarm_clock: Applying kubernetes configuration..."))
+		execute(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
+		execute(exec.Command("terraform", "plan"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
 	}
 }
 
 func execute(cmd *exec.Cmd, pathPrefix string, envars []string) {
 	dir := util.GetCwd()
 
-	kubDir := path.Join(pathPrefix, "kubernetes/terraform/environments/staging")
-	cmd.Dir = path.Join(dir, kubDir)
+	cmd.Dir = path.Join(dir, pathPrefix)
 
 	stdoutPipe, _ := cmd.StdoutPipe()
 	stderrPipe, _ := cmd.StderrPipe()

internal/util/util.go

@@ -21,9 +21,14 @@ func CreateDirIfDoesNotExist(path string) error {
 	return nil
 }
 
+func CleanGoIdentifier(identifier string) string {
+	return strings.ReplaceAll(identifier, "-", "")
+}
+
 var FuncMap = template.FuncMap{
-	"Title":   strings.Title,
-	"ToLower": strings.ToLower,
+	"Title":             strings.Title,
+	"ToLower":           strings.ToLower,
+	"CleanGoIdentifier": CleanGoIdentifier,
 }
 
 func GetCwd() string {

templates/golang/main.tmpl

@@ -22,8 +22,8 @@ func main() {
 	//Server initialization & registration
 	healthServer := health.NewHealthServer()
 	healthpb.RegisterHealthServer(s, healthServer)
-	{{ .Config.Name }}Server := {{ .Config.Name }}.New{{ .Config.Name | Title}}Server()
-	{{ .Config.Name }}pb.Register{{ .Config.Name | Title}}Server(s, {{ .Config.Name }}Server)
+	{{ .Config.Name | CleanGoIdentifier }}Server := {{ .Config.Name | CleanGoIdentifier }}.New{{ .Config.Name | Title | CleanGoIdentifier}}Server()
+	{{ .Config.Name | CleanGoIdentifier }}pb.Register{{ .Config.Name | Title | CleanGoIdentifier}}Server(s, {{ .Config.Name | CleanGoIdentifier }}Server)
 
 
 	log.Printf("Starting grpc server on %v...", grpcAddr)

templates/golang/server.tmpl

@@ -10,11 +10,11 @@ type {{ .Service.Name | Title }}Server struct {
 
 }
 
-func New{{ .Service.Name | Title }}Server() *{{ .Service.Name | Title }}Server {
-	return &{{ .Service.Name | Title }}Server{}
+func New{{ .Service.Name | Title | CleanGoIdentifier }}Server() *{{ .Service.Name | Title }}Server {
+	return &{{ .Service.Name | Title | CleanGoIdentifier }}Server{}
 }
 
-func (s *{{ .Service.Name | Title }}Server) Check(ctx context.Context, req *health_api.HealthCheckRequest) (*health_api.HealthCheckResponse, error) {
+func (s *{{ .Service.Name | Title | CleanGoIdentifier }}Server) Check(ctx context.Context, req *health_api.HealthCheckRequest) (*health_api.HealthCheckResponse, error) {
 	resp := &health_api.HealthCheckResponse{
 		Status: health_api.HealthCheckResponse_SERVING,
 	}

templates/kubernetes/terraform/global/remote-state/main.tf → templates/kubernetes/terraform/bootstrap/remote-state/main.tf

@@ -11,6 +11,16 @@ resource "aws_s3_bucket" "terraform_remote_state" {
   }
 }
 
+resource "aws_s3_bucket_public_access_block" "terraform_remote_state" {
+  bucket = "${aws_s3_bucket.terraform_remote_state.id}"
+
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
 resource "aws_dynamodb_table" "terraform_state_locks" {
   name           = "{{ .Config.Name }}-terraform-state-locks"
   read_capacity  = 2

templates/kubernetes/terraform/environments/development/kubernetes/main.tf

@@ -0,0 +1,32 @@
+terraform {
+  backend "s3" {
+    bucket         = "project-{{ .Config.Name }}-terraform-state"
+    key            = "infrastructure/terraform/environments/development/main"
+    encrypt        = true
+    region         = "{{ .Config.Infrastructure.AWS.Region }}"
+    dynamodb_table = "{{ .Config.Name }}-terraform-state-locks"
+  }
+}
+
+# Instantiate the development environment
+module "development" {
+  source      = "../../../modules/environment"
+  environment = "development"
+
+  # Project configuration
+  project             = "{{ .Config.Infrastructure.AWS.EKS.ClusterName }}"
+  region              = "{{ .Config.Infrastructure.AWS.Region }}"
+  allowed_account_ids = ["{{ .Config.Infrastructure.AWS.AccountId }}"]
+
+  # ECR configuration
+  ecr_repositories = ["{{ .Config.Infrastructure.AWS.EKS.ClusterName }}"]
+
+  # EKS configuration
+  eks_worker_instance_type = "t2.small"
+  eks_worker_asg_max_size  = 2
+
+  # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html
+  # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.14%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1
+  eks_worker_ami = "{{ .Config.Infrastructure.AWS.EKS.WorkerAMI }}"
+
+}

templates/kubernetes/terraform/environments/development/main.tf

@@ -4,29 +4,34 @@ terraform {
     key            = "infrastructure/terraform/environments/development/main"
     encrypt        = true
     region         = "{{ .Config.Infrastructure.AWS.Region }}"
-    dynamodb_table = "terraform-state-locks"
+    dynamodb_table = "{{ .Config.Name }}-terraform-state-locks"
   }
 }
 
-# Instantiate the development environment
-module "development" {
-  source      = "../../modules/environment"
+# Provision kubernetes resources required to run services/applications
+module "kubernetes" {
+  source = "../../modules/kubernetes"
+
   environment = "development"
+  region      = "{{ .Config.Infrastructure.AWS.Region }}"
 
-  # Project configuration
-  project             = "{{ .Config.Infrastructure.AWS.EKS.ClusterName }}"
-  region              = "{{ .Config.Infrastructure.AWS.Region }}"
-  allowed_account_ids = ["{{ .Config.Infrastructure.AWS.AccountId }}"]
+  # Authenticate with the EKS cluster via the cluster id
+  cluster_name = "{{ .Config.Infrastructure.AWS.EKS.ClusterName }}"
 
-  # ECR configuration
-  ecr_repositories = ["{{ .Config.Infrastructure.AWS.EKS.ClusterName }}"]
+  # Assume-role policy used by monitoring fluentd daemonset
+  assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json
+}
 
-  # EKS configuration
-  eks_worker_instance_type = "t2.small"
-  eks_worker_asg_max_size  = 2
+# Data sources for EKS IAM
+data "aws_caller_identity" "current" {}
 
-  # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html
-  # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.14%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1
-  eks_worker_ami = "ami-0392bafc801b7520f"
+data "aws_iam_policy_document" "assumerole_root_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
 
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+  }
 }

templates/kubernetes/terraform/environments/production/kubernetes/main.tf

@@ -0,0 +1,37 @@
+terraform {
+  backend "s3" {
+    bucket         = "project-{{ .Config.Name }}-terraform-state"
+    key            = "infrastructure/terraform/environments/production/main"
+    encrypt        = true
+    region         = "{{ .Config.Infrastructure.AWS.Region }}"
+    dynamodb_table = "{{ .Config.Name }}-terraform-state-locks"
+  }
+}
+
+# Provision kubernetes resources required to run services/applications
+module "kubernetes" {
+  source = "../../../modules/kubernetes"
+
+  environment = "production"
+  region      = "{{ .Config.Infrastructure.AWS.Region }}"
+
+  # Authenticate with the EKS cluster via the cluster id
+  cluster_name = "{{ .Config.Infrastructure.AWS.EKS.ClusterName }}"
+
+  # Assume-role policy used by monitoring fluentd daemonset
+  assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json
+}
+
+# Data sources for EKS IAM
+data "aws_caller_identity" "current" {}
+
+data "aws_iam_policy_document" "assumerole_root_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+  }
+}

templates/kubernetes/terraform/environments/production/main.tf

@@ -4,7 +4,7 @@ terraform {
     key            = "infrastructure/terraform/environments/production/main"
     encrypt        = true
     region         = "{{ .Config.Infrastructure.AWS.Region }}"
-    dynamodb_table = "terraform-state-locks"
+    dynamodb_table = "{{ .Config.Name }}-terraform-state-locks"
   }
 }
 
@@ -27,5 +27,5 @@ module "production" {
 
   # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html
   # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.14%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1
-  eks_worker_ami = "ami-0392bafc801b7520f"
+  eks_worker_ami = "{{ .Config.Infrastructure.AWS.EKS.WorkerAMI }}"
 }

templates/kubernetes/terraform/environments/staging/kubernetes/main.tf

@@ -0,0 +1,37 @@
+terraform {
+  backend "s3" {
+    bucket         = "project-{{ .Config.Name }}-terraform-state"
+    key            = "infrastructure/terraform/environments/staging/kubernetes"
+    encrypt        = true
+    region         = "{{ .Config.Infrastructure.AWS.Region }}"
+    dynamodb_table = "{{ .Config.Name }}-terraform-state-locks"
+  }
+}
+
+# Provision kubernetes resources required to run services/applications
+module "kubernetes" {
+  source = "../../../modules/kubernetes"
+
+  environment = "staging"
+  region      = "{{ .Config.Infrastructure.AWS.Region }}"
+
+  # Authenticate with the EKS cluster via the cluster id
+  cluster_name = "{{ .Config.Infrastructure.AWS.EKS.ClusterName }}"
+
+  # Assume-role policy used by monitoring fluentd daemonset
+  assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json
+}
+
+# Data sources for EKS IAM
+data "aws_caller_identity" "current" {}
+
+data "aws_iam_policy_document" "assumerole_root_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+  }
+}

templates/kubernetes/terraform/environments/staging/main.tf

@@ -4,7 +4,7 @@ terraform {
     key            = "infrastructure/terraform/environments/staging/main"
     encrypt        = true
     region         = "{{ .Config.Infrastructure.AWS.Region }}"
-    dynamodb_table = "terraform-state-locks"
+    dynamodb_table = "{{ .Config.Name }}-terraform-state-locks"
   }
 }
 
@@ -24,8 +24,9 @@ module "staging" {
   # EKS configuration
   eks_worker_instance_type = "t2.small"
   eks_worker_asg_max_size  = 2
+
   # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html
   # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.14%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1
-  eks_worker_ami = "ami-0392bafc801b7520f"
+  eks_worker_ami = "{{ .Config.Infrastructure.AWS.EKS.WorkerAMI }}"
 
 }

templates/kubernetes/terraform/modules/environment/main.tf

@@ -42,17 +42,3 @@ module "kube2iam" {
   eks_worker_iam_role_name = module.eks.worker_iam_role_name
   iam_account_id           = data.aws_caller_identity.current.account_id
 }
-
-# Provision kubernetes resources required to run services/applications
-module "kubernetes" {
-  source = "../../modules/kubernetes"
-
-  environment = var.environment
-  region      = var.region
-
-  # Authenticate with the EKS cluster via the cluster id
-  cluster_name = module.eks.cluster_id
-
-  # Assume-role policy used by monitoring fluentd daemonset
-  assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json
-}

templates/kubernetes/terraform/modules/vpc/main.tf

@@ -5,9 +5,9 @@ module "vpc" {
   cidr = "10.20.0.0/16"
 
   azs              = ["${var.region}a", "${var.region}b", "${var.region}c"] # Most regions have 3+ azs
-  private_subnets  = ["10.20.40.0/24", "10.20.42.0/24", "10.20.44.0/24"]
+  private_subnets  = ["10.20.10.0/22", "10.20.14.0/22", "10.20.18.0/22"]
   public_subnets   = ["10.20.41.0/24", "10.20.43.0/24", "10.20.45.0/24"]
-  database_subnets = ["10.20.50.0/24", "10.20.52.0/24", "10.20.54.0/24"]
+  database_subnets = ["10.20.60.0/24", "10.20.62.0/24", "10.20.64.0/24"]
 
   # Allow kubernetes ALB ingress controller to auto-detect
   private_subnet_tags = {