From 245aed5df26d34ffdb3e69a7133a2f56652c3829 Mon Sep 17 00:00:00 2001 From: Shah Newaz Khan Date: Thu, 17 Oct 2019 13:32:31 -0700 Subject: [PATCH 1/8] Added kubernetes specs --- cmd/generate.go | 8 +- go.mod | 5 +- go.sum | 10 +- internal/config/config.go | 8 +- internal/generate/kubernetes/generate.go | 13 ++ templates/kubernetes/teraform/README.md | 29 +++ templates/kubernetes/teraform/eks-cluster.tf | 87 ++++++++ .../kubernetes/teraform/eks-worker-ndoes.tf | 145 +++++++++++++ templates/kubernetes/teraform/outputs.tf | 55 +++++ templates/kubernetes/teraform/providers.tf | 19 ++ templates/kubernetes/teraform/variables.tf | 4 + templates/kubernetes/teraform/vpc.tf | 81 ++++++++ templates/kubernetes/teraform/workstation.tf | 18 ++ templator/templator.go | 194 ++++++++++++++++++ 14 files changed, 667 insertions(+), 9 deletions(-) create mode 100644 internal/generate/kubernetes/generate.go create mode 100644 templates/kubernetes/teraform/README.md create mode 100644 templates/kubernetes/teraform/eks-cluster.tf create mode 100644 templates/kubernetes/teraform/eks-worker-ndoes.tf create mode 100644 templates/kubernetes/teraform/outputs.tf create mode 100644 templates/kubernetes/teraform/providers.tf create mode 100644 templates/kubernetes/teraform/variables.tf create mode 100644 templates/kubernetes/teraform/vpc.tf create mode 100644 templates/kubernetes/teraform/workstation.tf create mode 100644 templator/templator.go diff --git a/cmd/generate.go b/cmd/generate.go index 3ae64c052..8da546d2b 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -1,15 +1,16 @@ package cmd import ( + "log" "sync" - "github.com/commitdev/commit0/internal/config" "github.com/commitdev/commit0/internal/generate/docker" "github.com/commitdev/commit0/internal/generate/golang" "github.com/commitdev/commit0/internal/generate/http" "github.com/commitdev/commit0/internal/generate/proto" "github.com/commitdev/commit0/internal/generate/react" + "github.com/commitdev/commit0/internal/generate/kubernetes" "github.com/commitdev/commit0/internal/templator" "github.com/commitdev/commit0/internal/util" "github.com/gobuffalo/packr/v2" @@ -22,9 +23,10 @@ var language string const ( Go = "go" React = "react" + Kubernetes = "kubernetes" ) -var supportedLanguages = [...]string{Go, React} +var supportedLanguages = [...]string{Go, React, Kubernetes} func init() { @@ -59,6 +61,8 @@ var generateCmd = &cobra.Command{ docker.GenerateGoDockerCompose(t, cfg, &wg) case React: react.Generate(t, cfg, &wg) + case Kubernetes: + kubernetes.Generate(Templator, cfg) } util.TemplateFileIfDoesNotExist("", "README.md", t.Readme, &wg, cfg) diff --git a/go.mod b/go.mod index 56434f967..03a2f63a6 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,12 @@ require ( github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect github.com/k0kubun/pp v3.0.1+incompatible github.com/mattn/go-colorable v0.1.2 // indirect - github.com/rogpeppe/go-internal v1.3.2 // indirect + github.com/rogpeppe/go-internal v1.5.0 // indirect github.com/spf13/cobra v0.0.5 github.com/stretchr/testify v1.4.0 // indirect - golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 // indirect + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect + golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.2.4 ) diff --git a/go.sum b/go.sum index b39800da0..6b23815f4 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.2 h1:XU784Pr0wdahMY2bYcyK6N1KuaRAdLtqD4qd8D18Bfs= -github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.5.0 h1:Usqs0/lDK/NqTkvrmKSwA/3XkZAs7ZAW/eLeQ2MVBTw= +github.com/rogpeppe/go-internal v1.5.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -74,6 +74,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -86,8 +88,8 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0 h1:Dh6fw+p6FyRl5x/FvNswO1ji0lIGzm3KP8Y9VkS9PTE= diff --git a/internal/config/config.go b/internal/config/config.go index cf853eeb7..0197d21c6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -55,7 +55,13 @@ type Commit0Config struct { Maintainers []Maintainers `yaml:"maintainers"` Network Network `yaml:"network"` Services []Service `yaml:"services"` - React React `yaml:react` + React React `yaml:"react"` + Kubernetes Kubernetes `yaml:"kubernetes"` +} + +type Kubernetes struct { + ClusterName string + DNSName string } func LoadConfig(filePath string) *Commit0Config { diff --git a/internal/generate/kubernetes/generate.go b/internal/generate/kubernetes/generate.go new file mode 100644 index 000000000..937d688c1 --- /dev/null +++ b/internal/generate/kubernetes/generate.go @@ -0,0 +1,13 @@ +package kubernetes + +import ( + //"github.com/commitdev/commit0/util" + + "github.com/commitdev/commit0/config" + "github.com/commitdev/commit0/templator" +) + +func Generate(templator *templator.Templator, config *config.Commit0Config) { + templator.Kubernetes.TemplateFiles(config, false) +} + diff --git a/templates/kubernetes/teraform/README.md b/templates/kubernetes/teraform/README.md new file mode 100644 index 000000000..f779a3278 --- /dev/null +++ b/templates/kubernetes/teraform/README.md @@ -0,0 +1,29 @@ +# EKS Terraform + +AWS Resources created: + +- EKS Cluster: AWS managed Kubernetes cluster of master servers +- AutoScaling Group containing 2 m4.large instances based on the latest EKS Amazon Linux 2 AMI: Operator managed Kubernetes worker nodes for running Kubernetes service deployments +- Associated VPC, Internet Gateway, Security Groups, and Subnets: Operator managed networking resources for the EKS Cluster and worker node instances +- Associated IAM Roles and Policies: Operator managed access resources for EKS and worker node instances + +## Pre-requisites + +- Setup the [AWS credentials](https://www.terraform.io/docs/providers/aws/index.html#environment-variables) for terraform + +## Spin up cluster + +```shell + +terraform plan +terraform apply + +``` + +### Connect to cluster +The EKS service does not provide a cluster-level API parameter or resource to automatically configure the underlying Kubernetes cluster to allow worker nodes to join the cluster via AWS IAM role authentication. + +- Run `aws eks update-kubeconfig --name staging` to configure `kubectl` +- Run `terraform output config_map_aws_auth` and save the configuration into a file, e.g. config_map_aws_auth.yaml +- Run `kubectl apply -f config_map_aws_auth.yaml` +- You can verify the worker nodes are joining the cluster via: `kubectl get nodes --watch` diff --git a/templates/kubernetes/teraform/eks-cluster.tf b/templates/kubernetes/teraform/eks-cluster.tf new file mode 100644 index 000000000..7bae07812 --- /dev/null +++ b/templates/kubernetes/teraform/eks-cluster.tf @@ -0,0 +1,87 @@ +# +# EKS Cluster Resources +# * IAM Role to allow EKS service to manage other AWS services +# * EC2 Security Group to allow networking traffic with EKS cluster +# * EKS Cluster +# + +resource "aws_iam_role" "demo-cluster" { + name = "terraform-eks-demo-cluster" + + assume_role_policy = < Date: Thu, 17 Oct 2019 21:53:39 -0700 Subject: [PATCH 2/8] Removed workstation ip whitelist for eks vpc --- templates/kubernetes/teraform/eks-cluster.tf | 10 ---------- templates/kubernetes/teraform/providers.tf | 6 ------ templates/kubernetes/teraform/workstation.tf | 18 ------------------ 3 files changed, 34 deletions(-) delete mode 100644 templates/kubernetes/teraform/workstation.tf diff --git a/templates/kubernetes/teraform/eks-cluster.tf b/templates/kubernetes/teraform/eks-cluster.tf index 7bae07812..8b18ef6ed 100644 --- a/templates/kubernetes/teraform/eks-cluster.tf +++ b/templates/kubernetes/teraform/eks-cluster.tf @@ -61,16 +61,6 @@ resource "aws_security_group_rule" "demo-cluster-ingress-node-https" { type = "ingress" } -resource "aws_security_group_rule" "demo-cluster-ingress-workstation-https" { - cidr_blocks = ["${local.workstation-external-cidr}"] - description = "Allow workstation to communicate with the cluster API Server" - from_port = 443 - protocol = "tcp" - security_group_id = "${aws_security_group.demo-cluster.id}" - to_port = 443 - type = "ingress" -} - resource "aws_eks_cluster" "demo" { name = "${var.cluster-name}" role_arn = "${aws_iam_role.demo-cluster.arn}" diff --git a/templates/kubernetes/teraform/providers.tf b/templates/kubernetes/teraform/providers.tf index 8c50c669d..c99e88e34 100644 --- a/templates/kubernetes/teraform/providers.tf +++ b/templates/kubernetes/teraform/providers.tf @@ -11,9 +11,3 @@ provider "aws" { data "aws_region" "current" {} data "aws_availability_zones" "available" {} - -# Not required: currently used in conjuction with using -# icanhazip.com to determine local workstation external IP -# to open EC2 Security Group access to the Kubernetes cluster. -# See workstation-external-ip.tf for additional information. -provider "http" {} diff --git a/templates/kubernetes/teraform/workstation.tf b/templates/kubernetes/teraform/workstation.tf deleted file mode 100644 index adb757b8b..000000000 --- a/templates/kubernetes/teraform/workstation.tf +++ /dev/null @@ -1,18 +0,0 @@ -# -# Workstation External IP -# -# This configuration is not required and is -# only provided as an example to easily fetch -# the external IP of your local workstation to -# configure inbound EC2 Security Group access -# to the Kubernetes cluster. -# - -data "http" "workstation-external-ip" { - url = "http://ipv4.icanhazip.com" -} - -# Override with variable or hardcoded value if necessary -locals { - workstation-external-cidr = "${chomp(data.http.workstation-external-ip.body)}/32" -} From 489516b8e9be457fdc23a2708638146ee7bed3f9 Mon Sep 17 00:00:00 2001 From: Shah Newaz Khan Date: Thu, 24 Oct 2019 00:07:26 -0700 Subject: [PATCH 3/8] Added deploy flag functionality to kubernetes --- internal/config/config.go | 1 + internal/generate/kubernetes/generate.go | 98 ++++++++++++++++++- .../{teraform => terraform}/README.md | 0 .../{teraform => terraform}/eks-cluster.tf | 0 .../eks-worker-ndoes.tf | 0 .../{teraform => terraform}/outputs.tf | 0 .../{teraform => terraform}/providers.tf | 0 .../{teraform => terraform}/variables.tf | 0 .../kubernetes/{teraform => terraform}/vpc.tf | 0 9 files changed, 97 insertions(+), 2 deletions(-) rename templates/kubernetes/{teraform => terraform}/README.md (100%) rename templates/kubernetes/{teraform => terraform}/eks-cluster.tf (100%) rename templates/kubernetes/{teraform => terraform}/eks-worker-ndoes.tf (100%) rename templates/kubernetes/{teraform => terraform}/outputs.tf (100%) rename templates/kubernetes/{teraform => terraform}/providers.tf (100%) rename templates/kubernetes/{teraform => terraform}/variables.tf (100%) rename templates/kubernetes/{teraform => terraform}/vpc.tf (100%) diff --git a/internal/config/config.go b/internal/config/config.go index 0197d21c6..dd29569fd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -62,6 +62,7 @@ type Commit0Config struct { type Kubernetes struct { ClusterName string DNSName string + Deploy bool } func LoadConfig(filePath string) *Commit0Config { diff --git a/internal/generate/kubernetes/generate.go b/internal/generate/kubernetes/generate.go index 937d688c1..ca45c937b 100644 --- a/internal/generate/kubernetes/generate.go +++ b/internal/generate/kubernetes/generate.go @@ -1,13 +1,107 @@ package kubernetes import ( - //"github.com/commitdev/commit0/util" - + "bytes" + "fmt" + "io" + "sync" + "log" + "os" + "os/exec" + "path/filepath" "github.com/commitdev/commit0/config" "github.com/commitdev/commit0/templator" ) func Generate(templator *templator.Templator, config *config.Commit0Config) { templator.Kubernetes.TemplateFiles(config, false) + + + if config.Kubernetes.Deploy { + _tf_init := tf_init() + _tf_plan := tf_plan() + execute(_tf_init) + execute(_tf_plan) + } + +} + +// Terraform init cmd +func tf_init() *exec.Cmd { + + return exec.Command("terraform","init") +} + +// Terraform plan cmd +func tf_plan() *exec.Cmd { + + return exec.Command("terraform","plan") } +// Executes cmd passed in +func execute( cmd *exec.Cmd){ + dir, err1 := filepath.Abs(filepath.Dir(os.Args[0])) + if err1 != nil { + log.Fatal(err1) + } + + cmd.Dir = dir + "/kubernetes/terraform" + + + var errStdout, errStderr error + stdoutIn, _ := cmd.StdoutPipe() + stderrIn, _ := cmd.StderrPipe() + stdout := NewCapturingPassThroughWriter(os.Stdout) + stderr := NewCapturingPassThroughWriter(os.Stderr) + err := cmd.Start() + if err != nil { + log.Fatalf("cmd.Start() failed with '%s'\n", err) + } + + var wg sync.WaitGroup + wg.Add(1) + + go func() { + _, errStdout = io.Copy(stdout, stdoutIn) + wg.Done() + }() + + _, errStderr = io.Copy(stderr, stderrIn) + wg.Wait() + + err = cmd.Wait() + if err != nil { + log.Fatalf("cmd.Run() failed with %s\n", err) + } + if errStdout != nil || errStderr != nil { + log.Fatal("failed to capture stdout or stderr\n") + } + outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes()) + fmt.Printf("\nout:\n%s\nerr:\n%s\n", outStr, errStr) + +} + +// CapturingPassThroughWriter is a writer that remembers +// data written to it and passes it to w +type CapturingPassThroughWriter struct { + buf bytes.Buffer + w io.Writer +} + +// NewCapturingPassThroughWriter creates new CapturingPassThroughWriter +func NewCapturingPassThroughWriter(w io.Writer) *CapturingPassThroughWriter { + return &CapturingPassThroughWriter{ + w: w, + } +} + +// Write writes data to the writer, returns number of bytes written and an error +func (w *CapturingPassThroughWriter) Write(d []byte) (int, error) { + w.buf.Write(d) + return w.w.Write(d) +} + +// Bytes returns bytes written to the writer +func (w *CapturingPassThroughWriter) Bytes() []byte { + return w.buf.Bytes() +} diff --git a/templates/kubernetes/teraform/README.md b/templates/kubernetes/terraform/README.md similarity index 100% rename from templates/kubernetes/teraform/README.md rename to templates/kubernetes/terraform/README.md diff --git a/templates/kubernetes/teraform/eks-cluster.tf b/templates/kubernetes/terraform/eks-cluster.tf similarity index 100% rename from templates/kubernetes/teraform/eks-cluster.tf rename to templates/kubernetes/terraform/eks-cluster.tf diff --git a/templates/kubernetes/teraform/eks-worker-ndoes.tf b/templates/kubernetes/terraform/eks-worker-ndoes.tf similarity index 100% rename from templates/kubernetes/teraform/eks-worker-ndoes.tf rename to templates/kubernetes/terraform/eks-worker-ndoes.tf diff --git a/templates/kubernetes/teraform/outputs.tf b/templates/kubernetes/terraform/outputs.tf similarity index 100% rename from templates/kubernetes/teraform/outputs.tf rename to templates/kubernetes/terraform/outputs.tf diff --git a/templates/kubernetes/teraform/providers.tf b/templates/kubernetes/terraform/providers.tf similarity index 100% rename from templates/kubernetes/teraform/providers.tf rename to templates/kubernetes/terraform/providers.tf diff --git a/templates/kubernetes/teraform/variables.tf b/templates/kubernetes/terraform/variables.tf similarity index 100% rename from templates/kubernetes/teraform/variables.tf rename to templates/kubernetes/terraform/variables.tf diff --git a/templates/kubernetes/teraform/vpc.tf b/templates/kubernetes/terraform/vpc.tf similarity index 100% rename from templates/kubernetes/teraform/vpc.tf rename to templates/kubernetes/terraform/vpc.tf From 8c27716c9cb7249f575cb36d873a84484870e92e Mon Sep 17 00:00:00 2001 From: Shah Newaz Khan Date: Thu, 24 Oct 2019 00:08:26 -0700 Subject: [PATCH 4/8] Go formated --- cmd/generate.go | 4 ++-- internal/config/config.go | 6 ++--- internal/generate/kubernetes/generate.go | 30 +++++++++++------------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index 8da546d2b..5452d518c 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -21,8 +21,8 @@ var configPath string var language string const ( - Go = "go" - React = "react" + Go = "go" + React = "react" Kubernetes = "kubernetes" ) diff --git a/internal/config/config.go b/internal/config/config.go index dd29569fd..c682735b4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -56,13 +56,13 @@ type Commit0Config struct { Network Network `yaml:"network"` Services []Service `yaml:"services"` React React `yaml:"react"` - Kubernetes Kubernetes `yaml:"kubernetes"` + Kubernetes Kubernetes `yaml:"kubernetes"` } type Kubernetes struct { ClusterName string - DNSName string - Deploy bool + DNSName string + Deploy bool } func LoadConfig(filePath string) *Commit0Config { diff --git a/internal/generate/kubernetes/generate.go b/internal/generate/kubernetes/generate.go index ca45c937b..aae24020c 100644 --- a/internal/generate/kubernetes/generate.go +++ b/internal/generate/kubernetes/generate.go @@ -3,19 +3,18 @@ package kubernetes import ( "bytes" "fmt" + "github.com/commitdev/commit0/config" + "github.com/commitdev/commit0/templator" "io" - "sync" - "log" + "log" "os" "os/exec" - "path/filepath" - "github.com/commitdev/commit0/config" - "github.com/commitdev/commit0/templator" + "path/filepath" + "sync" ) func Generate(templator *templator.Templator, config *config.Commit0Config) { templator.Kubernetes.TemplateFiles(config, false) - if config.Kubernetes.Deploy { _tf_init := tf_init() @@ -23,31 +22,30 @@ func Generate(templator *templator.Templator, config *config.Commit0Config) { execute(_tf_init) execute(_tf_plan) } - + } // Terraform init cmd func tf_init() *exec.Cmd { - - return exec.Command("terraform","init") + + return exec.Command("terraform", "init") } // Terraform plan cmd func tf_plan() *exec.Cmd { - - return exec.Command("terraform","plan") + + return exec.Command("terraform", "plan") } -// Executes cmd passed in -func execute( cmd *exec.Cmd){ +// Executes cmd passed in +func execute(cmd *exec.Cmd) { dir, err1 := filepath.Abs(filepath.Dir(os.Args[0])) - if err1 != nil { - log.Fatal(err1) + if err1 != nil { + log.Fatal(err1) } cmd.Dir = dir + "/kubernetes/terraform" - var errStdout, errStderr error stdoutIn, _ := cmd.StdoutPipe() stderrIn, _ := cmd.StderrPipe() From 46a3fcdcfdc5036ea62b3a3087a05a38f951fd9e Mon Sep 17 00:00:00 2001 From: Shah Newaz Khan Date: Thu, 24 Oct 2019 11:55:51 -0700 Subject: [PATCH 5/8] Moved to eks and vpc community modules | Added awsaccountid & awsregion commit0 kubernetes vars --- internal/config/config.go | 7 +- internal/generate/kubernetes/generate.go | 6 +- templates/kubernetes/terraform/README.md | 102 ++++- templates/kubernetes/terraform/eks-cluster.tf | 77 ---- .../kubernetes/terraform/eks-worker-ndoes.tf | 145 ------- .../environments/development/main.tf | 22 ++ .../terraform/environments/production/main.tf | 21 + .../terraform/environments/staging/main.tf | 21 + .../kubernetes/terraform/modules/ecr/main.tf | 9 + .../terraform/modules/ecr/variables.tf | 9 + .../terraform/modules/ecr/versions.tf | 4 + .../kubernetes/terraform/modules/eks/main.tf | 47 +++ .../terraform/modules/eks/outputs.tf | 19 + .../terraform/modules/eks/variables.tf | 37 ++ .../terraform/modules/eks/versions.tf | 4 + .../terraform/modules/environment/backend.tf | 9 + .../terraform/modules/environment/main.tf | 58 +++ .../terraform/modules/environment/provider.tf | 5 + .../modules/environment/variables.tf | 33 ++ .../terraform/modules/environment/versions.tf | 3 + .../terraform/modules/kube2iam/README.md | 6 + .../terraform/modules/kube2iam/main.tf | 130 +++++++ .../terraform/modules/kube2iam/variables.tf | 15 + .../terraform/modules/kube2iam/versions.tf | 4 + .../terraform/modules/kubernetes/README.md | 27 ++ .../modules/kubernetes/example/README.md | 0 .../modules/kubernetes/example/main.tf | 30 ++ .../modules/kubernetes/example/variables.tf | 0 .../modules/kubernetes/ingress/README.md | 4 + .../modules/kubernetes/ingress/main.tf | 362 ++++++++++++++++++ .../modules/kubernetes/ingress/variables.tf | 11 + .../modules/kubernetes/kube2iam/README.md | 6 + .../modules/kubernetes/kube2iam/main.tf | 105 +++++ .../modules/kubernetes/kube2iam/variables.tf | 7 + .../terraform/modules/kubernetes/main.tf | 20 + .../files/cwagentconfig.json.tpl | 21 + .../monitoring/cloudwatch_agent/main.tf | 220 +++++++++++ .../monitoring/cloudwatch_agent/variables.tf | 11 + .../monitoring/docs/test-logging-app.yaml | 10 + .../monitoring/fluentd/files/containers.conf | 44 +++ .../monitoring/fluentd/files/fluent.conf | 10 + .../monitoring/fluentd/files/host.conf | 69 ++++ .../monitoring/fluentd/files/systemd.conf | 75 ++++ .../kubernetes/monitoring/fluentd/main.tf | 220 +++++++++++ .../monitoring/fluentd/variables.tf | 11 + .../modules/kubernetes/monitoring/main.tf | 38 ++ .../kubernetes/monitoring/variables.tf | 15 + .../terraform/modules/kubernetes/provider.tf | 16 + .../terraform/modules/kubernetes/variables.tf | 15 + .../kubernetes/terraform/modules/vpc/main.tf | 34 ++ .../terraform/modules/vpc/outputs.tf | 35 ++ .../terraform/modules/vpc/variables.tf | 12 + .../terraform/modules/vpc/versions.tf | 4 + templates/kubernetes/terraform/outputs.tf | 55 --- templates/kubernetes/terraform/providers.tf | 13 - templates/kubernetes/terraform/variables.tf | 4 - templates/kubernetes/terraform/vpc.tf | 81 ---- 57 files changed, 1977 insertions(+), 401 deletions(-) delete mode 100644 templates/kubernetes/terraform/eks-cluster.tf delete mode 100644 templates/kubernetes/terraform/eks-worker-ndoes.tf create mode 100644 templates/kubernetes/terraform/environments/development/main.tf create mode 100644 templates/kubernetes/terraform/environments/production/main.tf create mode 100644 templates/kubernetes/terraform/environments/staging/main.tf create mode 100644 templates/kubernetes/terraform/modules/ecr/main.tf create mode 100644 templates/kubernetes/terraform/modules/ecr/variables.tf create mode 100644 templates/kubernetes/terraform/modules/ecr/versions.tf create mode 100644 templates/kubernetes/terraform/modules/eks/main.tf create mode 100644 templates/kubernetes/terraform/modules/eks/outputs.tf create mode 100644 templates/kubernetes/terraform/modules/eks/variables.tf create mode 100644 templates/kubernetes/terraform/modules/eks/versions.tf create mode 100644 templates/kubernetes/terraform/modules/environment/backend.tf create mode 100644 templates/kubernetes/terraform/modules/environment/main.tf create mode 100644 templates/kubernetes/terraform/modules/environment/provider.tf create mode 100644 templates/kubernetes/terraform/modules/environment/variables.tf create mode 100644 templates/kubernetes/terraform/modules/environment/versions.tf create mode 100644 templates/kubernetes/terraform/modules/kube2iam/README.md create mode 100644 templates/kubernetes/terraform/modules/kube2iam/main.tf create mode 100644 templates/kubernetes/terraform/modules/kube2iam/variables.tf create mode 100644 templates/kubernetes/terraform/modules/kube2iam/versions.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/README.md create mode 100644 templates/kubernetes/terraform/modules/kubernetes/example/README.md create mode 100644 templates/kubernetes/terraform/modules/kubernetes/example/main.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/example/variables.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/ingress/README.md create mode 100755 templates/kubernetes/terraform/modules/kubernetes/ingress/main.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/ingress/variables.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/kube2iam/README.md create mode 100755 templates/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/main.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent/files/cwagentconfig.json.tpl create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent/main.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent/variables.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/docs/test-logging-app.yaml create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/containers.conf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/fluent.conf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/host.conf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/systemd.conf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/main.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/variables.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/provider.tf create mode 100644 templates/kubernetes/terraform/modules/kubernetes/variables.tf create mode 100644 templates/kubernetes/terraform/modules/vpc/main.tf create mode 100644 templates/kubernetes/terraform/modules/vpc/outputs.tf create mode 100644 templates/kubernetes/terraform/modules/vpc/variables.tf create mode 100644 templates/kubernetes/terraform/modules/vpc/versions.tf delete mode 100644 templates/kubernetes/terraform/outputs.tf delete mode 100644 templates/kubernetes/terraform/providers.tf delete mode 100644 templates/kubernetes/terraform/variables.tf delete mode 100644 templates/kubernetes/terraform/vpc.tf diff --git a/internal/config/config.go b/internal/config/config.go index c682735b4..86fb7953d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -60,9 +60,10 @@ type Commit0Config struct { } type Kubernetes struct { - ClusterName string - DNSName string - Deploy bool + ClusterName string + Deploy bool + AWSAccountId string + AWSRegion string } func LoadConfig(filePath string) *Commit0Config { diff --git a/internal/generate/kubernetes/generate.go b/internal/generate/kubernetes/generate.go index aae24020c..4e26fc5ed 100644 --- a/internal/generate/kubernetes/generate.go +++ b/internal/generate/kubernetes/generate.go @@ -2,7 +2,6 @@ package kubernetes import ( "bytes" - "fmt" "github.com/commitdev/commit0/config" "github.com/commitdev/commit0/templator" "io" @@ -44,7 +43,7 @@ func execute(cmd *exec.Cmd) { log.Fatal(err1) } - cmd.Dir = dir + "/kubernetes/terraform" + cmd.Dir = dir + "/kubernetes/terraform/environments/staging" var errStdout, errStderr error stdoutIn, _ := cmd.StdoutPipe() @@ -74,9 +73,6 @@ func execute(cmd *exec.Cmd) { if errStdout != nil || errStderr != nil { log.Fatal("failed to capture stdout or stderr\n") } - outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes()) - fmt.Printf("\nout:\n%s\nerr:\n%s\n", outStr, errStr) - } // CapturingPassThroughWriter is a writer that remembers diff --git a/templates/kubernetes/terraform/README.md b/templates/kubernetes/terraform/README.md index f779a3278..aa075b7b2 100644 --- a/templates/kubernetes/terraform/README.md +++ b/templates/kubernetes/terraform/README.md @@ -1,29 +1,95 @@ -# EKS Terraform +## Guidelines & Style Convention Summary -AWS Resources created: +- All Terraform configuration should be formatted with `terraform fmt` before being accepted into this repository. +- This repository is Terraform version >= 0.12, as such, leverage features from this release whenever possible. + See https://www.terraform.io/upgrade-guides/0-12.html for more information. +- Leverage community-maintained Terraform modules whenever possible. +- Attempt to minimize duplication whenever possible, but only within reason -- sometimes duplication is an acceptable solution. +- Follow style conventions described in `docs/guide.pdf` whenever possible. +- Whenever possible, inject resources down versus referencing resources across modules. This has been made easier with new features in v0.12. +- Whenever possible, define the types of variables. -- EKS Cluster: AWS managed Kubernetes cluster of master servers -- AutoScaling Group containing 2 m4.large instances based on the latest EKS Amazon Linux 2 AMI: Operator managed Kubernetes worker nodes for running Kubernetes service deployments -- Associated VPC, Internet Gateway, Security Groups, and Subnets: Operator managed networking resources for the EKS Cluster and worker node instances -- Associated IAM Roles and Policies: Operator managed access resources for EKS and worker node instances +### Module Conventions -## Pre-requisites +- All modules should contain the following: -- Setup the [AWS credentials](https://www.terraform.io/docs/providers/aws/index.html#environment-variables) for terraform + `README.md`: A description of the module. + `main.tf`: Module entrypoint where instantiation of resources happens. + `variables.tf`: Module variables. + `outputs.tf`: Output values (optional). + `files/`: Any / all files required by the module. -## Spin up cluster +- All module variables must have a description. +- Again, leverage community-maintained Terraform modules whenever possible. +- Avoid writing a module that is simply a wrapper of a Terraform resource unless absolutely necessary. -```shell +### Environment Conventions -terraform plan -terraform apply +- All environments should contain the following: + + `main.tf`: Toplevel terraform configuration file that instantiates the `environment` module. + +- Configuration should be pushed "top->down" from the `environment` module to it's submodules. + +### The Environment Module + +- The `environment` module can be considered the top-level module, all other modules are imported from this module. +- Environment-specific variables should be exposed via the `variables.tf` file in this module, where they will be set from within the appropriate environment in the `environments/` directory. +- The `environment` module contains the following: + + `main.tf`: Module entrypoint where instantiation of resources happens. + `backend.tf`: Terraform remote state configuration. + `provider.tf`: Provider configuration. + `variables.tf`: Environment-specific variables are desclared here. + `versions.tf`: Terraform version information. + `files/`: (DEPRECATED) + +## Directory Structure ``` + README.md + environments/ + production/ + main.tf + staging/ + main.tf + development/ + main.tf + docs/ + guide.pdf + modules/ + environment/ + ... + / + files/ + scripts/ + main.tf + outputs.tf + variables.tf + / + ... +``` + +## AWS Guidelines + +- TODO: Identity/Access Management (IAM) Guidelines + +## Kubernetes Guidelines + +- When to use the Terraform Kuberenetes Provider and when to use manifests? + + - Use the Terraform Kubernetes Provider (`provider "kubernetes"`) whenever you are provisioning a resource that could be considered relatively static (think Ingress, RoleBinding, CluterRoleBinding, etc). + + - Use conventional Kubernetes manifests / `kubectl` when provisioning resouirces that could be considered dynamic (think Deployments). + +## Application + + 1. Set up a profile for your project with your credentials in a specific profile in `~/.aws/credentials` and then export the following env var: + `export AWS_PROFILE=` -### Connect to cluster -The EKS service does not provide a cluster-level API parameter or resource to automatically configure the underlying Kubernetes cluster to allow worker nodes to join the cluster via AWS IAM role authentication. + 2. Run the following from the appropriate environment directory under `environments/`: -- Run `aws eks update-kubeconfig --name staging` to configure `kubectl` -- Run `terraform output config_map_aws_auth` and save the configuration into a file, e.g. config_map_aws_auth.yaml -- Run `kubectl apply -f config_map_aws_auth.yaml` -- You can verify the worker nodes are joining the cluster via: `kubectl get nodes --watch` + ``` + environment/development$ terraform init + environment/development$ terraform plan + ``` diff --git a/templates/kubernetes/terraform/eks-cluster.tf b/templates/kubernetes/terraform/eks-cluster.tf deleted file mode 100644 index 8b18ef6ed..000000000 --- a/templates/kubernetes/terraform/eks-cluster.tf +++ /dev/null @@ -1,77 +0,0 @@ -# -# EKS Cluster Resources -# * IAM Role to allow EKS service to manage other AWS services -# * EC2 Security Group to allow networking traffic with EKS cluster -# * EKS Cluster -# - -resource "aws_iam_role" "demo-cluster" { - name = "terraform-eks-demo-cluster" - - assume_role_policy = < + @type tail + @id in_tail_container_logs + @label @containers + path /var/log/containers/*.log + pos_file /var/log/fluentd-containers.log.pos + tag * + read_from_head true + + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + + \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/fluent.conf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/fluent.conf new file mode 100644 index 000000000..31bc30944 --- /dev/null +++ b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/fluent.conf @@ -0,0 +1,10 @@ +@include containers.conf +@include systemd.conf +@include host.conf + + + @type null + + + log_level debug + \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/host.conf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/host.conf new file mode 100644 index 000000000..a43a2aa2a --- /dev/null +++ b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/host.conf @@ -0,0 +1,69 @@ + + @type tail + @id in_tail_dmesg + @label @hostlogs + path /var/log/dmesg + pos_file /var/log/dmesg.log.pos + tag host.dmesg + read_from_head true + + @type syslog + + + + + @type tail + @id in_tail_secure + @label @hostlogs + path /var/log/secure + pos_file /var/log/secure.log.pos + tag host.secure + read_from_head true + + @type syslog + + + + + @type tail + @id in_tail_messages + @label @hostlogs + path /var/log/messages + pos_file /var/log/messages.log.pos + tag host.messages + read_from_head true + + @type syslog + + + + \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/systemd.conf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/systemd.conf new file mode 100644 index 000000000..c1a6e8240 --- /dev/null +++ b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/files/systemd.conf @@ -0,0 +1,75 @@ + + @type systemd + @id in_systemd_kubelet + @label @systemd + filters [{ "_SYSTEMD_UNIT": "kubelet.service" }] + + field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} + field_map_strict true + + path /var/log/journal + pos_file /var/log/fluentd-journald-kubelet.pos + read_from_head true + tag kubelet.service + + + + @type systemd + @id in_systemd_kubeproxy + @label @systemd + filters [{ "_SYSTEMD_UNIT": "kubeproxy.service" }] + + field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} + field_map_strict true + + path /var/log/journal + pos_file /var/log/fluentd-journald-kubeproxy.pos + read_from_head true + tag kubeproxy.service + + + + @type systemd + @id in_systemd_docker + @label @systemd + filters [{ "_SYSTEMD_UNIT": "docker.service" }] + + field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} + field_map_strict true + + path /var/log/journal + pos_file /var/log/fluentd-journald-docker.pos + read_from_head true + tag docker.service + + + \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/main.tf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/main.tf new file mode 100644 index 000000000..de51422a0 --- /dev/null +++ b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/main.tf @@ -0,0 +1,220 @@ +data "local_file" "containers" { + filename = "${path.module}/files/containers.conf" +} + +data "local_file" "fluent" { + filename = "${path.module}/files/fluent.conf" +} + +data "local_file" "host" { + filename = "${path.module}/files/host.conf" +} + +data "local_file" "systemd" { + filename = "${path.module}/files/systemd.conf" +} + +resource "kubernetes_config_map" "cluster_info" { + metadata { + name = "cluster-info" + namespace = "amazon-cloudwatch" + } + data = { + "cluster.name" = var.cluster_name + "logs.region" = var.region + } +} + +resource "kubernetes_service_account" "fluentd" { + metadata { + name = "fluentd" + namespace = "amazon-cloudwatch" + } +} + +resource "kubernetes_cluster_role" "fluentd_role" { + metadata { + name = "fluentd-role" + } + rule { + verbs = ["get", "list", "watch"] + api_groups = [""] + resources = ["namespaces", "pods", "pods/logs"] + } +} + +resource "kubernetes_cluster_role_binding" "fluentd_role_binding" { + metadata { + name = "fluentd-role-binding" + } + subject { + kind = "ServiceAccount" + name = "fluentd" + namespace = "amazon-cloudwatch" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "fluentd-role" + } +} + +resource "kubernetes_config_map" "fluentd_config" { + metadata { + name = "fluentd-config" + namespace = "amazon-cloudwatch" + labels = { k8s-app = "fluentd-cloudwatch" } + } + data = { + "containers.conf" = data.local_file.containers.content + "fluent.conf" = data.local_file.fluent.content + "host.conf" = data.local_file.host.content + "systemd.conf" = data.local_file.systemd.content + } +} + +resource "kubernetes_daemonset" "fluentd_cloudwatch" { + depends_on = [ + kubernetes_config_map.cluster_info, + kubernetes_config_map.fluentd_config + ] + metadata { + name = "fluentd-cloudwatch" + namespace = "amazon-cloudwatch" + labels = { + k8s-app = "fluentd-cloudwatch" + } + } + spec { + selector { + match_labels = { + k8s-app = "fluentd-cloudwatch" + } + } + template { + metadata { + labels = { + k8s-app = "fluentd-cloudwatch" + } + annotations = { + configHash = "8915de4cf9c3551a8dc74c0137a3e83569d28c71044b0359c2578d2e0461825", + "iam.amazonaws.com/role" = "k8s-${var.environment}-monitoring" + } + } + spec { + volume { + name = "config-volume" + config_map { + name = "fluentd-config" + } + } + volume { + name = "fluentdconf" + } + volume { + name = "varlog" + host_path { + path = "/var/log" + } + } + volume { + name = "varlibdockercontainers" + host_path { + path = "/var/lib/docker/containers" + } + } + volume { + name = "runlogjournal" + host_path { + path = "/run/log/journal" + } + } + volume { + name = "dmesg" + host_path { + path = "/var/log/dmesg" + } + } + init_container { + name = "copy-fluentd-config" + image = "busybox" + command = ["sh", "-c", "cp /config-volume/..data/* /fluentd/etc"] + volume_mount { + name = "config-volume" + mount_path = "/config-volume" + } + volume_mount { + name = "fluentdconf" + mount_path = "/fluentd/etc" + } + } + init_container { + name = "update-log-driver" + image = "busybox" + command = ["sh", "-c", ""] + } + container { + name = "fluentd-cloudwatch" + image = "fluent/fluentd-kubernetes-daemonset:v1.3.3-debian-cloudwatch-1.4" + env { + name = "REGION" + value_from { + config_map_key_ref { + name = "cluster-info" + key = "logs.region" + } + } + } + env { + name = "CLUSTER_NAME" + value_from { + config_map_key_ref { + name = "cluster-info" + key = "cluster.name" + } + } + } + resources { + limits { + memory = "200Mi" + } + requests { + cpu = "100m" + memory = "200Mi" + } + } + volume_mount { + name = "config-volume" + mount_path = "/config-volume" + } + volume_mount { + name = "fluentdconf" + mount_path = "/fluentd/etc" + } + volume_mount { + name = "varlog" + mount_path = "/var/log" + } + volume_mount { + name = "varlibdockercontainers" + read_only = true + mount_path = "/var/lib/docker/containers" + } + volume_mount { + name = "runlogjournal" + read_only = true + mount_path = "/run/log/journal" + } + volume_mount { + name = "dmesg" + read_only = true + mount_path = "/var/log/dmesg" + } + } + termination_grace_period_seconds = 30 + service_account_name = "fluentd" + automount_service_account_token = true + } + } + } +} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/variables.tf new file mode 100644 index 000000000..2fac8d3e3 --- /dev/null +++ b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/variables.tf @@ -0,0 +1,11 @@ +variable "environment" { + description = "Environment (production/staging/development)" +} + +variable "region" { + description = "AWS Region" +} + +variable "cluster_name" { + description = "Cluster name to monitor" +} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf new file mode 100644 index 000000000..2297f5f13 --- /dev/null +++ b/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf @@ -0,0 +1,38 @@ +resource "aws_iam_role" "k8s_monitoring" { + name = "k8s-${var.environment}-monitoring" + assume_role_policy = var.assume_role_policy + force_detach_policies = true +} + +# Create amazon-cloudwatch kubernetes namespace for fluentd/cloudwatchagent +resource "kubernetes_namespace" "amazon_cloudwatch" { + metadata { + name = "amazon-cloudwatch" + labels = { + name = "amazon-cloudwatch" + } + } +} + +data "aws_iam_policy" "CloudWatchAgentServerPolicy" { + arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" +} + +resource "aws_iam_role_policy_attachment" "k8s_monitoring_role_policy" { + role = "${aws_iam_role.k8s_monitoring.id}" + policy_arn = "${data.aws_iam_policy.CloudWatchAgentServerPolicy.arn}" +} + +module "fluentd" { + source = "./fluentd" + environment = var.environment + region = var.region + cluster_name = var.cluster_name +} + +module "cloudwatch_agent" { + source = "./cloudwatch_agent" + environment = var.environment + region = var.region + cluster_name = var.cluster_name +} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf new file mode 100644 index 000000000..93e911d14 --- /dev/null +++ b/templates/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf @@ -0,0 +1,15 @@ +variable "region" { + description = "AWS Region" +} + +variable "environment" { + description = "Environment" +} + +variable "assume_role_policy" { + description = "Assume-role policy" +} + +variable "cluster_name" { + description = "Cluster name" +} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/provider.tf b/templates/kubernetes/terraform/modules/kubernetes/provider.tf new file mode 100644 index 000000000..680554dbf --- /dev/null +++ b/templates/kubernetes/terraform/modules/kubernetes/provider.tf @@ -0,0 +1,16 @@ +# https://github.com/terraform-providers/terraform-provider-kubernetes/issues/161#issuecomment-461190931 + +data "aws_eks_cluster" "cluster" { + name = var.cluster_name +} + +data "aws_eks_cluster_auth" "cluster_auth" { + name = "${data.aws_eks_cluster.cluster.name}" +} + +provider "kubernetes" { + host = "${data.aws_eks_cluster.cluster.endpoint}" + cluster_ca_certificate = "${base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)}" + token = "${data.aws_eks_cluster_auth.cluster_auth.token}" + load_config_file = false +} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/variables.tf new file mode 100644 index 000000000..050b728b8 --- /dev/null +++ b/templates/kubernetes/terraform/modules/kubernetes/variables.tf @@ -0,0 +1,15 @@ +variable "region" { + description = "AWS Region" +} + +variable "environment" { + description = "Environment" +} + +variable "cluster_name" { + description = "Kubernetes cluster name" +} + +variable "assume_role_policy" { + description = "Assume-role policy for monitoring" +} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/vpc/main.tf b/templates/kubernetes/terraform/modules/vpc/main.tf new file mode 100644 index 000000000..1c2d7132b --- /dev/null +++ b/templates/kubernetes/terraform/modules/vpc/main.tf @@ -0,0 +1,34 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + + name = "${var.project}-${var.environment}-vpc" + cidr = "10.10.0.0/16" + + azs = ["${var.region}a", "${var.region}b", "${var.region}c"] # Most regions have 3+ azs + private_subnets = ["10.10.40.0/24", "10.10.42.0/24", "10.10.44.0/24"] + public_subnets = ["10.10.41.0/24", "10.10.43.0/24", "10.10.45.0/24"] + database_subnets = ["10.10.50.0/24", "10.10.52.0/24", "10.10.54.0/24"] + + # Allow kubernetes ALB ingress controller to auto-detect + private_subnet_tags = { + "kubernetes.io/cluster/${var.project}-${var.environment}" = "owned" + "kubernetes.io/role/internal-elb" = "1" + } + + public_subnet_tags = { + "kubernetes.io/cluster/${var.project}-${var.environment}" = "owned" + "kubernetes.io/role/elb" = "1" + } + + enable_nat_gateway = true + enable_vpn_gateway = false + enable_dns_hostnames = true + + create_database_subnet_group = true + create_database_subnet_route_table = true + + tags = { + environment = var.environment + } + +} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/vpc/outputs.tf b/templates/kubernetes/terraform/modules/vpc/outputs.tf new file mode 100644 index 000000000..cd9f19e60 --- /dev/null +++ b/templates/kubernetes/terraform/modules/vpc/outputs.tf @@ -0,0 +1,35 @@ +output "vpc_id" { + description = "The ID of the created VPC" + value = module.vpc.vpc_id +} + +output "vpc_cidr_block" { + description = "The CIDR block of the VPC" + value = module.vpc.vpc_cidr_block +} + +output "azs" { + description = "Availability zones for the VPC" + value = module.vpc.azs +} + +output "private_subnets" { + description = "List of private subnets" + value = module.vpc.private_subnets +} + +output "public_subnets" { + description = "List of public subnets" + value = module.vpc.public_subnets +} + +output "database_subnets" { + description = "List of public subnets" + value = module.vpc.database_subnets +} + +output "database_subnet_group" { + description = "List of subnet groups" + value = module.vpc.database_subnet_group +} + diff --git a/templates/kubernetes/terraform/modules/vpc/variables.tf b/templates/kubernetes/terraform/modules/vpc/variables.tf new file mode 100644 index 000000000..a7c32fe5a --- /dev/null +++ b/templates/kubernetes/terraform/modules/vpc/variables.tf @@ -0,0 +1,12 @@ +variable "project" { + description = "The name of the project, mostly for tagging" +} + +variable "environment" { + description = "The environment (dev/staging/prod)" +} + +variable "region" { + description = "The AWS region" +} + diff --git a/templates/kubernetes/terraform/modules/vpc/versions.tf b/templates/kubernetes/terraform/modules/vpc/versions.tf new file mode 100644 index 000000000..ac97c6ac8 --- /dev/null +++ b/templates/kubernetes/terraform/modules/vpc/versions.tf @@ -0,0 +1,4 @@ + +terraform { + required_version = ">= 0.12" +} diff --git a/templates/kubernetes/terraform/outputs.tf b/templates/kubernetes/terraform/outputs.tf deleted file mode 100644 index 571f07868..000000000 --- a/templates/kubernetes/terraform/outputs.tf +++ /dev/null @@ -1,55 +0,0 @@ -# -# Outputs -# - -locals { - config_map_aws_auth = < Date: Mon, 28 Oct 2019 01:00:08 -0700 Subject: [PATCH 6/8] Fixed merge conficts from rebase --- cmd/generate.go | 9 +- go.sum | 2 - internal/generate/kubernetes/generate.go | 8 +- internal/templator/templator.go | 2 + templator/templator.go | 194 ----------------------- 5 files changed, 10 insertions(+), 205 deletions(-) delete mode 100644 templator/templator.go diff --git a/cmd/generate.go b/cmd/generate.go index 5452d518c..be67854cd 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -1,20 +1,19 @@ package cmd import ( - - "log" - "sync" "github.com/commitdev/commit0/internal/config" "github.com/commitdev/commit0/internal/generate/docker" "github.com/commitdev/commit0/internal/generate/golang" "github.com/commitdev/commit0/internal/generate/http" + "github.com/commitdev/commit0/internal/generate/kubernetes" "github.com/commitdev/commit0/internal/generate/proto" "github.com/commitdev/commit0/internal/generate/react" - "github.com/commitdev/commit0/internal/generate/kubernetes" "github.com/commitdev/commit0/internal/templator" "github.com/commitdev/commit0/internal/util" "github.com/gobuffalo/packr/v2" "github.com/spf13/cobra" + "log" + "sync" ) var configPath string @@ -62,7 +61,7 @@ var generateCmd = &cobra.Command{ case React: react.Generate(t, cfg, &wg) case Kubernetes: - kubernetes.Generate(Templator, cfg) + kubernetes.Generate(t, cfg, &wg) } util.TemplateFileIfDoesNotExist("", "README.md", t.Readme, &wg, cfg) diff --git a/go.sum b/go.sum index 6b23815f4..da4d2ce59 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/internal/generate/kubernetes/generate.go b/internal/generate/kubernetes/generate.go index 4e26fc5ed..41ed0c6d7 100644 --- a/internal/generate/kubernetes/generate.go +++ b/internal/generate/kubernetes/generate.go @@ -2,8 +2,8 @@ package kubernetes import ( "bytes" - "github.com/commitdev/commit0/config" - "github.com/commitdev/commit0/templator" + "github.com/commitdev/commit0/internal/config" + "github.com/commitdev/commit0/internal/templator" "io" "log" "os" @@ -12,8 +12,8 @@ import ( "sync" ) -func Generate(templator *templator.Templator, config *config.Commit0Config) { - templator.Kubernetes.TemplateFiles(config, false) +func Generate(templator *templator.Templator, config *config.Commit0Config, wg *sync.WaitGroup) { + templator.Kubernetes.TemplateFiles(config, false, wg) if config.Kubernetes.Deploy { _tf_init := tf_init() diff --git a/internal/templator/templator.go b/internal/templator/templator.go index 2369164ab..aed7b7822 100644 --- a/internal/templator/templator.go +++ b/internal/templator/templator.go @@ -41,6 +41,7 @@ type Templator struct { Go *GoTemplator Docker *DockerTemplator React *DirectoryTemplator + Kubernetes *DirectoryTemplator } func NewTemplator(box *packr.Box) *Templator { @@ -54,6 +55,7 @@ func NewTemplator(box *packr.Box) *Templator { Readme: NewSingleFileTemplator(box, "util/README.tmpl"), Docker: NewDockerFileTemplator(box), React: NewDirectoryTemplator(box, "react"), + Kubernetes: NewDirectoryTemplator(box, "kubernetes"), } } diff --git a/templator/templator.go b/templator/templator.go deleted file mode 100644 index 4ccd37956..000000000 --- a/templator/templator.go +++ /dev/null @@ -1,194 +0,0 @@ -package templator - -import ( - "path/filepath" - "strings" - "text/template" - - "github.com/commitdev/commit0/config" - "github.com/commitdev/commit0/util" - "github.com/gobuffalo/packr/v2" - "github.com/gobuffalo/packr/v2/file" -) - -type DockerTemplator struct { - ApplicationDocker *template.Template - HttpGatewayDocker *template.Template - DockerIgnore *template.Template - DockerCompose *template.Template -} - -type GoTemplator struct { - GoMain *template.Template - GoMod *template.Template - GoModIDL *template.Template - GoServer *template.Template - GoHealthServer *template.Template - GoHttpGW *template.Template -} - -type Templator struct { - Commit0 *template.Template - GitIgnore *template.Template - MakefileTemplate *template.Template - ProtoHealthTemplate *template.Template - ProtoServiceTemplate *template.Template - Go *GoTemplator - Docker *DockerTemplator - React *DirectoryTemplator - Kubernetes *DirectoryTemplator -} - -func NewTemplator(box *packr.Box) *Templator { - makeFileTemplateSource, _ := box.FindString("proto/makefile.tmpl") - makeFileTemplate, _ := template.New("ProtoToolTemplate").Parse(makeFileTemplateSource) - - protoHealthTemplateSource, _ := box.FindString("proto/health_proto.tmpl") - protoHealthTemplate, _ := template.New("ProtoHealthTemplate").Parse(protoHealthTemplateSource) - - protoServiceTemplateSource, _ := box.FindString("proto/service_proto.tmpl") - protoServiceTemplate, _ := template.New("ProtoServiceTemplate").Funcs(util.FuncMap).Parse(protoServiceTemplateSource) - - return &Templator{ - MakefileTemplate: makeFileTemplate, - ProtoHealthTemplate: protoHealthTemplate, - ProtoServiceTemplate: protoServiceTemplate, - Go: NewGoTemplator(box), - Commit0: NewCommit0Templator(box), - GitIgnore: NewGitIgnoreTemplator(box), - Docker: NewDockerFileTemplator(box), - React: NewDirectoryTemplator(box, "react"), - Kubernetes: NewDirectoryTemplator(box, "kubernetes"), - } -} - -func NewGoTemplator(box *packr.Box) *GoTemplator { - goServerTemplateSource, _ := box.FindString("golang/server.tmpl") - goServerTemplate, _ := template.New("GoServerTemplate").Funcs(util.FuncMap).Parse(goServerTemplateSource) - - goHealthTemplateSource, _ := box.FindString("golang/health_server.tmpl") - goHealthServerTemplate, _ := template.New("GoHealthServerTemplate").Parse(goHealthTemplateSource) - - goModTemplateSource, _ := box.FindString("golang/go_mod.tmpl") - goModTemplate, _ := template.New("GoModTemplate").Parse(goModTemplateSource) - - goModIDLTemplateSource, _ := box.FindString("golang/go_mod_idl.tmpl") - goModIDLTemplate, _ := template.New("GoModTemplate").Parse(goModIDLTemplateSource) - - goMainTemplateSource, _ := box.FindString("golang/main.tmpl") - goMainTemplate, _ := template.New("GoMainTemplate").Funcs(util.FuncMap).Parse(goMainTemplateSource) - - goHttpTemplateSource, _ := box.FindString("golang/http_gw.tmpl") - goHttpTemplate, _ := template.New("GoHttpGWTemplate").Funcs(util.FuncMap).Parse(goHttpTemplateSource) - - return &GoTemplator{ - GoMain: goMainTemplate, - GoMod: goModTemplate, - GoModIDL: goModIDLTemplate, - GoServer: goServerTemplate, - GoHealthServer: goHealthServerTemplate, - GoHttpGW: goHttpTemplate, - } - -} - -func NewCommit0Templator(box *packr.Box) *template.Template { - templateSource, _ := box.FindString("commit0/commit0.tmpl") - template, _ := template.New("Commit0Template").Funcs(util.FuncMap).Parse(templateSource) - - return template -} - -func NewGitIgnoreTemplator(box *packr.Box) *template.Template { - templateSource, _ := box.FindString("util/gitignore.tmpl") - template, _ := template.New("GitIgnore").Parse(templateSource) - return template -} - -func NewDockerFileTemplator(box *packr.Box) *DockerTemplator { - appTemplateSource, _ := box.FindString("docker/dockerfile_app.tmpl") - appTemplate, _ := template.New("AppDockerfile").Parse(appTemplateSource) - - httpTemplateSource, _ := box.FindString("docker/dockerfile_http.tmpl") - httpTemplate, _ := template.New("HttpDockerfile").Parse(httpTemplateSource) - - ignoreTemplateSource, _ := box.FindString("docker/dockerignore.tmpl") - ignoreTemplate, _ := template.New("Dockerignore").Parse(ignoreTemplateSource) - - composeTemplateSource, _ := box.FindString("docker/dockercompose.tmpl") - composeTemplate, _ := template.New("Dockercompose").Parse(composeTemplateSource) - - return &DockerTemplator{ - ApplicationDocker: appTemplate, - HttpGatewayDocker: httpTemplate, - DockerIgnore: ignoreTemplate, - DockerCompose: composeTemplate, - } -} - -type DirectoryTemplator struct { - Templates []*template.Template -} - -func (d *DirectoryTemplator) TemplateFiles(config *config.Commit0Config, overwrite bool) { - for _, template := range d.Templates { - d, f := filepath.Split(template.Name()) - if strings.HasSuffix(f, ".tmpl") { - f = strings.Replace(f, ".tmpl", "", -1) - } - if overwrite { - util.TemplateFileAndOverwrite(d, f, template, config) - } else { - util.TemplateFileIfDoesNotExist(d, f, template, config) - } - } -} - -func NewDirectoryTemplator(box *packr.Box, dir string) *DirectoryTemplator { - templates := []*template.Template{} - for _, file := range getFileNames(box, dir) { - templateSource, _ := box.FindString(file) - template, err := template.New(file).Funcs(util.FuncMap).Parse(templateSource) - if err != nil { - panic(err) - } - templates = append(templates, template) - } - return &DirectoryTemplator{ - Templates: templates, - } -} - -func getFileNames(box *packr.Box, dir string) []string { - keys := []string{} - box.WalkPrefix(dir, func(path string, info file.File) error { - if info == nil { - return nil - } - finfo, _ := info.FileInfo() - if !finfo.IsDir() { - keys = append(keys, path) - } - return nil - }) - return removeTmplDuplicates(keys) -} - -func removeTmplDuplicates(keys []string) []string { - filteredKeys := []string{} - for _, key := range keys { - if !containsStr(keys, key+".tmpl") { - filteredKeys = append(filteredKeys, key) - } - } - return filteredKeys -} - -func containsStr(arr []string, key string) bool { - for _, val := range arr { - if val == key { - return true - } - } - return false -} From c721d7c2035575750678f411ed6554e3d926beec Mon Sep 17 00:00:00 2001 From: Shah Newaz Khan Date: Tue, 29 Oct 2019 00:14:06 -0700 Subject: [PATCH 7/8] Replaced execute stdout & stderr with multiwriter --- internal/generate/kubernetes/generate.go | 42 +++++++----------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/internal/generate/kubernetes/generate.go b/internal/generate/kubernetes/generate.go index 41ed0c6d7..e511cfead 100644 --- a/internal/generate/kubernetes/generate.go +++ b/internal/generate/kubernetes/generate.go @@ -2,14 +2,16 @@ package kubernetes import ( "bytes" - "github.com/commitdev/commit0/internal/config" - "github.com/commitdev/commit0/internal/templator" + "fmt" "io" "log" "os" "os/exec" "path/filepath" "sync" + + "github.com/commitdev/commit0/internal/config" + "github.com/commitdev/commit0/internal/templator" ) func Generate(templator *templator.Templator, config *config.Commit0Config, wg *sync.WaitGroup) { @@ -36,7 +38,6 @@ func tf_plan() *exec.Cmd { return exec.Command("terraform", "plan") } -// Executes cmd passed in func execute(cmd *exec.Cmd) { dir, err1 := filepath.Abs(filepath.Dir(os.Args[0])) if err1 != nil { @@ -45,11 +46,13 @@ func execute(cmd *exec.Cmd) { cmd.Dir = dir + "/kubernetes/terraform/environments/staging" - var errStdout, errStderr error + var stdoutBuf, stderrBuf bytes.Buffer stdoutIn, _ := cmd.StdoutPipe() stderrIn, _ := cmd.StderrPipe() - stdout := NewCapturingPassThroughWriter(os.Stdout) - stderr := NewCapturingPassThroughWriter(os.Stderr) + + var errStdout, errStderr error + stdout := io.MultiWriter(os.Stdout, &stdoutBuf) + stderr := io.MultiWriter(os.Stderr, &stderrBuf) err := cmd.Start() if err != nil { log.Fatalf("cmd.Start() failed with '%s'\n", err) @@ -73,29 +76,6 @@ func execute(cmd *exec.Cmd) { if errStdout != nil || errStderr != nil { log.Fatal("failed to capture stdout or stderr\n") } -} - -// CapturingPassThroughWriter is a writer that remembers -// data written to it and passes it to w -type CapturingPassThroughWriter struct { - buf bytes.Buffer - w io.Writer -} - -// NewCapturingPassThroughWriter creates new CapturingPassThroughWriter -func NewCapturingPassThroughWriter(w io.Writer) *CapturingPassThroughWriter { - return &CapturingPassThroughWriter{ - w: w, - } -} - -// Write writes data to the writer, returns number of bytes written and an error -func (w *CapturingPassThroughWriter) Write(d []byte) (int, error) { - w.buf.Write(d) - return w.w.Write(d) -} - -// Bytes returns bytes written to the writer -func (w *CapturingPassThroughWriter) Bytes() []byte { - return w.buf.Bytes() + outStr, errStr := string(stdoutBuf.Bytes()), string(stderrBuf.Bytes()) + fmt.Printf("\nout:\n%s\nerr:\n%s\n", outStr, errStr) } From 016a4b5e3c37a64e510ee3a6d81f873b0d234afa Mon Sep 17 00:00:00 2001 From: Bill Monkman Date: Tue, 29 Oct 2019 14:59:30 -0700 Subject: [PATCH 8/8] Some fixes to k8s generation and tf execution --- cmd/generate.go | 12 +++- internal/config/config.go | 8 +-- internal/generate/kubernetes/generate.go | 69 +++++++------------ templates/commit0/commit0.tmpl | 6 ++ .../kubernetes/terraform/modules/eks/main.tf | 4 +- 5 files changed, 48 insertions(+), 51 deletions(-) diff --git a/cmd/generate.go b/cmd/generate.go index be67854cd..eb32dd811 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -1,6 +1,9 @@ package cmd import ( + "log" + "sync" + "github.com/commitdev/commit0/internal/config" "github.com/commitdev/commit0/internal/generate/docker" "github.com/commitdev/commit0/internal/generate/golang" @@ -12,8 +15,6 @@ import ( "github.com/commitdev/commit0/internal/util" "github.com/gobuffalo/packr/v2" "github.com/spf13/cobra" - "log" - "sync" ) var configPath string @@ -71,7 +72,14 @@ var generateCmd = &cobra.Command{ docker.GenerateGoHTTPGWDockerFile(t, cfg, &wg) } + // Wait for all the templates to be generated wg.Wait() + + switch language { + case Kubernetes: + kubernetes.Execute(cfg) + } + }, } diff --git a/internal/config/config.go b/internal/config/config.go index 86fb7953d..a28230f62 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -60,10 +60,10 @@ type Commit0Config struct { } type Kubernetes struct { - ClusterName string - Deploy bool - AWSAccountId string - AWSRegion string + ClusterName string `yaml:"clusterName"` + Deploy bool `yaml:"deploy"` + AWSAccountId string `yaml:"awsAccountId"` + AWSRegion string `yaml:"awsRegion"` } func LoadConfig(filePath string) *Commit0Config { diff --git a/internal/generate/kubernetes/generate.go b/internal/generate/kubernetes/generate.go index e511cfead..cbe2da50e 100644 --- a/internal/generate/kubernetes/generate.go +++ b/internal/generate/kubernetes/generate.go @@ -1,13 +1,11 @@ package kubernetes import ( - "bytes" "fmt" "io" "log" "os" "os/exec" - "path/filepath" "sync" "github.com/commitdev/commit0/internal/config" @@ -17,65 +15,50 @@ import ( func Generate(templator *templator.Templator, config *config.Commit0Config, wg *sync.WaitGroup) { templator.Kubernetes.TemplateFiles(config, false, wg) - if config.Kubernetes.Deploy { - _tf_init := tf_init() - _tf_plan := tf_plan() - execute(_tf_init) - execute(_tf_plan) - } - } -// Terraform init cmd -func tf_init() *exec.Cmd { - - return exec.Command("terraform", "init") -} - -// Terraform plan cmd -func tf_plan() *exec.Cmd { - - return exec.Command("terraform", "plan") +func Execute(config *config.Commit0Config) { + if config.Kubernetes.Deploy { + log.Println("Planning infrastructure...") + execute(exec.Command("terraform", "init")) + execute(exec.Command("terraform", "plan")) + } } func execute(cmd *exec.Cmd) { - dir, err1 := filepath.Abs(filepath.Dir(os.Args[0])) - if err1 != nil { - log.Fatal(err1) + dir, err := os.Getwd() + if err != nil { + log.Fatalf("Getting working directory failed: %v\n", err) } - cmd.Dir = dir + "/kubernetes/terraform/environments/staging" + cmd.Dir = fmt.Sprintf("%s/kubernetes/terraform/environments/staging", dir) - var stdoutBuf, stderrBuf bytes.Buffer - stdoutIn, _ := cmd.StdoutPipe() - stderrIn, _ := cmd.StderrPipe() + stdoutPipe, _ := cmd.StdoutPipe() + stderrPipe, _ := cmd.StderrPipe() var errStdout, errStderr error - stdout := io.MultiWriter(os.Stdout, &stdoutBuf) - stderr := io.MultiWriter(os.Stderr, &stderrBuf) - err := cmd.Start() + err = cmd.Start() if err != nil { - log.Fatalf("cmd.Start() failed with '%s'\n", err) + log.Fatalf("Starting terraform command failed: %v\n", err) } - var wg sync.WaitGroup - wg.Add(1) - go func() { - _, errStdout = io.Copy(stdout, stdoutIn) - wg.Done() + _, errStdout = io.Copy(os.Stdout, stdoutPipe) + }() + go func() { + _, errStderr = io.Copy(os.Stderr, stderrPipe) }() - - _, errStderr = io.Copy(stderr, stderrIn) - wg.Wait() err = cmd.Wait() if err != nil { - log.Fatalf("cmd.Run() failed with %s\n", err) + log.Fatalf("Executing terraform command failed: %v\n", err) + } + + if errStdout != nil { + log.Printf("Failed to capture stdout: %v\n", errStdout) } - if errStdout != nil || errStderr != nil { - log.Fatal("failed to capture stdout or stderr\n") + + if errStderr != nil { + log.Printf("Failed to capture stderr: %v\n", errStderr) } - outStr, errStr := string(stdoutBuf.Bytes()), string(stderrBuf.Bytes()) - fmt.Printf("\nout:\n%s\nerr:\n%s\n", outStr, errStr) } diff --git a/templates/commit0/commit0.tmpl b/templates/commit0/commit0.tmpl index ad05ac0bd..d0afd1ef0 100644 --- a/templates/commit0/commit0.tmpl +++ b/templates/commit0/commit0.tmpl @@ -7,6 +7,12 @@ maintainers: # - name: bob # email: bob@test.com +kubernetes: + clusterName: staging + deploy: true + awsAccountId: 1234 + awsRegion: us-east-1 + network: grpc: host: 0.0.0.0 diff --git a/templates/kubernetes/terraform/modules/eks/main.tf b/templates/kubernetes/terraform/modules/eks/main.tf index e84735869..2594c5a1f 100644 --- a/templates/kubernetes/terraform/modules/eks/main.tf +++ b/templates/kubernetes/terraform/modules/eks/main.tf @@ -9,7 +9,7 @@ module "eks" { source = "terraform-aws-modules/eks/aws" version = "6.0.2" - cluster_name = "${var.project}" + cluster_name = var.project cluster_version = "1.14" subnets = var.private_subnets vpc_id = var.vpc_id @@ -44,4 +44,4 @@ module "eks" { tags = { environment = var.environment } -} \ No newline at end of file +}