Merge pull request #76 from commitdev/add-init-and-apply-flags

Various fixes to applying kube tf and added flags

cmd/generate.go

@@ -10,10 +10,13 @@ import (
 )
 
 var configPath string
+var runInit bool
+var runApply bool
 
 func init() {
-
 	generateCmd.PersistentFlags().StringVarP(&configPath, "config", "c", util.CommitYml, "config path")
+	generateCmd.PersistentFlags().BoolVarP(&runInit, "init", "", false, "Initialize config after generating")
+	generateCmd.PersistentFlags().BoolVarP(&runApply, "apply", "", false, "Apply config after generating")
 
 	rootCmd.AddCommand(generateCmd)
 }
@@ -29,7 +32,7 @@ var generateCmd = &cobra.Command{
 		cfg := config.LoadConfig(configPath)
 		cfg.Print()
 
-		generate.GenerateArtifactsHelper(t, cfg, "")
+		generate.GenerateArtifactsHelper(t, cfg, "", runInit, runApply)
 
 	},
 }

internal/api/create_project.go

@@ -49,5 +49,5 @@ func GenerateArtifacts(projectConfig util.ProjectConfiguration) {
 	cfg := config.LoadConfig(generatedYml)
 	cfg.Print()
 
-	generate.GenerateArtifactsHelper(t, cfg, projectConfig.ProjectName)
+	generate.GenerateArtifactsHelper(t, cfg, projectConfig.ProjectName, false, false)
 }

internal/config/config.go

@@ -78,7 +78,7 @@ type aws struct {
 	Region    string
 	EKS       eks
 	Cognito   cognito
-	S3Hosting s3Hosting `yaml:"s3_hosting"`
+	S3Hosting s3Hosting `yaml:"s3Hosting"`
 	Terraform terraform
 }
 
@@ -87,17 +87,16 @@ type terraform struct {
 }
 
 type cognito struct {
-	Deploy bool
+	Enabled bool
 }
 
 type s3Hosting struct {
-	Deploy bool
+	Enabled bool // @TODO Remove this option? If they have frontend and AWS enabled we should set up S3, etc.
 }
 
 type eks struct {
 	ClusterName string `yaml:"clusterName"`
 	WorkerAMI   string `yaml:"workerAMI"`
-	Deploy      bool
 }
 
 func LoadConfig(filePath string) *Commit0Config {

internal/generate/generate_helper.go

@@ -16,7 +16,7 @@ import (
 	"github.com/logrusorgru/aurora"
 )
 
-func GenerateArtifactsHelper(t *templator.Templator, cfg *config.Commit0Config, pathPrefix string) {
+func GenerateArtifactsHelper(t *templator.Templator, cfg *config.Commit0Config, pathPrefix string, runInit bool, runApply bool) {
 	var wg sync.WaitGroup
 	if !util.ValidateLanguage(cfg.Frontend.Framework) {
 		log.Fatalln(aurora.Red(emoji.Sprintf(":exclamation: '%s' is not a supported framework.", cfg.Frontend.Framework)))
@@ -50,21 +50,15 @@ func GenerateArtifactsHelper(t *templator.Templator, cfg *config.Commit0Config,
 	// Wait for all the templates to be generated
 	wg.Wait()
 
-	log.Println("Executing commands")
-	// @TODO : Move this stuff to another command? Or genericize it a bit.
-	if cfg.Infrastructure.AWS.EKS.Deploy {
-		terraform.Execute(cfg, pathPrefix)
-		kubernetes.Execute(cfg, pathPrefix)
+	log.Println(aurora.Cyan(emoji.Sprintf("Initializing Infrastructure")))
+	if cfg.Infrastructure.AWS.EKS.ClusterName != "" && runInit {
+		terraform.Init(cfg, pathPrefix)
 	}
 
-	if cfg.Infrastructure.AWS.Cognito.Deploy {
-		outputs := []string{
-			"cognito_pool_id",
-			"cognito_client_id",
-		}
-		outputValues := terraform.GetOutputs(cfg, pathPrefix, outputs)
-		cfg.Frontend.Env.CognitoPoolID = outputValues["cognito_pool_id"]
-		cfg.Frontend.Env.CognitoClientID = outputValues["cognito_client_id"]
+	log.Println(aurora.Cyan(emoji.Sprintf("Creating Infrastructure")))
+	if cfg.Infrastructure.AWS.EKS.ClusterName != "" && runApply {
+		terraform.Execute(cfg, pathPrefix)
+		kubernetes.Execute(cfg, pathPrefix)
 	}
 
 	// @TODO : This strucuture probably needs to be adjusted. Probably too generic.

internal/generate/kubernetes/generate.go

@@ -9,6 +9,7 @@ import (
 	"github.com/commitdev/commit0/internal/config"
 	"github.com/commitdev/commit0/internal/templator"
 	"github.com/commitdev/commit0/internal/util"
+	"github.com/kyokomi/emoji"
 	"github.com/logrusorgru/aurora"
 )
 
@@ -19,14 +20,12 @@ func Generate(t *templator.Templator, cfg *config.Commit0Config, wg *sync.WaitGr
 }
 
 // Execute terrafrom init & plan
-func Execute(config *config.Commit0Config, pathPrefix string) {
-	if config.Infrastructure.AWS.EKS.Deploy {
-		envars := util.MakeAwsEnvars(util.GetSecrets())
+func Execute(cfg *config.Commit0Config, pathPrefix string) {
+	envars := util.MakeAwsEnvars(util.GetSecrets())
 
-		pathPrefix = filepath.Join(pathPrefix, "kubernetes/terraform")
+	pathPrefix = filepath.Join(pathPrefix, "kubernetes/terraform")
 
-		log.Println(aurora.Cyan(":alarm_clock: Applying kubernetes configuration..."))
-		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
-		util.ExecuteCommand(exec.Command("terraform", "plan"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
-	}
+	log.Println(aurora.Cyan(emoji.Sprintf(":alarm_clock: Applying kubernetes configuration...")))
+	util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging"), envars)
+	util.ExecuteCommand(exec.Command("terraform", "apply", "-auto-approve"), filepath.Join(pathPrefix, "environments/staging"), envars)
 }

internal/generate/terraform/generate.go

@@ -37,25 +37,26 @@ func Generate(t *templator.Templator, cfg *config.Commit0Config, wg *sync.WaitGr
 	t.Terraform.TemplateFiles(data, false, wg, pathPrefix)
 }
 
+// GetOutputs captures the terraform output for the specific variables
 func GetOutputs(config *config.Commit0Config, pathPrefix string, outputs []string) map[string]string {
 	outputsMap := make(map[string]string)
 
 	log.Println("Preparing aws environment...")
 
 	envars := util.MakeAwsEnvars(util.GetSecrets())
 
-	path := filepath.Join(pathPrefix, "terraform")
+	pathPrefix = filepath.Join(pathPrefix, "environments/staging")
 
 	for _, output := range outputs {
-		outputValue := util.ExecuteCommandOutput(exec.Command("terraform", "output", output), path, envars)
+		outputValue := util.ExecuteCommandOutput(exec.Command("terraform", "output", output), pathPrefix, envars)
 		outputsMap[output] = outputValue
 	}
 
 	return outputsMap
 }
 
-// Execute terrafrom init & plan
-func Execute(config *config.Commit0Config, pathPrefix string) {
+// Init sets up anything required by Execute
+func Init(config *config.Commit0Config, pathPrefix string) {
 	// @TODO : Change this check. Most likely we should discover the accountid
 	if config.Infrastructure.AWS.AccountId != "" {
 		log.Println("Preparing aws environment...")
@@ -68,16 +69,31 @@ func Execute(config *config.Commit0Config, pathPrefix string) {
 		log.Println(aurora.Cyan(emoji.Sprintf(":alarm_clock: Initializing remote backend...")))
 		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "bootstrap/remote-state"), envars)
 		util.ExecuteCommand(exec.Command("terraform", "apply", "-auto-approve"), filepath.Join(pathPrefix, "bootstrap/remote-state"), envars)
+	}
+}
 
-		log.Println(aurora.Cyan(":alarm_clock: Planning infrastructure..."))
-		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging"), envars)
-		util.ExecuteCommand(exec.Command("terraform", "plan"), filepath.Join(pathPrefix, "environments/staging"), envars)
+// Execute terrafrom init & plan. May modify the config passed in
+func Execute(cfg *config.Commit0Config, pathPrefix string) {
+	// @TODO : Change this check. Most likely we should discover the accountid
+	if cfg.Infrastructure.AWS.AccountId != "" {
+		log.Println("Preparing aws environment...")
 
-		log.Println(aurora.Cyan(":alarm_clock: Applying infrastructure configuration..."))
-		util.ExecuteCommand(exec.Command("terraform", "apply"), filepath.Join(pathPrefix, "environments/staging"), envars)
+		envars := util.MakeAwsEnvars(util.GetSecrets())
+
+		pathPrefix = filepath.Join(pathPrefix, "terraform")
 
-		log.Println(aurora.Cyan(":alarm_clock: Applying kubernetes configuration..."))
-		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
-		util.ExecuteCommand(exec.Command("terraform", "plan"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
+		log.Println(aurora.Cyan(emoji.Sprintf(":alarm_clock: Applying infrastructure configuration...")))
+		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging"), envars)
+		util.ExecuteCommand(exec.Command("terraform", "apply", "-auto-approve"), filepath.Join(pathPrefix, "environments/staging"), envars)
+
+		if cfg.Infrastructure.AWS.Cognito.Enabled {
+			outputs := []string{
+				"cognito_pool_id",
+				"cognito_client_id",
+			}
+			outputValues := GetOutputs(cfg, pathPrefix, outputs)
+			cfg.Frontend.Env.CognitoPoolID = outputValues["cognito_pool_id"]
+			cfg.Frontend.Env.CognitoClientID = outputValues["cognito_client_id"]
+		}
 	}
 }

internal/util/util.go

@@ -136,7 +136,7 @@ func ExecuteCommandOutput(cmd *exec.Cmd, pathPrefix string, envars []string) str
 
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		log.Fatalf("Executing terraform output failed: %v\n", err)
+		log.Fatalf("Executing command failed: (%v) %s\n", err, out)
 	}
 	return string(out)
 }

templates/commit0/commit0.tmpl

@@ -13,11 +13,10 @@ infrastructure:
     region: us-east-1
     eks:
       clusterName: staging
-      deploy: true
     cognito:
-      deploy: true
-    s3_hosting:
-      deploy: true
+      enabled: true
+    s3Hosting:
+      enabled: true
 
 frontend:
   framework: {{.FrontendFramework}}

templates/kubernetes/terraform/modules/kubernetes/ingress/main.tf

@@ -22,6 +22,7 @@ resource "kubernetes_config_map" "nginx_configuration" {
     use-forwarded-headers = "true",
     use-proxy-protocol    = "false"
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 # resource "kubernetes_config_map" "nginx_configuration" {
@@ -33,6 +34,7 @@ resource "kubernetes_config_map" "nginx_configuration" {
 #       "app.kubernetes.io/part-of" = "ingress-nginx"
 #     }
 #   }
+#  depends_on = [kubernetes_namespace.ingress_nginx]
 # }
 
 resource "kubernetes_config_map" "tcp_services" {
@@ -44,6 +46,7 @@ resource "kubernetes_config_map" "tcp_services" {
       "app.kubernetes.io/part-of" = "ingress-nginx"
     }
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_config_map" "udp_services" {
@@ -55,6 +58,7 @@ resource "kubernetes_config_map" "udp_services" {
       "app.kubernetes.io/part-of" = "ingress-nginx"
     }
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_service_account" "nginx_ingress_serviceaccount" {
@@ -66,6 +70,7 @@ resource "kubernetes_service_account" "nginx_ingress_serviceaccount" {
       "app.kubernetes.io/part-of" = "ingress-nginx"
     }
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_cluster_role" "nginx_ingress_clusterrole" {
@@ -138,6 +143,7 @@ resource "kubernetes_role" "nginx_ingress_role" {
     api_groups = [""]
     resources  = ["endpoints"]
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_role_binding" "nginx_ingress_role_nisa_binding" {
@@ -159,6 +165,7 @@ resource "kubernetes_role_binding" "nginx_ingress_role_nisa_binding" {
     kind      = "Role"
     name      = "nginx-ingress-role"
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_cluster_role_binding" "nginx_ingress_clusterrole_nisa_binding" {
@@ -324,6 +331,7 @@ resource "kubernetes_service" "ingress_nginx" {
     type                    = "LoadBalancer"
     external_traffic_policy = "Local"
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 # HTTPS Load balancer
@@ -359,4 +367,5 @@ resource "kubernetes_service" "ingress_nginx" {
 #     }
 #     type = "LoadBalancer"
 #   }
-# }
+#  depends_on = [kubernetes_namespace.ingress_nginx]
+# }

templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent/main.tf → templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf

@@ -1,12 +1,9 @@
-data "aws_iam_policy" "CloudWatchAgentServerPolicy" {
-  arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
-}
-
 resource "kubernetes_service_account" "cloudwatch_agent" {
   metadata {
     name      = "cloudwatch-agent"
     namespace = "amazon-cloudwatch"
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_cluster_role" "cloudwatch_agent_role" {
@@ -76,6 +73,7 @@ resource "kubernetes_config_map" "cwagentconfig" {
       }
     )
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_daemonset" "cloudwatch_agent" {
@@ -217,4 +215,4 @@ resource "kubernetes_daemonset" "cloudwatch_agent" {
       }
     }
   }
-}
+}

templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/main.tf → templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf

@@ -23,13 +23,15 @@ resource "kubernetes_config_map" "cluster_info" {
     "cluster.name" = var.cluster_name
     "logs.region"  = var.region
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_service_account" "fluentd" {
   metadata {
     name      = "fluentd"
     namespace = "amazon-cloudwatch"
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_cluster_role" "fluentd_role" {
@@ -57,6 +59,7 @@ resource "kubernetes_cluster_role_binding" "fluentd_role_binding" {
     kind      = "ClusterRole"
     name      = "fluentd-role"
   }
+  depends_on = [kubernetes_service_account.fluentd]
 }
 
 resource "kubernetes_config_map" "fluentd_config" {
@@ -71,6 +74,7 @@ resource "kubernetes_config_map" "fluentd_config" {
     "host.conf"       = data.local_file.host.content
     "systemd.conf"    = data.local_file.systemd.content
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_daemonset" "fluentd_cloudwatch" {
@@ -217,4 +221,4 @@ resource "kubernetes_daemonset" "fluentd_cloudwatch" {
       }
     }
   }
-}
+}