Various fixes to applying kube tf and added flags

Added --init and --apply flags to generate command, removed "Deploy" config
Added dependencies to kubernetes terraform to prevent errors when creating namespaces
Merged fluentd and cloudwatch agent modules into monitoring, otherwise dependencies wouldn't work

cmd/generate.go

@@ -10,10 +10,13 @@ import (
 )
 
 var configPath string
+var runInit bool
+var runApply bool
 
 func init() {
-
 	generateCmd.PersistentFlags().StringVarP(&configPath, "config", "c", util.CommitYml, "config path")
+	generateCmd.PersistentFlags().BoolVarP(&runInit, "init", "", false, "Initialize config after generating")
+	generateCmd.PersistentFlags().BoolVarP(&runApply, "apply", "", false, "Apply config after generating")
 
 	rootCmd.AddCommand(generateCmd)
 }
@@ -29,7 +32,7 @@ var generateCmd = &cobra.Command{
 		cfg := config.LoadConfig(configPath)
 		cfg.Print()
 
-		generate.GenerateArtifactsHelper(t, cfg, "")
+		generate.GenerateArtifactsHelper(t, cfg, "", runInit, runApply)
 
 	},
 }

internal/api/create_project.go

@@ -49,5 +49,5 @@ func GenerateArtifacts(projectConfig util.ProjectConfiguration) {
 	cfg := config.LoadConfig(generatedYml)
 	cfg.Print()
 
-	generate.GenerateArtifactsHelper(t, cfg, projectConfig.ProjectName)
+	generate.GenerateArtifactsHelper(t, cfg, projectConfig.ProjectName, false, false)
 }

internal/config/config.go

@@ -87,17 +87,16 @@ type terraform struct {
 }
 
 type cognito struct {
-	Deploy bool
+	Enabled bool
 }
 
 type s3Hosting struct {
-	Deploy bool
+	Enabled bool // @TODO Remove this option? If they have frontend and AWS enabled we should set up S3, etc.
 }
 
 type eks struct {
 	ClusterName string `yaml:"clusterName"`
 	WorkerAMI   string `yaml:"workerAMI"`
-	Deploy      bool
 }
 
 func LoadConfig(filePath string) *Commit0Config {

internal/generate/generate_helper.go

@@ -16,7 +16,7 @@ import (
 	"github.com/logrusorgru/aurora"
 )
 
-func GenerateArtifactsHelper(t *templator.Templator, cfg *config.Commit0Config, pathPrefix string) {
+func GenerateArtifactsHelper(t *templator.Templator, cfg *config.Commit0Config, pathPrefix string, runInit bool, runApply bool) {
 	var wg sync.WaitGroup
 	if !util.ValidateLanguage(cfg.Frontend.Framework) {
 		log.Fatalln(aurora.Red(emoji.Sprintf(":exclamation: '%s' is not a supported framework.", cfg.Frontend.Framework)))
@@ -50,21 +50,15 @@ func GenerateArtifactsHelper(t *templator.Templator, cfg *config.Commit0Config,
 	// Wait for all the templates to be generated
 	wg.Wait()
 
-	log.Println("Executing commands")
-	// @TODO : Move this stuff to another command? Or genericize it a bit.
-	if cfg.Infrastructure.AWS.EKS.Deploy {
-		terraform.Execute(cfg, pathPrefix)
-		kubernetes.Execute(cfg, pathPrefix)
+	log.Println(aurora.Cyan(emoji.Sprintf("Initializing Infrastructure")))
+	if cfg.Infrastructure.AWS.EKS.ClusterName != "" && runInit {
+		terraform.Init(cfg, pathPrefix)
 	}
 
-	if cfg.Infrastructure.AWS.Cognito.Deploy {
-		outputs := []string{
-			"cognito_pool_id",
-			"cognito_client_id",
-		}
-		outputValues := terraform.GetOutputs(cfg, pathPrefix, outputs)
-		cfg.Frontend.Env.CognitoPoolID = outputValues["cognito_pool_id"]
-		cfg.Frontend.Env.CognitoClientID = outputValues["cognito_client_id"]
+	log.Println(aurora.Cyan(emoji.Sprintf("Creating Infrastructure")))
+	if cfg.Infrastructure.AWS.EKS.ClusterName != "" && runApply {
+		terraform.Execute(cfg, pathPrefix)
+		kubernetes.Execute(cfg, pathPrefix)
 	}
 
 	// @TODO : This strucuture probably needs to be adjusted. Probably too generic.

internal/generate/kubernetes/generate.go

@@ -19,14 +19,12 @@ func Generate(t *templator.Templator, cfg *config.Commit0Config, wg *sync.WaitGr
 }
 
 // Execute terrafrom init & plan
-func Execute(config *config.Commit0Config, pathPrefix string) {
-	if config.Infrastructure.AWS.EKS.Deploy {
-		envars := util.MakeAwsEnvars(util.GetSecrets())
+func Execute(cfg *config.Commit0Config, pathPrefix string) {
+	envars := util.MakeAwsEnvars(util.GetSecrets())
 
-		pathPrefix = filepath.Join(pathPrefix, "kubernetes/terraform")
+	pathPrefix = filepath.Join(pathPrefix, "kubernetes/terraform")
 
-		log.Println(aurora.Cyan(":alarm_clock: Applying kubernetes configuration..."))
-		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
-		util.ExecuteCommand(exec.Command("terraform", "plan"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
-	}
+	log.Println(aurora.Cyan(":alarm_clock: Applying kubernetes configuration..."))
+	util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging"), envars)
+	util.ExecuteCommand(exec.Command("terraform", "apply", "-auto-approve"), filepath.Join(pathPrefix, "environments/staging"), envars)
 }

internal/generate/terraform/generate.go

@@ -37,6 +37,7 @@ func Generate(t *templator.Templator, cfg *config.Commit0Config, wg *sync.WaitGr
 	t.Terraform.TemplateFiles(data, false, wg, pathPrefix)
 }
 
+// GetOutputs captures the terraform output for the specific variables
 func GetOutputs(config *config.Commit0Config, pathPrefix string, outputs []string) map[string]string {
 	outputsMap := make(map[string]string)
 
@@ -54,8 +55,8 @@ func GetOutputs(config *config.Commit0Config, pathPrefix string, outputs []strin
 	return outputsMap
 }
 
-// Execute terrafrom init & plan
-func Execute(config *config.Commit0Config, pathPrefix string) {
+// Init sets up anything required by Execute
+func Init(config *config.Commit0Config, pathPrefix string) {
 	// @TODO : Change this check. Most likely we should discover the accountid
 	if config.Infrastructure.AWS.AccountId != "" {
 		log.Println("Preparing aws environment...")
@@ -68,16 +69,31 @@ func Execute(config *config.Commit0Config, pathPrefix string) {
 		log.Println(aurora.Cyan(emoji.Sprintf(":alarm_clock: Initializing remote backend...")))
 		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "bootstrap/remote-state"), envars)
 		util.ExecuteCommand(exec.Command("terraform", "apply", "-auto-approve"), filepath.Join(pathPrefix, "bootstrap/remote-state"), envars)
+	}
+}
 
-		log.Println(aurora.Cyan(":alarm_clock: Planning infrastructure..."))
-		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging"), envars)
-		util.ExecuteCommand(exec.Command("terraform", "plan"), filepath.Join(pathPrefix, "environments/staging"), envars)
+// Execute terrafrom init & plan. May modify the config passed in
+func Execute(cfg *config.Commit0Config, pathPrefix string) {
+	// @TODO : Change this check. Most likely we should discover the accountid
+	if cfg.Infrastructure.AWS.AccountId != "" {
+		log.Println("Preparing aws environment...")
 
-		log.Println(aurora.Cyan(":alarm_clock: Applying infrastructure configuration..."))
-		util.ExecuteCommand(exec.Command("terraform", "apply"), filepath.Join(pathPrefix, "environments/staging"), envars)
+		envars := util.MakeAwsEnvars(util.GetSecrets())
+
+		pathPrefix = filepath.Join(pathPrefix, "terraform")
 
-		log.Println(aurora.Cyan(":alarm_clock: Applying kubernetes configuration..."))
-		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
-		util.ExecuteCommand(exec.Command("terraform", "plan"), filepath.Join(pathPrefix, "environments/staging/kubernetes"), envars)
+		log.Println(aurora.Cyan(":alarm_clock: Applying infrastructure configuration..."))
+		util.ExecuteCommand(exec.Command("terraform", "init"), filepath.Join(pathPrefix, "environments/staging"), envars)
+		util.ExecuteCommand(exec.Command("terraform", "apply", "-auto-approve"), filepath.Join(pathPrefix, "environments/staging"), envars)
+
+		if cfg.Infrastructure.AWS.Cognito.Enabled {
+			outputs := []string{
+				"cognito_pool_id",
+				"cognito_client_id",
+			}
+			outputValues := GetOutputs(cfg, pathPrefix, outputs)
+			cfg.Frontend.Env.CognitoPoolID = outputValues["cognito_pool_id"]
+			cfg.Frontend.Env.CognitoClientID = outputValues["cognito_client_id"]
+		}
 	}
 }

templates/commit0/commit0.tmpl

@@ -13,11 +13,14 @@ infrastructure:
     region: us-east-1
     eks:
       clusterName: staging
+<<<<<<< HEAD
       deploy: true
     cognito:
       deploy: true
     s3_hosting:
       deploy: true
+=======
+>>>>>>> Various fixes to applying kube tf and added flags
 
 frontend:
   framework: {{.FrontendFramework}}

templates/kubernetes/terraform/modules/kubernetes/ingress/main.tf

@@ -22,6 +22,7 @@ resource "kubernetes_config_map" "nginx_configuration" {
     use-forwarded-headers = "true",
     use-proxy-protocol    = "false"
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 # resource "kubernetes_config_map" "nginx_configuration" {
@@ -33,6 +34,7 @@ resource "kubernetes_config_map" "nginx_configuration" {
 #       "app.kubernetes.io/part-of" = "ingress-nginx"
 #     }
 #   }
+#  depends_on = [kubernetes_namespace.ingress_nginx]
 # }
 
 resource "kubernetes_config_map" "tcp_services" {
@@ -44,6 +46,7 @@ resource "kubernetes_config_map" "tcp_services" {
       "app.kubernetes.io/part-of" = "ingress-nginx"
     }
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_config_map" "udp_services" {
@@ -55,6 +58,7 @@ resource "kubernetes_config_map" "udp_services" {
       "app.kubernetes.io/part-of" = "ingress-nginx"
     }
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_service_account" "nginx_ingress_serviceaccount" {
@@ -66,6 +70,7 @@ resource "kubernetes_service_account" "nginx_ingress_serviceaccount" {
       "app.kubernetes.io/part-of" = "ingress-nginx"
     }
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_cluster_role" "nginx_ingress_clusterrole" {
@@ -138,6 +143,7 @@ resource "kubernetes_role" "nginx_ingress_role" {
     api_groups = [""]
     resources  = ["endpoints"]
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_role_binding" "nginx_ingress_role_nisa_binding" {
@@ -159,6 +165,7 @@ resource "kubernetes_role_binding" "nginx_ingress_role_nisa_binding" {
     kind      = "Role"
     name      = "nginx-ingress-role"
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 resource "kubernetes_cluster_role_binding" "nginx_ingress_clusterrole_nisa_binding" {
@@ -324,6 +331,7 @@ resource "kubernetes_service" "ingress_nginx" {
     type                    = "LoadBalancer"
     external_traffic_policy = "Local"
   }
+  depends_on = [kubernetes_namespace.ingress_nginx]
 }
 
 # HTTPS Load balancer
@@ -359,4 +367,5 @@ resource "kubernetes_service" "ingress_nginx" {
 #     }
 #     type = "LoadBalancer"
 #   }
-# }
+#  depends_on = [kubernetes_namespace.ingress_nginx]
+# }

templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent/main.tf → templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf

@@ -1,12 +1,9 @@
-data "aws_iam_policy" "CloudWatchAgentServerPolicy" {
-  arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
-}
-
 resource "kubernetes_service_account" "cloudwatch_agent" {
   metadata {
     name      = "cloudwatch-agent"
     namespace = "amazon-cloudwatch"
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_cluster_role" "cloudwatch_agent_role" {
@@ -76,6 +73,7 @@ resource "kubernetes_config_map" "cwagentconfig" {
       }
     )
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_daemonset" "cloudwatch_agent" {
@@ -217,4 +215,4 @@ resource "kubernetes_daemonset" "cloudwatch_agent" {
       }
     }
   }
-}
+}

templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd/main.tf → templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf

@@ -23,13 +23,15 @@ resource "kubernetes_config_map" "cluster_info" {
     "cluster.name" = var.cluster_name
     "logs.region"  = var.region
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_service_account" "fluentd" {
   metadata {
     name      = "fluentd"
     namespace = "amazon-cloudwatch"
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_cluster_role" "fluentd_role" {
@@ -57,6 +59,7 @@ resource "kubernetes_cluster_role_binding" "fluentd_role_binding" {
     kind      = "ClusterRole"
     name      = "fluentd-role"
   }
+  depends_on = [kubernetes_service_account.fluentd]
 }
 
 resource "kubernetes_config_map" "fluentd_config" {
@@ -71,6 +74,7 @@ resource "kubernetes_config_map" "fluentd_config" {
     "host.conf"       = data.local_file.host.content
     "systemd.conf"    = data.local_file.systemd.content
   }
+  depends_on = [kubernetes_namespace.amazon_cloudwatch]
 }
 
 resource "kubernetes_daemonset" "fluentd_cloudwatch" {
@@ -217,4 +221,4 @@ resource "kubernetes_daemonset" "fluentd_cloudwatch" {
       }
     }
   }
-}
+}

templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf

@@ -22,17 +22,3 @@ resource "aws_iam_role_policy_attachment" "k8s_monitoring_role_policy" {
   role       = "${aws_iam_role.k8s_monitoring.id}"
   policy_arn = "${data.aws_iam_policy.CloudWatchAgentServerPolicy.arn}"
 }
-
-module "fluentd" {
-  source       = "./fluentd"
-  environment  = var.environment
-  region       = var.region
-  cluster_name = var.cluster_name
-}
-
-module "cloudwatch_agent" {
-  source       = "./cloudwatch_agent"
-  environment  = var.environment
-  region       = var.region
-  cluster_name = var.cluster_name
-}

templates/terraform/modules/environment/main.tf

@@ -43,14 +43,14 @@ module "kube2iam" {
   iam_account_id           = data.aws_caller_identity.current.account_id
 }
 
-# {{ if .Config.Infrastructure.AWS.Cognito.Deploy }}
+# {{ if .Config.Infrastructure.AWS.Cognito.Enabled }}
 resource "cognito" "auth" {
   user_pool   = var.user_pool
   hostname    = var.hostname
 }
 # {{- end}}
 
-# {{ if .Config.Infrastructure.AWS.S3Hosting.Deploy }}
+# {{ if .Config.Infrastructure.AWS.S3Hosting.Enabled }}
 resource "s3_hosting" "assets" {
   bucket_name   = var.s3_hosting_bucket_name
 }