From 4d97868e2542f3875b8105a504b2c2c8960d358f Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Sun, 15 Oct 2023 21:08:19 -0400 Subject: [PATCH 1/2] initial work argo-workflows Signed-off-by: Carlos Santana --- .../examples/eks/argo-wf-ingress/README.md | 42 +++ .../eks/argo-wf-ingress/bootstrap/addons.yaml | 32 ++ .../argo-wf-ingress/bootstrap/workloads.yaml | 19 ++ .../examples/eks/argo-wf-ingress/destroy.sh | 16 + .../examples/eks/argo-wf-ingress/main.tf | 291 ++++++++++++++++++ .../examples/eks/argo-wf-ingress/outputs.tf | 33 ++ .../examples/eks/argo-wf-ingress/variables.tf | 24 ++ .../examples/eks/argo-wf-ingress/versions.tf | 25 ++ 8 files changed, 482 insertions(+) create mode 100644 argocd/iac/terraform/examples/eks/argo-wf-ingress/README.md create mode 100644 argocd/iac/terraform/examples/eks/argo-wf-ingress/bootstrap/addons.yaml create mode 100644 argocd/iac/terraform/examples/eks/argo-wf-ingress/bootstrap/workloads.yaml create mode 100755 argocd/iac/terraform/examples/eks/argo-wf-ingress/destroy.sh create mode 100644 argocd/iac/terraform/examples/eks/argo-wf-ingress/main.tf create mode 100644 argocd/iac/terraform/examples/eks/argo-wf-ingress/outputs.tf create mode 100644 argocd/iac/terraform/examples/eks/argo-wf-ingress/variables.tf create mode 100644 argocd/iac/terraform/examples/eks/argo-wf-ingress/versions.tf diff --git a/argocd/iac/terraform/examples/eks/argo-wf-ingress/README.md b/argocd/iac/terraform/examples/eks/argo-wf-ingress/README.md new file mode 100644 index 00000000..af13411a --- /dev/null +++ b/argocd/iac/terraform/examples/eks/argo-wf-ingress/README.md @@ -0,0 +1,42 @@ +# Argo Workflows with ingress domain name + +Example on how to deploy Amazon EKS with addons configured via ArgoCD. +In this example the ArgoCD is configured with ingress using a https domain name managed on Route53 + + +**Create DNS Hosted Zone in Route 53:** + +In this step you will delegate your registered domain DNS to Amazon Route53. You can either delegate the top level domain or a subdomain. +```shell +export TF_VAR_domain_name= # For example: example.com or subdomain.example.com +``` + +You can use the Console, or the `aws` cli to create a hosted zone. Execute the following command only once: +```sh +aws route53 create-hosted-zone --name $TF_VAR_domain_name --caller-reference "$(date)" +``` +Use the NameServers in the DelegatoinSet to update your registered domain NS records at the registrar. + + +After creating the Route53 zone deploy the EKS Cluster +```shell +terraform init +terraform apply +``` + +Access Terraform output to configure `kubectl` and `argocd` +```shell +terraform output +``` + +To access ArgoCD thru ingress https use the following command to get URL and passwords +```shell +echo "URL: https://$(kubectl get ing -n argocd argo-cd-argocd-server -o jsonpath='{.spec.tls[0].hosts[0]}')" +echo "Username: admin" +echo "Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" +``` + +Destroy EKS Cluster +```shell +./destroy.sh +``` diff --git a/argocd/iac/terraform/examples/eks/argo-wf-ingress/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/argo-wf-ingress/bootstrap/addons.yaml new file mode 100644 index 00000000..02535923 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/argo-wf-ingress/bootstrap/addons.yaml @@ -0,0 +1,32 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: bootstrap-addons + namespace: argocd +spec: + syncPolicy: + preserveResourcesOnDeletion: true + generators: + - clusters: + selector: + matchExpressions: + - key: akuity.io/argo-cd-cluster-name + operator: NotIn + values: [in-cluster] + template: + metadata: + name: 'bootstrap-addons' + spec: + project: default + source: + repoURL: '{{metadata.annotations.addons_repo_url}}' + path: '{{metadata.annotations.addons_repo_basepath}}{{metadata.annotations.addons_repo_path}}' + targetRevision: '{{metadata.annotations.addons_repo_revision}}' + directory: + recurse: true + exclude: exclude/* + destination: + namespace: 'argocd' + name: '{{name}}' + syncPolicy: + automated: {} diff --git a/argocd/iac/terraform/examples/eks/argo-wf-ingress/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/argo-wf-ingress/bootstrap/workloads.yaml new file mode 100644 index 00000000..c9e5de21 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/argo-wf-ingress/bootstrap/workloads.yaml @@ -0,0 +1,19 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: workloads + namespace: 'argocd' + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + destination: + server: https://kubernetes.default.svc + namespace: 'guestbook' + project: default + source: + path: helm-guestbook + repoURL: https://github.com/argoproj/argocd-example-apps + syncPolicy: + automated: {} + syncOptions: + - CreateNamespace=true \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/argo-wf-ingress/destroy.sh b/argocd/iac/terraform/examples/eks/argo-wf-ingress/destroy.sh new file mode 100755 index 00000000..cd8a9477 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/argo-wf-ingress/destroy.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -x + +# Delete the Ingress/SVC before removing the addons +TMPFILE=$(mktemp) +terraform output -raw configure_kubectl > "$TMPFILE" +source "$TMPFILE" + +kubectl delete ing -n argocd argo-cd-argocd-server + +terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -target="module.eks" -auto-approve +terraform destroy -target="module.vpc" -auto-approve +terraform destroy -auto-approve diff --git a/argocd/iac/terraform/examples/eks/argo-wf-ingress/main.tf b/argocd/iac/terraform/examples/eks/argo-wf-ingress/main.tf new file mode 100644 index 00000000..76b2b038 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/argo-wf-ingress/main.tf @@ -0,0 +1,291 @@ +provider "aws" { + region = local.region +} +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } + } +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } +} + +locals { + name = "ex-${replace(basename(path.cwd), "_", "-")}" + environment = "dev" + region = "us-west-2" + cluster_version = "1.27" + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" + gitops_addons_basepath = var.gitops_addons_basepath + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision + + enable_ingress = true + domain_private_zone = false + # change to a valid domain name you created a route53 zone + # aws route53 create-hosted-zone --name example.com --caller-reference "$(date)" + domain_name = var.domain_name + argocd_subdomain = "argocd" + argocd_host = "${local.argocd_subdomain}.${local.domain_name}" + argocd_domain_arn = try(data.aws_route53_zone.domain_name[0].arn, "") + argo_workflows_subdomain = "argoworkflows" + argo_workflows_host = "${local.argo_workflows_subdomain}.${local.domain_name}" + + + aws_addons = { + enable_cert_manager = true + #enable_aws_efs_csi_driver = true + #enable_aws_fsx_csi_driver = true + #enable_aws_cloudwatch_metrics = true + #enable_aws_privateca_issuer = true + #enable_cluster_autoscaler = true + enable_external_dns = true + #enable_external_secrets = true + enable_aws_load_balancer_controller = true + #enable_fargate_fluentbit = true + #enable_aws_for_fluentbit = true + #enable_aws_node_termination_handler = true + #enable_karpenter = true + #enable_velero = true + #enable_aws_gateway_api_controller = true + #enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi + #enable_aws_secrets_store_csi_driver_provider = true + enable_aws_argocd_ingress = true + enable_aws_argo_workflows_ingress = true + } + oss_addons = { + enable_argocd = false + #enable_argo_rollouts = true + #enable_argo_events = true + #enable_argo_workflows = true + #enable_cluster_proportional_autoscaler = true + #enable_gatekeeper = true + #enable_gpu_operator = true + #enable_ingress_nginx = true + #enable_kyverno = true + #enable_kube_prometheus_stack = true + #enable_metrics_server = true + #enable_prometheus_adapter = true + #enable_secrets_store_csi_driver = true + #enable_vpa = true + #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + } + addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }, { aws_cluster_name = module.eks.cluster_name }) + + addons_metadata = merge( + module.eks_blueprints_addons.gitops_metadata, + { + aws_cluster_name = module.eks.cluster_name + aws_region = local.region + aws_account_id = data.aws_caller_identity.current.account_id + aws_vpc_id = module.vpc.vpc_id + }, + { + argocd_hosts = "[${local.argocd_host}]" + argo_workflows_hosts = "[${local.argo_workflows_host}]" + external_dns_domain_filters = "[${local.domain_name}]" + external_dns_policy = "sync" + }, + { + addons_repo_url = local.gitops_addons_url + addons_repo_basepath = local.gitops_addons_basepath + addons_repo_path = local.gitops_addons_path + addons_repo_revision = local.gitops_addons_revision + } + ) + + argocd_apps = { + addons = file("${path.module}/bootstrap/addons.yaml") + workloads = file("${path.module}/bootstrap/workloads.yaml") + } + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Blueprint = local.name + GithubRepo = "github.com/csantanapr/terraform-gitops-bridge" + } +} + +################################################################################ +# GitOps Bridge: Bootstrap +################################################################################ +module "gitops_bridge_bootstrap" { + source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" + + cluster = { + cluster_name = module.eks.cluster_name + environment = local.environment + metadata = local.addons_metadata + addons = local.addons + } + apps = local.argocd_apps +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + # Using GitOps Bridge + create_kubernetes_resources = false + + # EKS Blueprints Addons + enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) + enable_external_dns = try(local.aws_addons.enable_external_dns, false) + enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(local.aws_addons.enable_karpenter, false) + enable_velero = try(local.aws_addons.enable_velero, false) + enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + + external_dns_route53_zone_arns = [local.argocd_domain_arn] # ArgoCD Server and UI domain name is registered in Route 53 + + tags = local.tags +} + +################################################################################ +# EKS Cluster +################################################################################ +#tfsec:ignore:aws-eks-enable-control-plane-logging +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.13" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 3 + max_size = 10 + desired_size = 3 + } + } + # EKS Addons + cluster_addons = { + vpc-cni = { + # Specify the VPC CNI addon should be deployed before compute to ensure + # the addon is configured before data plane compute resources are created + # See README for further details + before_compute = true + most_recent = true # To ensure access to the latest settings provided + configuration_values = jsonencode({ + env = { + # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) + } + } + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} + +################################################################################ +# Route 53 +################################################################################ +# To get the hosted zone to be use in argocd domain +data "aws_route53_zone" "domain_name" { + count = local.enable_ingress ? 1 : 0 + name = local.domain_name + private_zone = local.domain_private_zone +} + + +################################################################################ +# ACM Certificate +################################################################################ + +resource "aws_acm_certificate" "cert" { + count = local.enable_ingress ? 1 : 0 + domain_name = "*.${local.domain_name}" + validation_method = "DNS" +} + +resource "aws_route53_record" "cert" { + count = local.enable_ingress ? 1 : 0 + zone_id = data.aws_route53_zone.domain_name[0].zone_id + name = tolist(aws_acm_certificate.cert[0].domain_validation_options)[0].resource_record_name + type = tolist(aws_acm_certificate.cert[0].domain_validation_options)[0].resource_record_type + records = [tolist(aws_acm_certificate.cert[0].domain_validation_options)[0].resource_record_value] + ttl = 60 + allow_overwrite = true +} + +resource "aws_acm_certificate_validation" "cert" { + count = local.enable_ingress ? 1 : 0 + certificate_arn = aws_acm_certificate.cert[0].arn + validation_record_fqdns = [for record in aws_route53_record.cert : record.fqdn] +} diff --git a/argocd/iac/terraform/examples/eks/argo-wf-ingress/outputs.tf b/argocd/iac/terraform/examples/eks/argo-wf-ingress/outputs.tf new file mode 100644 index 00000000..77b4f836 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/argo-wf-ingress/outputs.tf @@ -0,0 +1,33 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + EOT +} + +output "configure_argocd" { + description = "Terminal Setup" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web" + kubectl config set-context --current --namespace argocd + argocd login --port-forward --username admin --password $(argocd admin initial-password | head -1) + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo Port Forward: http://localhost:8080 + kubectl port-forward -n argocd svc/argo-cd-argocd-server 8080:80 + EOT +} + +output "access_argocd" { + description = "ArgoCD Access" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + echo "ArgoCD URL: https://$(kubectl get ing -n argocd argo-cd-argocd-server -o jsonpath='{.spec.tls[0].hosts[0]}')" + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + EOT +} diff --git a/argocd/iac/terraform/examples/eks/argo-wf-ingress/variables.tf b/argocd/iac/terraform/examples/eks/argo-wf-ingress/variables.tf new file mode 100644 index 00000000..599d0b82 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/argo-wf-ingress/variables.tf @@ -0,0 +1,24 @@ +variable "domain_name" { + description = "Route 53 domain name" + type = string +} +variable "gitops_addons_org" { + description = "Git repository org/user contains for addons" + default = "https://github.com/gitops-bridge-dev" +} +variable "gitops_addons_repo" { + description = "Git repository contains for addons" + default = "gitops-bridge-argocd-control-plane-template" +} +variable "gitops_addons_basepath" { + description = "Git repository base path for addons" + default = "" +} +variable "gitops_addons_path" { + description = "Git repository path for addons" + default = "bootstrap/control-plane/addons" +} +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + default = "argo-workflows-ingress" +} diff --git a/argocd/iac/terraform/examples/eks/argo-wf-ingress/versions.tf b/argocd/iac/terraform/examples/eks/argo-wf-ingress/versions.tf new file mode 100644 index 00000000..2de60d58 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/argo-wf-ingress/versions.tf @@ -0,0 +1,25 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.67.0" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.10.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.22.0" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/ipv4-prefix-delegation/terraform.tfstate" + # } +} From 1557b4720382a21069e2e018a1800651fb524b22 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Mon, 16 Oct 2023 16:21:59 -0400 Subject: [PATCH 2/2] destroy Signed-off-by: Carlos Santana --- argocd/iac/terraform/examples/eks/argo-wf-ingress/destroy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/iac/terraform/examples/eks/argo-wf-ingress/destroy.sh b/argocd/iac/terraform/examples/eks/argo-wf-ingress/destroy.sh index cd8a9477..73b0fcd1 100755 --- a/argocd/iac/terraform/examples/eks/argo-wf-ingress/destroy.sh +++ b/argocd/iac/terraform/examples/eks/argo-wf-ingress/destroy.sh @@ -7,7 +7,7 @@ TMPFILE=$(mktemp) terraform output -raw configure_kubectl > "$TMPFILE" source "$TMPFILE" -kubectl delete ing -n argocd argo-cd-argocd-server +kubectl delete ing -A --all terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve terraform destroy -target="module.eks_blueprints_addons" -auto-approve