From ad2c4856ee18d46f9e8b7c99e2a98b2db9f9c988 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Fri, 1 Sep 2023 22:05:20 -0400 Subject: [PATCH 1/2] update external-secrets example Signed-off-by: Carlos Santana --- .../external-secrets/bootstrap/addons.yaml | 41 +++++--- .../external-secrets/bootstrap/workloads.yaml | 56 +++++++---- .../eks/external-secrets/k8s/Chart.yaml | 24 +++++ .../k8s/templates/secrets.yaml | 46 +++++++++ .../eks/external-secrets/k8s/values.yaml | 6 ++ .../examples/eks/external-secrets/main.tf | 95 ++++++++++++------- .../eks/external-secrets/variables.tf | 33 +++++-- 7 files changed, 233 insertions(+), 68 deletions(-) create mode 100644 argocd/iac/terraform/examples/eks/external-secrets/k8s/Chart.yaml create mode 100644 argocd/iac/terraform/examples/eks/external-secrets/k8s/templates/secrets.yaml create mode 100644 argocd/iac/terraform/examples/eks/external-secrets/k8s/values.yaml diff --git a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml index 61216a94..98ad3d93 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml @@ -1,19 +1,32 @@ +--- apiVersion: argoproj.io/v1alpha1 -kind: Application +kind: ApplicationSet metadata: name: bootstrap-addons - namespace: 'argocd' spec: - destination: - server: https://kubernetes.default.svc - namespace: 'argocd' - project: default - source: - path: ${path} - repoURL: ${repoURL} - targetRevision: ${targetRevision} - directory: - recurse: true - exclude: exclude/* syncPolicy: - automated: {} + preserveResourcesOnDeletion: true + generators: + - clusters: + selector: + matchExpressions: + - key: akuity.io/argo-cd-cluster-name + operator: NotIn + values: [in-cluster] + template: + metadata: + name: 'bootstrap-addons' + spec: + project: default + source: + repoURL: '{{metadata.annotations.gitops_bridge_repo_url}}' + path: '{{metadata.annotations.gitops_bridge_repo_path}}' + targetRevision: '{{metadata.annotations.gitops_bridge_repo_revision}}' + directory: + recurse: true + exclude: exclude/* + destination: + namespace: 'argocd' + name: '{{name}}' + syncPolicy: + automated: {} diff --git a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml index 1e7f70c8..caf2adbe 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml @@ -1,20 +1,44 @@ +--- apiVersion: argoproj.io/v1alpha1 -kind: Application +kind: ApplicationSet metadata: - name: bootstrap-workloads - namespace: 'argocd' - finalizers: - - resources-finalizer.argocd.argoproj.io + name: external-secrets-example spec: - destination: - server: https://kubernetes.default.svc - namespace: 'guestbook' - project: default - source: - path: helm-guestbook - repoURL: git@github.com:argoproj/argocd-example-apps.git - targetRevision: HEAD syncPolicy: - automated: {} - syncOptions: - - CreateNamespace=true \ No newline at end of file + preserveResourcesOnDeletion: true + generators: + - clusters: + selector: + matchExpressions: + - key: akuity.io/argo-cd-cluster-name + operator: NotIn + values: [in-cluster] + template: + metadata: + name: 'external-secrets-example' + spec: + project: default + source: + repoURL: '{{metadata.annotations.workload_repo_url}}' + path: '{{metadata.annotations.workload_repo_path}}' + targetRevision: '{{metadata.annotations.workload_repo_revision}}' + helm: + releaseName: 'external-secrets-example' + values: | + region: '{{metadata.annotations.aws_region}}' + externalSecret: + clusterSecretStore: + secret: '{{metadata.annotations.workload_sm_secret}}' + secretStore: + secret: '{{metadata.annotations.workload_pm_secret}}' + destination: + namespace: '{{metadata.annotations.external_secrets_namespace}}' + name: '{{name}}' + syncPolicy: + automated: + retry: + backoff: + duration: 1m + limit: 10 + syncOptions: + - CreateNamespace=true \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/external-secrets/k8s/Chart.yaml b/argocd/iac/terraform/examples/eks/external-secrets/k8s/Chart.yaml new file mode 100644 index 00000000..bfaee78d --- /dev/null +++ b/argocd/iac/terraform/examples/eks/external-secrets/k8s/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: resources +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.0.0" diff --git a/argocd/iac/terraform/examples/eks/external-secrets/k8s/templates/secrets.yaml b/argocd/iac/terraform/examples/eks/external-secrets/k8s/templates/secrets.yaml new file mode 100644 index 00000000..ac512a78 --- /dev/null +++ b/argocd/iac/terraform/examples/eks/external-secrets/k8s/templates/secrets.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: external-secrets.io/v1beta1 +kind: ClusterSecretStore +metadata: + name: cluster-secretstore-sm +spec: + provider: + aws: + service: SecretsManager + region: {{ .Values.region }} +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: external-secrets-sm +spec: + refreshInterval: 1h + secretStoreRef: + name: cluster-secretstore-sm + kind: ClusterSecretStore + dataFrom: + - extract: + key: {{ .Values.externalSecret.clusterSecretStore.secret }} +--- +apiVersion: external-secrets.io/v1beta1 +kind: SecretStore +metadata: + name: secretstore-ps +spec: + provider: + aws: + service: ParameterStore + region: {{ .Values.region }} +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: external-secrets-ps +spec: + refreshInterval: 1h + secretStoreRef: + name: secretstore-ps + kind: SecretStore + dataFrom: + - extract: + key: {{ .Values.externalSecret.secretStore.secret }} \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/external-secrets/k8s/values.yaml b/argocd/iac/terraform/examples/eks/external-secrets/k8s/values.yaml new file mode 100644 index 00000000..dd8a2ddd --- /dev/null +++ b/argocd/iac/terraform/examples/eks/external-secrets/k8s/values.yaml @@ -0,0 +1,6 @@ +region: us-west-2 +externalSecret: + clusterSecretStore: + secret: secret + secretStore: + secret: /external-secrets/secret \ No newline at end of file diff --git a/argocd/iac/terraform/examples/eks/external-secrets/main.tf b/argocd/iac/terraform/examples/eks/external-secrets/main.tf index 18df2e7e..3e596af0 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/main.tf +++ b/argocd/iac/terraform/examples/eks/external-secrets/main.tf @@ -43,16 +43,26 @@ provider "kubernetes" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - environment = "dev" - region = "us-west-2" - cluster_version = "1.27" - gitops_url = var.gitops_url - gitops_revision = var.gitops_revision - gitops_path = var.gitops_path + name = "ex-${replace(basename(path.cwd), "_", "-")}" + environment = "dev" + region = "us-west-2" + cluster_version = "1.27" + gitops_addons_org = var.gitops_addons_org + gitops_addons_repo = var.gitops_addons_repo + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision + gitops_addons_url = "${local.gitops_addons_org}/${local.gitops_addons_repo}" + + gitops_workload_org = var.gitops_workload_org + gitops_workload_repo = var.gitops_workload_repo + gitops_workload_path = var.gitops_workload_path + gitops_workload_revision = var.gitops_workload_revision + gitops_workload_url = "${local.gitops_workload_org}/${local.gitops_workload_repo}" + + # Secret names in AWS + workload_sm_secret = local.name + workload_pm_secret = "/${local.name}/secret" - aws_secret_manager_secret_name = "argocd-ssh-key" - git_private_ssh_key = "~/.ssh/id_rsa" # Update with the git ssh key to be used by ArgoCD aws_addons = { enable_cert_manager = true @@ -62,7 +72,7 @@ locals { #enable_aws_privateca_issuer = true #enable_cluster_autoscaler = true #enable_external_dns = true - enable_external_secrets = true + enable_external_secrets = true #enable_aws_load_balancer_controller = true #enable_fargate_fluentbit = true #enable_aws_for_fluentbit = true @@ -99,18 +109,24 @@ locals { aws_vpc_id = module.vpc.vpc_id }, { - gitops_bridge_repo_url = local.gitops_url - gitops_bridge_repo_revision = local.gitops_revision + gitops_bridge_repo_url = local.gitops_addons_url + gitops_bridge_repo_path = local.gitops_addons_path + gitops_bridge_repo_revision = local.gitops_addons_revision + }, + { + workload_repo_url = local.gitops_addons_url + workload_repo_path = local.gitops_addons_path + workload_repo_revision = local.gitops_addons_revision + }, + { + workload_sm_secret = aws_ssm_parameter.secret_parameter.name + workload_pm_secret = aws_secretsmanager_secret.secret.name } ) argocd_bootstrap_app_of_apps = { - addons = templatefile("${path.module}/bootstrap/addons.yaml", { - repoURL = local.gitops_url - targetRevision = local.gitops_revision - path = local.gitops_path - }) - workloads = file("${path.module}/bootstrap/workloads.yaml") + addons = file("${path.module}/bootstrap/addons.yaml") + addons = file("${path.module}/bootstrap/workloads.yaml") } vpc_cidr = "10.0.0.0/16" @@ -122,6 +138,35 @@ locals { } } +################################################################################ +# Secret Manager & Parameter Store +################################################################################ +resource "aws_kms_key" "secrets" { + enable_key_rotation = true +} +resource "aws_secretsmanager_secret" "secret" { + name = local.workload_sm_secret + recovery_window_in_days = 0 + kms_key_id = aws_kms_key.secrets.arn +} +resource "aws_secretsmanager_secret_version" "secret" { + secret_id = aws_secretsmanager_secret.secret.id + secret_string = jsonencode({ + username = "secretuser", + password = "secretpassword" + }) +} +resource "aws_ssm_parameter" "secret_parameter" { + name = local.workload_pm_secret + type = "SecureString" + value = jsonencode({ + username = "secretuser", + password = "secretpassword" + }) + key_id = aws_kms_key.secrets.arn +} + + ################################################################################ # GitOps Bridge: Metadata ################################################################################ @@ -144,20 +189,6 @@ module "gitops_bridge_bootstrap" { argocd_bootstrap_app_of_apps = local.argocd_bootstrap_app_of_apps } -################################################################################ -# AWS Secret Manager -################################################################################ -#tfsec:ignore:aws-ssm-secret-use-customer-key -resource "aws_secretsmanager_secret" "git_ssh_key" { - name = local.aws_secret_manager_secret_name - recovery_window_in_days = 0 # Set to zero for this example to force delete during Terraform destroy -} -resource "aws_secretsmanager_secret_version" "git_ssh_key" { - secret_id = aws_secretsmanager_secret.git_ssh_key.id - secret_string = file(pathexpand(local.git_private_ssh_key)) -} - - ################################################################################ # EKS Blueprints Addons ################################################################################ diff --git a/argocd/iac/terraform/examples/eks/external-secrets/variables.tf b/argocd/iac/terraform/examples/eks/external-secrets/variables.tf index dbd72d91..25949a6d 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/variables.tf +++ b/argocd/iac/terraform/examples/eks/external-secrets/variables.tf @@ -1,12 +1,33 @@ -variable "gitops_url" { +variable "gitops_addons_org" { + description = "Git repository org/user contains for addons" + default = "https://github.com/gitops-bridge-dev" +} +variable "gitops_addons_repo" { description = "Git repository contains for addons" - default = "https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template" + default = "gitops-bridge-argocd-control-plane-template" } -variable "gitops_revision" { +variable "gitops_addons_path" { + description = "Git repository path for addons" + default = "bootstrap/control-plane/addons" +} +variable "gitops_addons_revision" { description = "Git repository revision/branch/ref for addons" default = "HEAD" } -variable "gitops_path" { - description = "Git repository path for addons" - default = "bootstrap/control-plane/addons" + +variable "gitops_workload_org" { + description = "Git repository org/user contains for workload" + default = "https://github.com/gitops-bridge-dev" +} +variable "gitops_workload_repo" { + description = "Git repository contains for workload" + default = "gitops-bridge" +} +variable "gitops_workload_path" { + description = "Git repository path for workload" + default = "argocd/iac/terraform/examples/eks/external-secrets/k8s" +} +variable "gitops_workload_revision" { + description = "Git repository revision/branch/ref for workload" + default = "HEAD" } From 423a713fabbe14d72c38aae5296d53464d5edb60 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Fri, 1 Sep 2023 22:42:02 -0400 Subject: [PATCH 2/2] fix bugs Signed-off-by: Carlos Santana --- .../examples/eks/external-secrets/README.md | 21 +++++++------------ .../external-secrets/bootstrap/addons.yaml | 1 + .../external-secrets/bootstrap/workloads.yaml | 5 +++-- .../examples/eks/external-secrets/main.tf | 18 ++++++++-------- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/argocd/iac/terraform/examples/eks/external-secrets/README.md b/argocd/iac/terraform/examples/eks/external-secrets/README.md index 32d95cda..3f806a40 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/README.md +++ b/argocd/iac/terraform/examples/eks/external-secrets/README.md @@ -2,13 +2,8 @@ This example shows how to deploy Amazon EKS with addons configured via ArgoCD -The example demonstrate how to use private git repository for workload apps. - -The example stores your ssh key in AWS Secret Manager, and External Secret Operator to create the secret -for ArgoCD to access the git repositories. - -## Prerequisites -- Create a Github ssh key file, example assumes the file path `~/.ssh/id_rsa`, update `main.tf` if using a different location +The example demonstrate how to use [External Secret Operator(ESO)](https://external-secrets.io) with +AWS Secret Manager and AWS Systems Manager Parameter Store Deploy EKS Cluster ```shell @@ -21,16 +16,16 @@ Access Terraform output to configure `kubectl` and `argocd` terraform output ``` -There is a file `github.yaml` located in the addons git repository `clusters/ex-external-secrets/secret/` this file creates the resources `ClusterSecretStore` and `ExternalSecret`. Update git url in this file when you change the git repository for the workloads specified in `bootstrap/workloads.yaml`. Also update the region in this file if you are using a different region for AWS Secret Manager. - -To verify that the ArgoCD secret with ssh key is created run the following command +Verify that the secrets `external-secrets-ps` and `external-secrets-sm` are present ```shell -kubectl get secret private-repo-creds -n argocd +kubectl get secrets -n external-secrets ``` + Expected output, should have 3 data items in secret ``` -NAME TYPE DATA AGE -private-repo-creds Opaque 3 6m45s +NAME TYPE DATA AGE +external-secrets-ps Opaque 2 1m +external-secrets-sm Opaque 2 1m ``` Destroy EKS Cluster diff --git a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml index 98ad3d93..85aeda76 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml +++ b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/addons.yaml @@ -3,6 +3,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: name: bootstrap-addons + namespace: argocd spec: syncPolicy: preserveResourcesOnDeletion: true diff --git a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml index caf2adbe..176f3508 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml +++ b/argocd/iac/terraform/examples/eks/external-secrets/bootstrap/workloads.yaml @@ -3,6 +3,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: name: external-secrets-example + namespace: argocd spec: syncPolicy: preserveResourcesOnDeletion: true @@ -30,12 +31,12 @@ spec: clusterSecretStore: secret: '{{metadata.annotations.workload_sm_secret}}' secretStore: - secret: '{{metadata.annotations.workload_pm_secret}}' + secret: '{{metadata.annotations.workload_ps_secret}}' destination: namespace: '{{metadata.annotations.external_secrets_namespace}}' name: '{{name}}' syncPolicy: - automated: + automated: {} retry: backoff: duration: 1m diff --git a/argocd/iac/terraform/examples/eks/external-secrets/main.tf b/argocd/iac/terraform/examples/eks/external-secrets/main.tf index 3e596af0..07eff019 100644 --- a/argocd/iac/terraform/examples/eks/external-secrets/main.tf +++ b/argocd/iac/terraform/examples/eks/external-secrets/main.tf @@ -65,7 +65,7 @@ locals { aws_addons = { - enable_cert_manager = true + #enable_cert_manager = true #enable_aws_efs_csi_driver = true #enable_aws_fsx_csi_driver = true #enable_aws_cloudwatch_metrics = true @@ -92,7 +92,7 @@ locals { #enable_ingress_nginx = true #enable_kyverno = true #enable_kube_prometheus_stack = true - enable_metrics_server = true + #enable_metrics_server = true #enable_prometheus_adapter = true #enable_secrets_store_csi_driver = true #enable_vpa = true @@ -114,19 +114,19 @@ locals { gitops_bridge_repo_revision = local.gitops_addons_revision }, { - workload_repo_url = local.gitops_addons_url - workload_repo_path = local.gitops_addons_path - workload_repo_revision = local.gitops_addons_revision + workload_repo_url = local.gitops_workload_url + workload_repo_path = local.gitops_workload_path + workload_repo_revision = local.gitops_workload_revision }, { - workload_sm_secret = aws_ssm_parameter.secret_parameter.name - workload_pm_secret = aws_secretsmanager_secret.secret.name + workload_sm_secret = aws_secretsmanager_secret.secret.name + workload_ps_secret = aws_ssm_parameter.secret_parameter.name } ) argocd_bootstrap_app_of_apps = { - addons = file("${path.module}/bootstrap/addons.yaml") - addons = file("${path.module}/bootstrap/workloads.yaml") + addons = file("${path.module}/bootstrap/addons.yaml") + workloads = file("${path.module}/bootstrap/workloads.yaml") } vpc_cidr = "10.0.0.0/16"