From ad0294b6202cebb9992a127b1572360fca23d2bd Mon Sep 17 00:00:00 2001 From: maxim Date: Tue, 31 Aug 2021 12:31:38 +0600 Subject: [PATCH 1/3] use tfsec for static analysing terraform code --- .github/workflows/terraform-ci.yml | 61 ++++++++++++++++++++ README.md | 90 +++++++++++++++++++----------- terraform/layer1-aws/aws-eks.tf | 1 + 3 files changed, 119 insertions(+), 33 deletions(-) diff --git a/.github/workflows/terraform-ci.yml b/.github/workflows/terraform-ci.yml index b329261c..2ddd37ba 100644 --- a/.github/workflows/terraform-ci.yml +++ b/.github/workflows/terraform-ci.yml @@ -30,6 +30,18 @@ jobs: - name: Terraform Validate l2 working-directory: ./terraform/layer2-k8s run: terraform validate -no-color . + - name: Upload files for l1 + uses: actions/upload-artifact@v2 + with: + name: l1 + path: ./terraform/layer1-aws/.terraform + retention-days: 1 + - name: Upload files for l2 + uses: actions/upload-artifact@v2 + with: + name: l2 + path: ./terraform/layer2-k8s/.terraform + retention-days: 1 # Checks that all Terraform configuration files format terraform-format: @@ -63,3 +75,52 @@ jobs: - name: Terraform tflint l2 working-directory: ./terraform/layer2-k8s run: tflint --no-color + + terraform-tfsec-l1: + name: 'Terraform-tfsec-l1' + needs: terraform-validate + runs-on: ubuntu-latest + container: + image: tfsec/tfsec + options: --user root + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Download init for l1 + uses: actions/download-artifact@v2 + with: + name: l1 + path: ./terraform/layer1-aws/.terraform + - name: tfsec l1 + working-directory: ./terraform + run: tfsec layer1-aws + - uses: geekyeggo/delete-artifact@v1 + with: + name: l1 + failOnError: false + if: ${{ always() }} + + terraform-tfsec-l2: + name: 'Terraform-tfsec-l2' + needs: terraform-validate + runs-on: ubuntu-latest + container: + image: tfsec/tfsec + options: --user root + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Download init for l2 + uses: actions/download-artifact@v2 + with: + name: l2 + path: ./terraform/layer2-k8s/.terraform + - name: Terraform tfsec l2 + working-directory: ./terraform + run: tfsec layer2-k8s + - uses: geekyeggo/delete-artifact@v1 + with: + name: l2 + failOnError: false + if: ${{ always() }} + diff --git a/README.md b/README.md index 64ddcaad..f05ebb72 100644 --- a/README.md +++ b/README.md @@ -42,39 +42,51 @@ You can find more about this project in Anton Babenko stream: ## Table of contents -- [Architecture diagram](#architecture-diagram) -- [Current infrastructure cost](#current-infrastructure-cost) -- [Namespace structure in the K8S cluster](#namespace-structure-in-the-k8s-cluster) -- [Useful tools](#useful-tools) -- [Useful VSCode extensions](#useful-vscode-extensions) -- [AWS account](#aws-account) - - [IAM settings](#iam-settings) - - [Setting up awscli](#setting-up-awscli) -- [How to use this repo](#how-to-use-this-repo) - - [Getting ready](#getting-ready) - - [S3 state backend](#s3-state-backend) - - [Secrets](#secrets) - - [Domain and SSL](#domain-and-ssl) - - [Working with terraform](#working-with-terraform) - - [init](#init) - - [plan](#plan) - - [apply](#apply) - - [terragrunt](#terragrunt) -- [What to do after deployment](#what-to-do-after-deployment) - - [examples](#examples) -- [Coding conventions](#coding-conventions) - - [Names and approaches used in code](#names-and-approaches-used-in-code) - - [Base project name](#base-project-name) - - [Unique prefix of resource names](#unique-prefix-of-resource-names) - - [Separators](#separators) - - [Resource names](#resource-names) - - [Variable names](#variable-names) - - [Output names](#output-names) - - [Names of terraform files, directories, and modules](#names-of-terraform-files-directories-and-modules) - - [General configuration files](#general-configuration-files) - - [Specific configuration files](#specific-configuration-files) - - [Modules](#modules) - - [Project structure](#project-structure) +- [Boilerplate for a basic AWS infrastructure with EKS cluster](#boilerplate-for-a-basic-aws-infrastructure-with-eks-cluster) + - [Advantages of this boilerplate](#advantages-of-this-boilerplate) + - [Why you should use this boilerplate](#why-you-should-use-this-boilerplate) + - [Description](#description) + - [Table of contents](#table-of-contents) + - [Architecture diagram](#architecture-diagram) + - [Current infrastructure cost](#current-infrastructure-cost) + - [Namespace structure in the K8S cluster](#namespace-structure-in-the-k8s-cluster) + - [Useful tools](#useful-tools) + - [Useful VSCode extensions](#useful-vscode-extensions) + - [AWS account](#aws-account) + - [IAM settings](#iam-settings) + - [Setting up awscli](#setting-up-awscli) + - [How to use this repo](#how-to-use-this-repo) + - [Getting ready](#getting-ready) + - [S3 state backend](#s3-state-backend) + - [Inputs](#inputs) + - [Secrets](#secrets) + - [Domain and SSL](#domain-and-ssl) + - [Working with terraform](#working-with-terraform) + - [init](#init) + - [plan](#plan) + - [apply](#apply) + - [terragrunt](#terragrunt) + - [Apply infrastructure by layers with `terragrunt`](#apply-infrastructure-by-layers-with-terragrunt) + - [Target apply by `terragrunt`](#target-apply-by-terragrunt) + - [Destroy infrastructure by `terragrunt`](#destroy-infrastructure-by-terragrunt) + - [What to do after deployment](#what-to-do-after-deployment) + - [Update terraform version](#update-terraform-version) + - [Updated terraform providers](#updated-terraform-providers) + - [examples](#examples) + - [TFSEC](#tfsec) + - [Coding conventions](#coding-conventions) + - [Names and approaches used in code](#names-and-approaches-used-in-code) + - [Base project name](#base-project-name) + - [Unique prefix of resource names](#unique-prefix-of-resource-names) + - [Separators](#separators) + - [Resource names](#resource-names) + - [Variable names](#variable-names) + - [Output names](#output-names) + - [Names of terraform files, directories, and modules](#names-of-terraform-files-directories-and-modules) + - [General configuration files](#general-configuration-files) + - [Specific configuration files](#specific-configuration-files) + - [Modules](#modules) + - [Project structure](#project-structure) ## Architecture diagram @@ -455,6 +467,18 @@ Each layer has an `examples/` directory that contains working examples that expa This will allow you to expand your basic functionality by launching a monitoring system based on ELK or Prometheus Stack, etc. +## TFSEC +We use GitHub Actions and [tfsec](https://github.com/aquasecurity/tfsec) to check our terraform code using static analysis to spot potential security issues. However, we needed to skip some checks. The list of those checks is below: + +| Layer | Security issue | Description | Why skipped? | +|---------------|--------------------------|----------------|-----------------| +| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.cluster_egress_internet[0]' defines a fully open egress security group rule. | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | +| layer1-aws/aws-eks.tf | aws-eks-enable-control-plane-logging | Resource 'module.eks:aws_eks_cluster.this[0]' is missing the control plane log type 'scheduler' | By default we enable only audit logs. Can be changed via variable eks_cluster_enabled_log_types | +| layer1-aws/aws-eks.tf | aws-eks-encrypt-secrets | Resource 'module.eks:aws_eks_cluster.this[0]' has no encryptionConfigBlock block | By default encryption is disabled, but can be enabled via setting *eks_cluster_encryption_config_enable = true* in your tfvars file. | +| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access | Resource 'module.eks:aws_eks_cluster.this[0]' has public access is explicitly set to enabled | By default we create public accessible EKS cluster from anywhere | +| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access-to-cidr | Resource 'module.eks:aws_eks_cluster.this[0]' has public access cidr explicitly set to wide open | By default we create public accessible EKS cluster from anywhere | +| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.workers_egress_internet[0]' defines a fully open egress security group rule | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | + ## Coding conventions This section contains the most basic recommendations for users and contributors on coding, naming, etc. The goal is consistent, standardized, readable code. Additions, suggestions and changes are welcome. diff --git a/terraform/layer1-aws/aws-eks.tf b/terraform/layer1-aws/aws-eks.tf index 3b204137..5d671850 100644 --- a/terraform/layer1-aws/aws-eks.tf +++ b/terraform/layer1-aws/aws-eks.tf @@ -23,6 +23,7 @@ locals { ] } +#tfsec:ignore:aws-vpc-no-public-egress-sgr tfsec:ignore:aws-eks-enable-control-plane-logging tfsec:ignore:aws-eks-encrypt-secrets tfsec:ignore:aws-eks-no-public-cluster-access tfsec:ignore:aws-eks-no-public-cluster-access-to-cidr tfsec:ignore:aws-vpc-no-public-egress-sgr module "eks" { source = "terraform-aws-modules/eks/aws" version = "17.1.0" From 3c4c2cd0254cde38523abb175a91bbaf687bfbdf Mon Sep 17 00:00:00 2001 From: maxim Date: Tue, 31 Aug 2021 17:27:50 +0600 Subject: [PATCH 2/3] add tfsec ignores for layer2 --- README.md | 10 ++++++++++ .../layer2-k8s/eks-aws-loadbalancer-controller.tf | 1 + terraform/layer2-k8s/eks-cert-manager.tf | 1 + terraform/layer2-k8s/eks-cluster-autoscaler.tf | 1 + terraform/layer2-k8s/eks-external-dns.tf | 1 + terraform/layer2-k8s/eks-external-secrets.tf | 1 + terraform/layer2-k8s/eks-network-policy.tf | 1 + terraform/layer2-k8s/locals.tf | 2 +- 8 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f05ebb72..753544ee 100644 --- a/README.md +++ b/README.md @@ -478,6 +478,16 @@ We use GitHub Actions and [tfsec](https://github.com/aquasecurity/tfsec) to chec | layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access | Resource 'module.eks:aws_eks_cluster.this[0]' has public access is explicitly set to enabled | By default we create public accessible EKS cluster from anywhere | | layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access-to-cidr | Resource 'module.eks:aws_eks_cluster.this[0]' has public access cidr explicitly set to wide open | By default we create public accessible EKS cluster from anywhere | | layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.workers_egress_internet[0]' defines a fully open egress security group rule | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | +| modules/aws-iam-ssm/iam.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_secrets:data.aws_iam_policy_document.this' defines a policy with wildcarded resources. | We use aws-iam-ssm module for external-secrets and grant it access to all secrets. | +| modules/aws-iam-autoscaler/iam.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_autoscaler:data.aws_iam_policy_document.this' defines a policy with wildcarded resources | We use condition to allow run actions only for certain autoscaling groups | +| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-ingress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.deny-all' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.deny-all' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.allow-from-this' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.allow-from-ns[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/aws-iam-aws-loadbalancer-controller/iam.tf | aws-iam-no-policy-wildcards | Resource 'module.eks_alb_ingress[0]:module.aws_iam_aws_loadbalancer_controller:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use recommended [policy](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json) | +| layer2-k8s/locals.tf | general-secrets-sensitive-in-local | Local 'locals.' includes a potentially sensitive value which is defined within the project | tfsec complains on helm_repo_external_secrets url because it contains the word *secret* | +| modules/aws-iam-external-dns/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_dns:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use the policy from the [documentation](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) +| modules/aws-iam-external-dns/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_cert_manager:aws_iam_role_policy.this' defines a policy with wildcarded resources | Certmanager uses Route53 to create DNS records and validate wildcard certificates. By default we allow it to manage all zones | ## Coding conventions diff --git a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf index 840b2c18..a359551f 100644 --- a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf +++ b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf @@ -1,3 +1,4 @@ +#tfsec:ignore:aws-iam-no-policy-wildcards module "eks_alb_ingress" { source = "../modules/eks-aws-loadbalancer-controller" count = var.aws_loadbalancer_controller_enable ? 1 : 0 diff --git a/terraform/layer2-k8s/eks-cert-manager.tf b/terraform/layer2-k8s/eks-cert-manager.tf index 252c0e77..a98b59a2 100644 --- a/terraform/layer2-k8s/eks-cert-manager.tf +++ b/terraform/layer2-k8s/eks-cert-manager.tf @@ -1,3 +1,4 @@ +#tfsec:ignore:aws-iam-no-policy-wildcards module "aws_iam_cert_manager" { source = "../modules/aws-iam-external-dns" diff --git a/terraform/layer2-k8s/eks-cluster-autoscaler.tf b/terraform/layer2-k8s/eks-cluster-autoscaler.tf index ef5e37f9..1ebd2f84 100644 --- a/terraform/layer2-k8s/eks-cluster-autoscaler.tf +++ b/terraform/layer2-k8s/eks-cluster-autoscaler.tf @@ -1,3 +1,4 @@ +#tfsec:ignore:aws-iam-no-policy-wildcards module "aws_iam_autoscaler" { source = "../modules/aws-iam-autoscaler" diff --git a/terraform/layer2-k8s/eks-external-dns.tf b/terraform/layer2-k8s/eks-external-dns.tf index 3148cad0..6eae20fe 100644 --- a/terraform/layer2-k8s/eks-external-dns.tf +++ b/terraform/layer2-k8s/eks-external-dns.tf @@ -1,3 +1,4 @@ +#tfsec:ignore:aws-iam-no-policy-wildcards module "aws_iam_external_dns" { source = "../modules/aws-iam-external-dns" diff --git a/terraform/layer2-k8s/eks-external-secrets.tf b/terraform/layer2-k8s/eks-external-secrets.tf index b007e7f8..128e4d0a 100644 --- a/terraform/layer2-k8s/eks-external-secrets.tf +++ b/terraform/layer2-k8s/eks-external-secrets.tf @@ -1,3 +1,4 @@ +#tfsec:ignore:aws-iam-no-policy-wildcards module "aws_iam_external_secrets" { source = "../modules/aws-iam-ssm" diff --git a/terraform/layer2-k8s/eks-network-policy.tf b/terraform/layer2-k8s/eks-network-policy.tf index 63ba9956..0b305778 100644 --- a/terraform/layer2-k8s/eks-network-policy.tf +++ b/terraform/layer2-k8s/eks-network-policy.tf @@ -16,6 +16,7 @@ resource "helm_release" "calico_daemonset" { ] } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "dev_ns_network_policy" { source = "../modules/kubernetes-network-policy-namespace" namespace = kubernetes_namespace.dev.metadata[0].name diff --git a/terraform/layer2-k8s/locals.tf b/terraform/layer2-k8s/locals.tf index 26c0c27e..71e53fed 100644 --- a/terraform/layer2-k8s/locals.tf +++ b/terraform/layer2-k8s/locals.tf @@ -22,7 +22,7 @@ locals { helm_repo_eks = "https://aws.github.io/eks-charts" helm_repo_softonic = "https://charts.softonic.io" helm_repo_elastic = "https://helm.elastic.co" - helm_repo_external_secrets = "https://external-secrets.github.io/kubernetes-external-secrets" + helm_repo_external_secrets = "https://external-secrets.github.io/kubernetes-external-secrets" #tfsec:ignore:general-secrets-sensitive-in-local helm_repo_stakater = "https://stakater.github.io/stakater-charts" helm_repo_cluster_autoscaler = "https://kubernetes.github.io/autoscaler" helm_repo_ingress_nginx = "https://kubernetes.github.io/ingress-nginx" From 1007abcac1b506be9b04a6ea6c2977233565e6b8 Mon Sep 17 00:00:00 2001 From: maxim Date: Tue, 31 Aug 2021 17:29:26 +0600 Subject: [PATCH 3/3] delete repetitive tfsec ignore rule for eks module --- terraform/layer1-aws/aws-eks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/layer1-aws/aws-eks.tf b/terraform/layer1-aws/aws-eks.tf index 5d671850..dc6a52f3 100644 --- a/terraform/layer1-aws/aws-eks.tf +++ b/terraform/layer1-aws/aws-eks.tf @@ -23,7 +23,7 @@ locals { ] } -#tfsec:ignore:aws-vpc-no-public-egress-sgr tfsec:ignore:aws-eks-enable-control-plane-logging tfsec:ignore:aws-eks-encrypt-secrets tfsec:ignore:aws-eks-no-public-cluster-access tfsec:ignore:aws-eks-no-public-cluster-access-to-cidr tfsec:ignore:aws-vpc-no-public-egress-sgr +#tfsec:ignore:aws-vpc-no-public-egress-sgr tfsec:ignore:aws-eks-enable-control-plane-logging tfsec:ignore:aws-eks-encrypt-secrets tfsec:ignore:aws-eks-no-public-cluster-access tfsec:ignore:aws-eks-no-public-cluster-access-to-cidr module "eks" { source = "terraform-aws-modules/eks/aws" version = "17.1.0"