From 3013ed13a5c90810c99bc71f44aa66de1e5a6a87 Mon Sep 17 00:00:00 2001 From: maxim Date: Tue, 9 Nov 2021 17:31:21 +0600 Subject: [PATCH] feat: add keda helm chart --- README.md | 102 ++++++++++++++------------ terraform/layer2-k8s/eks-keda.tf | 63 ++++++++++++++++ terraform/layer2-k8s/helm-charts.yaml | 4 + 3 files changed, 121 insertions(+), 48 deletions(-) create mode 100644 terraform/layer2-k8s/eks-keda.tf diff --git a/README.md b/README.md index 868d56ec..3f590364 100644 --- a/README.md +++ b/README.md @@ -112,20 +112,20 @@ This diagram describes the default infrastructure: ## Current infrastructure cost -| Resource | Type/size | Price per hour $ | Price per GB $ | Number | Monthly cost | -|---------------|--------------------------|-----------------:|---------------:|-------:|------------------:| -| EKS | | 0.1 | | 1 | 73 | -| EC2 ondemand | t3.medium | 0.0456 | | 1 | 33,288 | -| EC2 Spot | t3.medium/t3a.medium | 0.0137/0.0125 | | 1 | 10 | -| EC2 Spot Ci | t3.medium/t3a.medium | 0.0137/0.0125 | | 0 | 10 | -| EBS | 100 Gb | | 0.11 | 2 | 22 | -| NAT gateway | | 0.048 | 0.048 | 1 | 35 | -| Load Balancer | Classic | 0.028 | 0.008 | 1 | 20.44 | -| S3 | Standart | | | 1 | 1 | -| ECR | 10 Gb | | | 2 | 1.00 | -| Route53 | 1 Hosted Zone | | | 1 | 0.50 | -| Cloudwatch | First 10 Metrics - free | | | | 0 | -| | | | | Total | 216.8 | +| Resource | Type/size | Price per hour $ | Price per GB $ | Number | Monthly cost | +| ------------- | ----------------------- | ---------------: | -------------: | -----: | -----------: | +| EKS | | 0.1 | | 1 | 73 | +| EC2 ondemand | t3.medium | 0.0456 | | 1 | 33,288 | +| EC2 Spot | t3.medium/t3a.medium | 0.0137/0.0125 | | 1 | 10 | +| EC2 Spot Ci | t3.medium/t3a.medium | 0.0137/0.0125 | | 0 | 10 | +| EBS | 100 Gb | | 0.11 | 2 | 22 | +| NAT gateway | | 0.048 | 0.048 | 1 | 35 | +| Load Balancer | Classic | 0.028 | 0.008 | 1 | 20.44 | +| S3 | Standart | | | 1 | 1 | +| ECR | 10 Gb | | | 2 | 1.00 | +| Route53 | 1 Hosted Zone | | | 1 | 0.50 | +| Cloudwatch | First 10 Metrics - free | | | | 0 | +| | | | | Total | 216.8 | > The cost is indicated without counting the amount of traffic for Nat Gateway Load Balancer and S3 ## Namespace structure in the K8S cluster @@ -134,22 +134,22 @@ This diagram describes the default infrastructure: This diagram shows the namespaces used in the cluster and the services deployed there -| Namespace | service | Description | -|-------------|---------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------| -| kube-system | [core-DNS](https://github.com/coredns/coredns) | DNS server used in the cluster | -| certmanager | [cert-manager](https://github.com/jetstack/cert-manager) | Service for automation of management and reception of TLS certificates | -| certmanager | [cluster-issuer](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/cluster-issuer) | Resource representing a certification center that can generate signed certificates using different CA | -| ing | [nginx-ingress](https://github.com/kubernetes/ingress-nginx) | Ingress controller that uses nginx as a reverse proxy | -| ing | [Certificate](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/certificate) | The certificate object used for nginx-ingress | -| dns | [external-dns](https://github.com/bitnami/charts/tree/master/bitnami/external-dns) | Service for organizing access to external DNS from the cluster | -| ci | [gitlab-runner](https://gitlab.com/gitlab-org/charts/gitlab-runner) | Gitlab runner used to launch gitlab-ci agents | -| sys | [aws-node-termination-handler](https://github.com/aws/eks-charts/tree/master/stable/aws-node-termination-handler) | Service for controlling the correct termination of EC2 | -| sys | [autoscaler](https://github.com/kubernetes/autoscaler) | Service that automatically adjusts the size of the k8s cluster depending on the requirements | -| sys | [kubernetes-external-secrets](https://github.com/external-secrets/kubernetes-external-secrets) | Service for working with external secret stores, such as secret-manager, ssm parameter store, etc | -| sys | [Reloader](https://github.com/stakater/Reloader) | Service that monitors changes in external secrets and updates them in the cluster | -| monitoring | [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) | Umbrella chart including a group of services used to monitor cluster performance and visualize data | -| monitoring | [loki-stack](https://github.com/grafana/loki/tree/master/production/helm/loki-stack) | Umbrella chart including a service used to collect container logs and visualize data | -| elk | [elk](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/elk) | Umbrella chart including a group of services for collecting logs and metrics and visualizing this data | +| Namespace | service | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| kube-system | [core-DNS](https://github.com/coredns/coredns) | DNS server used in the cluster | +| certmanager | [cert-manager](https://github.com/jetstack/cert-manager) | Service for automation of management and reception of TLS certificates | +| certmanager | [cluster-issuer](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/cluster-issuer) | Resource representing a certification center that can generate signed certificates using different CA | +| ing | [nginx-ingress](https://github.com/kubernetes/ingress-nginx) | Ingress controller that uses nginx as a reverse proxy | +| ing | [Certificate](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/certificate) | The certificate object used for nginx-ingress | +| dns | [external-dns](https://github.com/bitnami/charts/tree/master/bitnami/external-dns) | Service for organizing access to external DNS from the cluster | +| ci | [gitlab-runner](https://gitlab.com/gitlab-org/charts/gitlab-runner) | Gitlab runner used to launch gitlab-ci agents | +| sys | [aws-node-termination-handler](https://github.com/aws/eks-charts/tree/master/stable/aws-node-termination-handler) | Service for controlling the correct termination of EC2 | +| sys | [autoscaler](https://github.com/kubernetes/autoscaler) | Service that automatically adjusts the size of the k8s cluster depending on the requirements | +| sys | [kubernetes-external-secrets](https://github.com/external-secrets/kubernetes-external-secrets) | Service for working with external secret stores, such as secret-manager, ssm parameter store, etc | +| sys | [Reloader](https://github.com/stakater/Reloader) | Service that monitors changes in external secrets and updates them in the cluster | +| monitoring | [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) | Umbrella chart including a group of services used to monitor cluster performance and visualize data | +| monitoring | [loki-stack](https://github.com/grafana/loki/tree/master/production/helm/loki-stack) | Umbrella chart including a service used to collect container logs and visualize data | +| elk | [elk](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/elk) | Umbrella chart including a group of services for collecting logs and metrics and visualizing this data | ## Useful tools @@ -470,24 +470,30 @@ This will allow you to expand your basic functionality by launching a monitoring We use GitHub Actions and [tfsec](https://github.com/aquasecurity/tfsec) to check our terraform code using static analysis to spot potential security issues. However, we needed to skip some checks. The list of those checks is below: -| Layer | Security issue | Description | Why skipped? | -|---------------|--------------------------|----------------|-----------------| -| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.cluster_egress_internet[0]' defines a fully open egress security group rule. | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | -| layer1-aws/aws-eks.tf | aws-eks-enable-control-plane-logging | Resource 'module.eks:aws_eks_cluster.this[0]' is missing the control plane log type 'scheduler' | By default we enable only audit logs. Can be changed via variable eks_cluster_enabled_log_types | -| layer1-aws/aws-eks.tf | aws-eks-encrypt-secrets | Resource 'module.eks:aws_eks_cluster.this[0]' has no encryptionConfigBlock block | By default encryption is disabled, but can be enabled via setting *eks_cluster_encryption_config_enable = true* in your tfvars file. | -| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access | Resource 'module.eks:aws_eks_cluster.this[0]' has public access is explicitly set to enabled | By default we create public accessible EKS cluster from anywhere | -| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access-to-cidr | Resource 'module.eks:aws_eks_cluster.this[0]' has public access cidr explicitly set to wide open | By default we create public accessible EKS cluster from anywhere | -| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.workers_egress_internet[0]' defines a fully open egress security group rule | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_secrets:aws_iam_role_policy.this' defines a policy with wildcarded resources. | We use this policy for external-secrets and grant it access to all secrets. | -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_autoscaler:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use condition to allow run actions only for certain autoscaling groups | -| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-ingress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.deny-all' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | -| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.deny-all' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | -| kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.allow-from-this' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | -| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.allow-from-ns[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.eks_alb_ingress[0]:module.aws_iam_aws_loadbalancer_controller:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use recommended [policy](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json) | -| layer2-k8s/locals.tf | general-secrets-sensitive-in-local | Local 'locals.' includes a potentially sensitive value which is defined within the project | tfsec complains on helm_repo_external_secrets url because it contains the word *secret* | -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_dns:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use the policy from the [documentation](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_cert_manager:aws_iam_role_policy.this' defines a policy with wildcarded resources | Certmanager uses Route53 to create DNS records and validate wildcard certificates. By default we allow it to manage all zones | +| Layer | Security issue | Description | Why skipped? | +| --------------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.cluster_egress_internet[0]' defines a fully open egress security group rule. | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | +| layer1-aws/aws-eks.tf | aws-eks-enable-control-plane-logging | Resource 'module.eks:aws_eks_cluster.this[0]' is missing the control plane log type 'scheduler' | By default we enable only audit logs. Can be changed via variable eks_cluster_enabled_log_types | +| layer1-aws/aws-eks.tf | aws-eks-encrypt-secrets | Resource 'module.eks:aws_eks_cluster.this[0]' has no encryptionConfigBlock block | By default encryption is disabled, but can be enabled via setting *eks_cluster_encryption_config_enable = true* in your tfvars file. | +| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access | Resource 'module.eks:aws_eks_cluster.this[0]' has public access is explicitly set to enabled | By default we create public accessible EKS cluster from anywhere | +| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access-to-cidr | Resource 'module.eks:aws_eks_cluster.this[0]' has public access cidr explicitly set to wide open | By default we create public accessible EKS cluster from anywhere | +| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.workers_egress_internet[0]' defines a fully open egress security group rule | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_secrets:aws_iam_role_policy.this' defines a policy with wildcarded resources. | We use this policy for external-secrets and grant it access to all secrets. | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_autoscaler:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use condition to allow run actions only for certain autoscaling groups | +| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-ingress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.deny-all' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.deny-all' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.allow-from-this' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.allow-from-ns[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.eks_alb_ingress[0]:module.aws_iam_aws_loadbalancer_controller:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use recommended [policy](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json) | +| layer2-k8s/locals.tf | general-secrets-sensitive-in-local | Local 'locals.' includes a potentially sensitive value which is defined within the project | tfsec complains on helm_repo_external_secrets url because it contains the word *secret* | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_dns:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use the policy from the [documentation](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_cert_manager:aws_iam_role_policy.this' defines a policy with wildcarded resources | Certmanager uses Route53 to create DNS records and validate wildcard certificates. By default we allow it to manage all zones | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.keda_namespace:kubernetes_network_policy.this[2]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.keda_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.keda_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.keda_namespace:kubernetes_network_policy.this[2]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.keda_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | + ## Contributing diff --git a/terraform/layer2-k8s/eks-keda.tf b/terraform/layer2-k8s/eks-keda.tf new file mode 100644 index 00000000..ddf99dbb --- /dev/null +++ b/terraform/layer2-k8s/eks-keda.tf @@ -0,0 +1,63 @@ +locals { + keda = { + chart = local.helm_charts[index(local.helm_charts.*.id, "keda")].chart + repository = lookup(local.helm_charts[index(local.helm_charts.*.id, "keda")], "repository", null) + chart_version = lookup(local.helm_charts[index(local.helm_charts.*.id, "keda")], "version", null) + } +} + +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress +module "keda_namespace" { + source = "../modules/kubernetes-namespace" + name = "keda" + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "keda" + } + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } + ] +} + +resource "helm_release" "kedacore" { + name = "keda" + chart = local.keda.chart + repository = local.keda.repository + version = local.keda.chart_version + namespace = module.keda_namespace.name + wait = true + max_history = var.helm_release_history_size +} diff --git a/terraform/layer2-k8s/helm-charts.yaml b/terraform/layer2-k8s/helm-charts.yaml index 7c8dd79f..73613dbe 100644 --- a/terraform/layer2-k8s/helm-charts.yaml +++ b/terraform/layer2-k8s/helm-charts.yaml @@ -59,6 +59,10 @@ charts: chart: ../../helm-charts/istio/istio-resources repository: version: + - id: keda + chart: keda + repository: https://kedacore.github.io/charts + version: 2.4.0 - id: kiali-server chart: kiali-server repository: https://kiali.org/helm-charts