diff --git a/README.md b/README.md index 868d56ec..6803f356 100644 --- a/README.md +++ b/README.md @@ -112,20 +112,20 @@ This diagram describes the default infrastructure: ## Current infrastructure cost -| Resource | Type/size | Price per hour $ | Price per GB $ | Number | Monthly cost | -|---------------|--------------------------|-----------------:|---------------:|-------:|------------------:| -| EKS | | 0.1 | | 1 | 73 | -| EC2 ondemand | t3.medium | 0.0456 | | 1 | 33,288 | -| EC2 Spot | t3.medium/t3a.medium | 0.0137/0.0125 | | 1 | 10 | -| EC2 Spot Ci | t3.medium/t3a.medium | 0.0137/0.0125 | | 0 | 10 | -| EBS | 100 Gb | | 0.11 | 2 | 22 | -| NAT gateway | | 0.048 | 0.048 | 1 | 35 | -| Load Balancer | Classic | 0.028 | 0.008 | 1 | 20.44 | -| S3 | Standart | | | 1 | 1 | -| ECR | 10 Gb | | | 2 | 1.00 | -| Route53 | 1 Hosted Zone | | | 1 | 0.50 | -| Cloudwatch | First 10 Metrics - free | | | | 0 | -| | | | | Total | 216.8 | +| Resource | Type/size | Price per hour $ | Price per GB $ | Number | Monthly cost | +| ------------- | ----------------------- | ---------------: | -------------: | -----: | -----------: | +| EKS | | 0.1 | | 1 | 73 | +| EC2 ondemand | t3.medium | 0.0456 | | 1 | 33,288 | +| EC2 Spot | t3.medium/t3a.medium | 0.0137/0.0125 | | 1 | 10 | +| EC2 Spot Ci | t3.medium/t3a.medium | 0.0137/0.0125 | | 0 | 10 | +| EBS | 100 Gb | | 0.11 | 2 | 22 | +| NAT gateway | | 0.048 | 0.048 | 1 | 35 | +| Load Balancer | Classic | 0.028 | 0.008 | 1 | 20.44 | +| S3 | Standart | | | 1 | 1 | +| ECR | 10 Gb | | | 2 | 1.00 | +| Route53 | 1 Hosted Zone | | | 1 | 0.50 | +| Cloudwatch | First 10 Metrics - free | | | | 0 | +| | | | | Total | 216.8 | > The cost is indicated without counting the amount of traffic for Nat Gateway Load Balancer and S3 ## Namespace structure in the K8S cluster @@ -134,22 +134,22 @@ This diagram describes the default infrastructure: This diagram shows the namespaces used in the cluster and the services deployed there -| Namespace | service | Description | -|-------------|---------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------| -| kube-system | [core-DNS](https://github.com/coredns/coredns) | DNS server used in the cluster | -| certmanager | [cert-manager](https://github.com/jetstack/cert-manager) | Service for automation of management and reception of TLS certificates | -| certmanager | [cluster-issuer](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/cluster-issuer) | Resource representing a certification center that can generate signed certificates using different CA | -| ing | [nginx-ingress](https://github.com/kubernetes/ingress-nginx) | Ingress controller that uses nginx as a reverse proxy | -| ing | [Certificate](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/certificate) | The certificate object used for nginx-ingress | -| dns | [external-dns](https://github.com/bitnami/charts/tree/master/bitnami/external-dns) | Service for organizing access to external DNS from the cluster | -| ci | [gitlab-runner](https://gitlab.com/gitlab-org/charts/gitlab-runner) | Gitlab runner used to launch gitlab-ci agents | -| sys | [aws-node-termination-handler](https://github.com/aws/eks-charts/tree/master/stable/aws-node-termination-handler) | Service for controlling the correct termination of EC2 | -| sys | [autoscaler](https://github.com/kubernetes/autoscaler) | Service that automatically adjusts the size of the k8s cluster depending on the requirements | -| sys | [kubernetes-external-secrets](https://github.com/external-secrets/kubernetes-external-secrets) | Service for working with external secret stores, such as secret-manager, ssm parameter store, etc | -| sys | [Reloader](https://github.com/stakater/Reloader) | Service that monitors changes in external secrets and updates them in the cluster | -| monitoring | [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) | Umbrella chart including a group of services used to monitor cluster performance and visualize data | -| monitoring | [loki-stack](https://github.com/grafana/loki/tree/master/production/helm/loki-stack) | Umbrella chart including a service used to collect container logs and visualize data | -| elk | [elk](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/elk) | Umbrella chart including a group of services for collecting logs and metrics and visualizing this data | +| Namespace | service | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| kube-system | [core-DNS](https://github.com/coredns/coredns) | DNS server used in the cluster | +| certmanager | [cert-manager](https://github.com/jetstack/cert-manager) | Service for automation of management and reception of TLS certificates | +| certmanager | [cluster-issuer](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/cluster-issuer) | Resource representing a certification center that can generate signed certificates using different CA | +| ing | [nginx-ingress](https://github.com/kubernetes/ingress-nginx) | Ingress controller that uses nginx as a reverse proxy | +| ing | [Certificate](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/certificate) | The certificate object used for nginx-ingress | +| dns | [external-dns](https://github.com/bitnami/charts/tree/master/bitnami/external-dns) | Service for organizing access to external DNS from the cluster | +| ci | [gitlab-runner](https://gitlab.com/gitlab-org/charts/gitlab-runner) | Gitlab runner used to launch gitlab-ci agents | +| sys | [aws-node-termination-handler](https://github.com/aws/eks-charts/tree/master/stable/aws-node-termination-handler) | Service for controlling the correct termination of EC2 | +| sys | [autoscaler](https://github.com/kubernetes/autoscaler) | Service that automatically adjusts the size of the k8s cluster depending on the requirements | +| sys | [kubernetes-external-secrets](https://github.com/external-secrets/kubernetes-external-secrets) | Service for working with external secret stores, such as secret-manager, ssm parameter store, etc | +| sys | [Reloader](https://github.com/stakater/Reloader) | Service that monitors changes in external secrets and updates them in the cluster | +| monitoring | [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) | Umbrella chart including a group of services used to monitor cluster performance and visualize data | +| monitoring | [loki-stack](https://github.com/grafana/loki/tree/master/production/helm/loki-stack) | Umbrella chart including a service used to collect container logs and visualize data | +| elk | [elk](https://gitlab.com/madboiler/devops/aws-eks-base/-/tree/master/helm-charts/elk) | Umbrella chart including a group of services for collecting logs and metrics and visualizing this data | ## Useful tools @@ -470,24 +470,97 @@ This will allow you to expand your basic functionality by launching a monitoring We use GitHub Actions and [tfsec](https://github.com/aquasecurity/tfsec) to check our terraform code using static analysis to spot potential security issues. However, we needed to skip some checks. The list of those checks is below: -| Layer | Security issue | Description | Why skipped? | -|---------------|--------------------------|----------------|-----------------| -| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.cluster_egress_internet[0]' defines a fully open egress security group rule. | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | -| layer1-aws/aws-eks.tf | aws-eks-enable-control-plane-logging | Resource 'module.eks:aws_eks_cluster.this[0]' is missing the control plane log type 'scheduler' | By default we enable only audit logs. Can be changed via variable eks_cluster_enabled_log_types | -| layer1-aws/aws-eks.tf | aws-eks-encrypt-secrets | Resource 'module.eks:aws_eks_cluster.this[0]' has no encryptionConfigBlock block | By default encryption is disabled, but can be enabled via setting *eks_cluster_encryption_config_enable = true* in your tfvars file. | -| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access | Resource 'module.eks:aws_eks_cluster.this[0]' has public access is explicitly set to enabled | By default we create public accessible EKS cluster from anywhere | -| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access-to-cidr | Resource 'module.eks:aws_eks_cluster.this[0]' has public access cidr explicitly set to wide open | By default we create public accessible EKS cluster from anywhere | -| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.workers_egress_internet[0]' defines a fully open egress security group rule | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_secrets:aws_iam_role_policy.this' defines a policy with wildcarded resources. | We use this policy for external-secrets and grant it access to all secrets. | -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_autoscaler:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use condition to allow run actions only for certain autoscaling groups | -| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-ingress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.deny-all' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | -| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.deny-all' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | -| kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.allow-from-this' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | -| modules/kubernetes-network-policy-namespace/main.tf | kubernetes-network-no-public-egress | Resource 'module.dev_ns_network_policy:kubernetes_network_policy.allow-from-ns[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.eks_alb_ingress[0]:module.aws_iam_aws_loadbalancer_controller:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use recommended [policy](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json) | -| layer2-k8s/locals.tf | general-secrets-sensitive-in-local | Local 'locals.' includes a potentially sensitive value which is defined within the project | tfsec complains on helm_repo_external_secrets url because it contains the word *secret* | -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_dns:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use the policy from the [documentation](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) -| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_cert_manager:aws_iam_role_policy.this' defines a policy with wildcarded resources | Certmanager uses Route53 to create DNS records and validate wildcard certificates. By default we allow it to manage all zones | +| Layer | Security issue | Description | Why skipped? | +| ---------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.cluster_egress_internet[0]' defines a fully open egress security group rule. | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | +| layer1-aws/aws-eks.tf | aws-eks-enable-control-plane-logging | Resource 'module.eks:aws_eks_cluster.this[0]' is missing the control plane log type 'scheduler' | By default we enable only audit logs. Can be changed via variable eks_cluster_enabled_log_types | +| layer1-aws/aws-eks.tf | aws-eks-encrypt-secrets | Resource 'module.eks:aws_eks_cluster.this[0]' has no encryptionConfigBlock block | By default encryption is disabled, but can be enabled via setting *eks_cluster_encryption_config_enable = true* in your tfvars file. | +| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access | Resource 'module.eks:aws_eks_cluster.this[0]' has public access is explicitly set to enabled | By default we create public accessible EKS cluster from anywhere | +| layer1-aws/aws-eks.tf | aws-eks-no-public-cluster-access-to-cidr | Resource 'module.eks:aws_eks_cluster.this[0]' has public access cidr explicitly set to wide open | By default we create public accessible EKS cluster from anywhere | +| layer1-aws/aws-eks.tf | aws-vpc-no-public-egress-sgr | Resource 'module.eks:aws_security_group_rule.workers_egress_internet[0]' defines a fully open egress security group rule | We use recommended option. [More info](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_secrets:aws_iam_role_policy.this' defines a policy with wildcarded resources. | We use this policy for external-secrets and grant it access to all secrets. | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_autoscaler:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use condition to allow run actions only for certain autoscaling groups | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.eks_alb_ingress[0]:module.aws_iam_aws_loadbalancer_controller:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use recommended [policy](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json) | +| layer2-k8s/locals.tf | general-secrets-sensitive-in-local | Local 'locals.' includes a potentially sensitive value which is defined within the project | tfsec complains on helm_repo_external_secrets url because it contains the word *secret* | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_external_dns:aws_iam_role_policy.this' defines a policy with wildcarded resources | We use the policy from the [documentation](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#iam-policy) | +| modules/aws-iam-eks-trusted/main.tf | aws-iam-no-policy-wildcards | Resource 'module.aws_iam_cert_manager:aws_iam_role_policy.this' defines a policy with wildcarded resources | Certmanager uses Route53 to create DNS records and validate wildcard certificates. By default we allow it to manage all zones | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.monitoring_namespace:kubernetes_network_policy.this[3]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[5]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.external_dns_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.aws_node_termination_handler_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.reloader_namespace:kubernetes_network_policy.this[2]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.loki_namespace:kubernetes_network_policy.this[4]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.monitoring_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.external_secrets_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.external_secrets_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.cluster_autoscaler_namespace:kubernetes_network_policy.this[2]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.certmanager_namespace:kubernetes_network_policy.this[3]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.loki_namespace:kubernetes_network_policy.this[3]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.cluster_autoscaler_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.certmanager_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.aws_node_termination_handler_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[4]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.certmanager_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.reloader_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.cluster_autoscaler_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.certmanager_namespace:kubernetes_network_policy.this[2]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.aws_node_termination_handler_namespace:kubernetes_network_policy.this[2]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.reloader_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.monitoring_namespace:kubernetes_network_policy.this[2]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.loki_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.loki_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.external_dns_namespace:kubernetes_network_policy.this[2]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.monitoring_namespace:kubernetes_network_policy.this[4]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.loki_namespace:kubernetes_network_policy.this[2]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[2]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.cluster_autoscaler_namespace:kubernetes_network_policy.this[3]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[3]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.external_dns_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.monitoring_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.gitlab_runner_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.elk_namespace:kubernetes_network_policy.this[3]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.elk_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.gitlab_runner_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.gitlab_runner_namespace:kubernetes_network_policy.this[2]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.gitlab_runner_namespace:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.elk_namespace:kubernetes_network_policy.this[4]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.elk_namespace:kubernetes_network_policy.this[2]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.elk_namespace:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.aws_load_balancer_controller_namespace[0]:kubernetes_network_policy.this[0]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.aws_load_balancer_controller_namespace[0]:kubernetes_network_policy.this[3]' allows egress traffic to the internet | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.aws_load_balancer_controller_namespace[0]:kubernetes_network_policy.this[2]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-egress | Resource 'module.aws_load_balancer_controller_namespace[0]:kubernetes_network_policy.this[1]' allows all egress traffic by default | We don't want to deny egress traffic in a default installation | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.reloader_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.certmanager_namespace:kubernetes_network_policy.this[3]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.cluster_autoscaler_namespace:kubernetes_network_policy.this[3]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.external_dns_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.monitoring_namespace:kubernetes_network_policy.this[3]' allows ingress traffic from the internet | We allow traffic from 0.0.0.0/0 to trigger webhooks only on certain port and certain pods | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.monitoring_namespace:kubernetes_network_policy.this[4]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.loki_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.external_secrets_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.certmanager_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[5]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.loki_namespace:kubernetes_network_policy.this[4]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.aws_node_termination_handler_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.reloader_namespace:kubernetes_network_policy.this[2]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[2]' allows ingress traffic from the internet | We allow traffic from 0.0.0.0/0 to trigger webhooks only on certain port and certain pods | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.cluster_autoscaler_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.certmanager_namespace:kubernetes_network_policy.this[2]' allows ingress traffic from the internet | We allow traffic from 0.0.0.0/0 to trigger webhooks only on certain port and certain pods | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.aws_node_termination_handler_namespace:kubernetes_network_policy.this[2]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.monitoring_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.external_dns_namespace:kubernetes_network_policy.this[2]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.ingress_nginx_namespace:kubernetes_network_policy.this[3]' allows ingress traffic from the internet | We allow traffic from 0.0.0.0/0 to trigger webhooks only on certain port and certain pods | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.elk_namespace:kubernetes_network_policy.this[4]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.elk_namespace:kubernetes_network_policy.this[3]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.gitlab_runner_namespace:kubernetes_network_policy.this[2]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.elk_namespace:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.aws_load_balancer_controller_namespace[0]:kubernetes_network_policy.this[3]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.aws_load_balancer_controller_namespace[0]:kubernetes_network_policy.this[0]' allows all ingress traffic by default | We deny all ingress trafic by default, but tfsec doesn't work as expected (bug) | +| modules/kubernetes-namespace/network-policy.tf | kubernetes-network-no-public-ingress | Resource 'module.aws_load_balancer_controller_namespace[0]:kubernetes_network_policy.this[2]' allows ingress traffic from the internet | We allow traffic from 0.0.0.0/0 to trigger webhooks only on certain port and certain pods | + ## Contributing @@ -495,5 +568,3 @@ If you're interested in contributing to the project: - Start by reading the [Contributing](https://github.com/maddevsio/aws-eks-base/.github/CONTRIBUTING.md) guide - Explore [current issues](https://github.com/maddevsio/aws-eks-base/issues?q=is%3Aopen+is%3Aissue). - -[![Analytics](https://ga-beacon.appspot.com/UA-83208754-8/aws-eks-base/readme?pixel)](https://github.com/igrigorik/ga-beacon) diff --git a/docs/FAQ.md b/docs/FAQ.md index 4d9fe81d..a8e252ac 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -100,6 +100,7 @@ module "test_namespace" { { name = "allow-this-namespace" policy_types = ["Ingress"] + pod_selector = {} ingress = { from = [ { @@ -115,6 +116,7 @@ module "test_namespace" { { name = "allow-from-ingress-namespace" policy_types = ["Ingress"] + pod_selector = {} ingress = { from = [ { @@ -130,6 +132,7 @@ module "test_namespace" { { name = "allow-egress-to-dev" policy_type = ["Egress"] + pod_selector = {} egress = { ports = [ { diff --git a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf index 84e34299..e22ea51d 100644 --- a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf +++ b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf @@ -13,23 +13,77 @@ locals { }) } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "aws_load_balancer_controller_namespace" { - source = "../modules/kubernetes-namespace" - name = "aws-load-balancer-controller" -} - -resource "helm_release" "aws_loadbalancer_controller" { count = var.aws_loadbalancer_controller_enable ? 1 : 0 - name = "aws-load-balancer-controller" - chart = local.aws-load-balancer-controller.chart - repository = local.aws-load-balancer-controller.repository - version = local.aws-load-balancer-controller.chart_version - namespace = module.aws_load_balancer_controller_namespace.name - max_history = var.helm_release_history_size - - values = [ - local.alb_ingress_controller + source = "../modules/kubernetes-namespace" + name = "aws-load-balancer-controller" + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "aws-load-balancer-controller" + } + } + } + ] + } + }, + { + name = "allow-control-plane" + policy_types = ["Ingress"] + pod_selector = { + match_expressions = { + key = "app.kubernetes.io/name" + operator = "In" + values = ["aws-load-balancer-controller"] + } + } + ingress = { + ports = [ + { + port = "9443" + protocol = "TCP" + } + ] + from = [ + { + ip_block = { + cidr = "0.0.0.0/0" + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } ] } @@ -249,3 +303,18 @@ module "aws_iam_aws_loadbalancer_controller" { ] }) } + +resource "helm_release" "aws_loadbalancer_controller" { + count = var.aws_loadbalancer_controller_enable ? 1 : 0 + + name = "aws-load-balancer-controller" + chart = local.aws-load-balancer-controller.chart + repository = local.aws-load-balancer-controller.repository + version = local.aws-load-balancer-controller.chart_version + namespace = module.aws_load_balancer_controller_namespace[count.index].name + max_history = var.helm_release_history_size + + values = [ + local.alb_ingress_controller + ] +} diff --git a/terraform/layer2-k8s/eks-aws-node-termination-handler.tf b/terraform/layer2-k8s/eks-aws-node-termination-handler.tf index 0ad891d0..93955132 100644 --- a/terraform/layer2-k8s/eks-aws-node-termination-handler.tf +++ b/terraform/layer2-k8s/eks-aws-node-termination-handler.tf @@ -6,9 +6,50 @@ locals { } } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "aws_node_termination_handler_namespace" { source = "../modules/kubernetes-namespace" name = "aws-node-termination-handler" + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "aws-node-termination-handler" + } + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } + ] } resource "helm_release" "aws_node_termination_handler" { diff --git a/terraform/layer2-k8s/eks-cert-manager.tf b/terraform/layer2-k8s/eks-cert-manager.tf index 8e6a51fa..e5680ad9 100644 --- a/terraform/layer2-k8s/eks-cert-manager.tf +++ b/terraform/layer2-k8s/eks-cert-manager.tf @@ -14,23 +14,76 @@ data "template_file" "cert_manager" { } } -resource "helm_release" "cert_manager" { - name = "cert-manager" - chart = local.cert-manager.chart - repository = local.cert-manager.repository - version = local.cert-manager.chart_version - namespace = module.certmanager_namespace.name - wait = true - max_history = var.helm_release_history_size - - values = [ - data.template_file.cert_manager.rendered, - ] -} - +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "certmanager_namespace" { source = "../modules/kubernetes-namespace" name = "certmanager" + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "certmanager" + } + } + } + ] + } + }, + { + name = "allow-control-plane" + policy_types = ["Ingress"] + pod_selector = { + match_expressions = { + key = "app.kubernetes.io/name" + operator = "In" + values = ["webhook"] + } + } + ingress = { + ports = [ + { + port = "10250" + protocol = "TCP" + } + ] + from = [ + { + ip_block = { + cidr = "0.0.0.0/0" + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } + ] } #tfsec:ignore:aws-iam-no-policy-wildcards @@ -73,3 +126,17 @@ module "aws_iam_cert_manager" { ] }) } + +resource "helm_release" "cert_manager" { + name = "cert-manager" + chart = local.cert-manager.chart + repository = local.cert-manager.repository + version = local.cert-manager.chart_version + namespace = module.certmanager_namespace.name + wait = true + max_history = var.helm_release_history_size + + values = [ + data.template_file.cert_manager.rendered, + ] +} diff --git a/terraform/layer2-k8s/eks-cluster-autoscaler.tf b/terraform/layer2-k8s/eks-cluster-autoscaler.tf index 661dbb59..15365f82 100644 --- a/terraform/layer2-k8s/eks-cluster-autoscaler.tf +++ b/terraform/layer2-k8s/eks-cluster-autoscaler.tf @@ -17,24 +17,78 @@ data "template_file" "cluster_autoscaler" { } } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "cluster_autoscaler_namespace" { source = "../modules/kubernetes-namespace" name = "cluster-autoscaler" -} - -resource "helm_release" "cluster_autoscaler" { - name = "cluster-autoscaler" - chart = local.cluster-autoscaler.chart - repository = local.cluster-autoscaler.repository - version = local.cluster-autoscaler.chart_version - namespace = module.cluster_autoscaler_namespace.name - max_history = var.helm_release_history_size - - values = [ - data.template_file.cluster_autoscaler.rendered, + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "cluster-autoscaler" + } + } + } + ] + } + }, + { + name = "allow-monitoring" + policy_types = ["Ingress"] + pod_selector = { + match_expressions = { + key = "app.kubernetes.io/name" + operator = "In" + values = ["aws-cluster-autoscaler"] + } + } + ingress = { + ports = [ + { + port = "8085" + protocol = "TCP" + } + ] + from = [ + { + namespace_selector = { + match_labels = { + name = "monitoring" + } + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } ] - - depends_on = [helm_release.prometheus_operator] } #tfsec:ignore:aws-iam-no-policy-wildcards @@ -78,3 +132,18 @@ module "aws_iam_autoscaler" { ] }) } + +resource "helm_release" "cluster_autoscaler" { + name = "cluster-autoscaler" + chart = local.cluster-autoscaler.chart + repository = local.cluster-autoscaler.repository + version = local.cluster-autoscaler.chart_version + namespace = module.cluster_autoscaler_namespace.name + max_history = var.helm_release_history_size + + values = [ + data.template_file.cluster_autoscaler.rendered, + ] + + depends_on = [helm_release.prometheus_operator] +} diff --git a/terraform/layer2-k8s/eks-external-dns.tf b/terraform/layer2-k8s/eks-external-dns.tf index 99a0d92b..dfc6aa31 100644 --- a/terraform/layer2-k8s/eks-external-dns.tf +++ b/terraform/layer2-k8s/eks-external-dns.tf @@ -16,21 +16,49 @@ data "template_file" "external_dns" { } } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "external_dns_namespace" { source = "../modules/kubernetes-namespace" name = "external-dns" -} - -resource "helm_release" "external_dns" { - name = "external-dns" - chart = local.external-dns.chart - repository = local.external-dns.repository - version = local.external-dns.chart_version - namespace = module.external_dns_namespace.name - max_history = var.helm_release_history_size - - values = [ - data.template_file.external_dns.rendered, + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "external-dns" + } + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } ] } @@ -74,3 +102,16 @@ module "aws_iam_external_dns" { ] }) } + +resource "helm_release" "external_dns" { + name = "external-dns" + chart = local.external-dns.chart + repository = local.external-dns.repository + version = local.external-dns.chart_version + namespace = module.external_dns_namespace.name + max_history = var.helm_release_history_size + + values = [ + data.template_file.external_dns.rendered, + ] +} diff --git a/terraform/layer2-k8s/eks-external-secrets.tf b/terraform/layer2-k8s/eks-external-secrets.tf index 4789fc31..07d5c4f0 100644 --- a/terraform/layer2-k8s/eks-external-secrets.tf +++ b/terraform/layer2-k8s/eks-external-secrets.tf @@ -20,37 +20,79 @@ data "template_file" "external_secrets" { } } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "external_secrets_namespace" { source = "../modules/kubernetes-namespace" name = "external-secrets" -} - -resource "helm_release" "external_secrets" { - name = "external-secrets" - chart = local.external-secrets.chart - repository = local.external-secrets.repository - version = local.external-secrets.chart_version - namespace = module.external_secrets_namespace.name - max_history = var.helm_release_history_size - - values = [ - data.template_file.external_secrets.rendered, + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "external-secrets" + } + } + } + ] + } + } ] } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "reloader_namespace" { source = "../modules/kubernetes-namespace" name = "reloader" -} - -resource "helm_release" "reloader" { - name = "reloader" - chart = local.reloader.chart - repository = local.reloader.repository - version = local.reloader.chart_version - namespace = module.reloader_namespace.name - wait = false - max_history = var.helm_release_history_size + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "reloader" + } + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } + ] } #tfsec:ignore:aws-iam-no-policy-wildcards @@ -71,3 +113,26 @@ module "aws_iam_external_secrets" { ] }) } + +resource "helm_release" "external_secrets" { + name = "external-secrets" + chart = local.external-secrets.chart + repository = local.external-secrets.repository + version = local.external-secrets.chart_version + namespace = module.external_secrets_namespace.name + max_history = var.helm_release_history_size + + values = [ + data.template_file.external_secrets.rendered, + ] +} + +resource "helm_release" "reloader" { + name = "reloader" + chart = local.reloader.chart + repository = local.reloader.repository + version = local.reloader.chart_version + namespace = module.reloader_namespace.name + wait = false + max_history = var.helm_release_history_size +} diff --git a/terraform/layer2-k8s/eks-kube-prometheus-stack.tf b/terraform/layer2-k8s/eks-kube-prometheus-stack.tf index e3fe0d24..742c7315 100644 --- a/terraform/layer2-k8s/eks-kube-prometheus-stack.tf +++ b/terraform/layer2-k8s/eks-kube-prometheus-stack.tf @@ -26,30 +26,95 @@ locals { }) } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "monitoring_namespace" { source = "../modules/kubernetes-namespace" name = "monitoring" -} - -resource "helm_release" "prometheus_operator" { - name = "kube-prometheus-stack" - chart = local.kube-prometheus-stack.chart - repository = local.kube-prometheus-stack.repository - version = local.kube-prometheus-stack.chart_version - namespace = module.monitoring_namespace.name - wait = false - max_history = var.helm_release_history_size + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "monitoring" + } + } + } + ] + } + }, + { + name = "allow-ingress" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { - values = [ - local.kube_prometheus_stack_template + from = [ + { + namespace_selector = { + match_labels = { + name = "ingress-nginx" + } + } + } + ] + } + }, + { + name = "allow-control-plane" + policy_types = ["Ingress"] + pod_selector = { + match_expressions = { + key = "app.kubernetes.io/name" + operator = "In" + values = ["kube-prometheus-stack-operator"] + } + } + ingress = { + ports = [ + { + port = "10250" + protocol = "TCP" + } + ] + from = [ + { + ip_block = { + cidr = "0.0.0.0/0" + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } ] } -resource "random_string" "grafana_password" { - length = 20 - special = true -} - module "aws_iam_grafana" { source = "../modules/aws-iam-eks-trusted" @@ -89,6 +154,25 @@ module "aws_iam_grafana" { }) } +resource "random_string" "grafana_password" { + length = 20 + special = true +} + +resource "helm_release" "prometheus_operator" { + name = "kube-prometheus-stack" + chart = local.kube-prometheus-stack.chart + repository = local.kube-prometheus-stack.repository + version = local.kube-prometheus-stack.chart_version + namespace = module.monitoring_namespace.name + wait = false + max_history = var.helm_release_history_size + + values = [ + local.kube_prometheus_stack_template + ] +} + output "grafana_domain_name" { value = local.grafana_domain_name description = "Grafana dashboards address" diff --git a/terraform/layer2-k8s/eks-loki-stack.tf b/terraform/layer2-k8s/eks-loki-stack.tf index e42950c9..2a27e2c5 100644 --- a/terraform/layer2-k8s/eks-loki-stack.tf +++ b/terraform/layer2-k8s/eks-loki-stack.tf @@ -16,9 +16,100 @@ locals { }) } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "loki_namespace" { source = "../modules/kubernetes-namespace" name = "loki" + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "loki" + } + } + } + ] + } + }, + { + name = "allow-ingress" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + + from = [ + { + namespace_selector = { + match_labels = { + name = "ingress-nginx" + } + } + } + ] + } + }, + { + name = "allow-monitoring" + policy_types = ["Ingress"] + pod_selector = { + match_expressions = { + key = "release" + operator = "In" + values = ["loki-stack"] + } + } + ingress = { + ports = [ + { + port = "http-metrics" + protocol = "TCP" + } + ] + from = [ + { + namespace_selector = { + match_labels = { + name = "monitoring" + } + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } + ] +} + +resource "random_string" "grafana_loki_password" { + length = 20 + special = true } resource "helm_release" "loki_stack" { @@ -36,8 +127,3 @@ resource "helm_release" "loki_stack" { depends_on = [helm_release.prometheus_operator] } - -resource "random_string" "grafana_loki_password" { - length = 20 - special = true -} diff --git a/terraform/layer2-k8s/eks-namespaces.tf b/terraform/layer2-k8s/eks-namespaces.tf index 9db472d3..453a6a4d 100644 --- a/terraform/layer2-k8s/eks-namespaces.tf +++ b/terraform/layer2-k8s/eks-namespaces.tf @@ -1,3 +1,4 @@ +# Calico is not supported when using Fargate with Amazon EKS (NetworkPolicies won't work) module "fargate_namespace" { source = "../modules/kubernetes-namespace" name = "fargate" diff --git a/terraform/layer2-k8s/eks-nginx-ingress-controller.tf b/terraform/layer2-k8s/eks-nginx-ingress-controller.tf index 4e713538..db537872 100644 --- a/terraform/layer2-k8s/eks-nginx-ingress-controller.tf +++ b/terraform/layer2-k8s/eks-nginx-ingress-controller.tf @@ -23,9 +23,134 @@ data "template_file" "nginx_ingress" { } } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "ingress_nginx_namespace" { source = "../modules/kubernetes-namespace" name = "ingress-nginx" + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "ingress-nginx" + } + } + } + ] + } + }, + { + name = "allow-ingress" + policy_types = ["Ingress"] + pod_selector = { + match_expressions = { + key = "app.kubernetes.io/name" + operator = "In" + values = ["ingress-nginx"] + } + } + ingress = { + ports = [ + { + port = "80" + protocol = "TCP" + }, + { + port = "443" + protocol = "TCP" + } + ] + from = [ + { + ip_block = { + cidr = "0.0.0.0/0" + } + } + ] + } + }, + { + name = "allow-control-plane" + policy_types = ["Ingress"] + pod_selector = { + match_expressions = { + key = "app.kubernetes.io/name" + operator = "In" + values = ["ingress-nginx"] + } + } + ingress = { + ports = [ + { + port = "8443" + protocol = "TCP" + } + ] + from = [ + { + ip_block = { + cidr = "0.0.0.0/0" + } + } + ] + } + }, + { + name = "allow-monitoring" + policy_types = ["Ingress"] + pod_selector = { + match_expressions = { + key = "app.kubernetes.io/name" + operator = "In" + values = ["ingress-nginx"] + } + } + ingress = { + ports = [ + { + port = "metrics" + protocol = "TCP" + } + ] + from = [ + { + namespace_selector = { + match_labels = { + name = "monitoring" + } + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } + ] } resource "helm_release" "ingress_nginx" { diff --git a/terraform/layer2-k8s/examples/eks-elk.tf b/terraform/layer2-k8s/examples/eks-elk.tf index be0fd7e5..e58bbf4a 100644 --- a/terraform/layer2-k8s/examples/eks-elk.tf +++ b/terraform/layer2-k8s/examples/eks-elk.tf @@ -24,23 +24,132 @@ data "template_file" "elk" { } } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "elk_namespace" { source = "../modules/kubernetes-namespace" name = "elk" + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "elk" + } + } + } + ] + } + }, + { + name = "allow-ingress" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + + from = [ + { + namespace_selector = { + match_labels = { + name = "ingress-nginx" + } + } + } + ] + } + }, + { + name = "allow-apm" + policy_types = ["Ingress"] + pod_selector = { + match_expressions = { + key = "app" + operator = "In" + values = ["apm-server"] + } + } + ingress = { + ports = [ + { + port = "8200" + protocol = "TCP" + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } + ] } -resource "helm_release" "elk" { - name = "elk" - chart = local.elk.chart - repository = local.elk.repository - version = local.elk.chart_version - namespace = module.elk_namespace.name - wait = false - max_history = var.helm_release_history_size +module "elastic_tls" { + source = "../modules/self-signed-certificate" - values = [ - data.template_file.elk.rendered - ] + name = local.name + common_name = "elasticsearch-master" + dns_names = [local.domain_name, "*.${local.domain_name}", "elasticsearch-master", "elasticsearch-master.${module.elk_namespace.name}", "kibana", "kibana.${module.elk_namespace.name}", "kibana-kibana", "kibana-kibana.${module.elk_namespace.name}", "logstash", "logstash.${module.elk_namespace.name}"] + validity_period_hours = 8760 + early_renewal_hours = 336 +} + +module "aws_iam_elastic_stack" { + source = "../modules/aws-iam-user-with-policy" + + name = "${local.name}-elk" + policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListBucketMultipartUploads", + "s3:ListBucketVersions" + ], + "Resource" : [ + "arn:aws:s3:::${local.elastic_stack_bucket_name}" + ] + }, + { + "Effect" : "Allow", + "Action" : [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource" : [ + "arn:aws:s3:::${local.elastic_stack_bucket_name}/*" + ] + } + ] + }) } ### ADDITIONAL RESOURCES FOR ELK @@ -59,16 +168,6 @@ resource "kubernetes_storage_class" "elk" { } } -module "elastic_tls" { - source = "../modules/self-signed-certificate" - - name = local.name - common_name = "elasticsearch-master" - dns_names = [local.domain_name, "*.${local.domain_name}", "elasticsearch-master", "elasticsearch-master.${module.elk_namespace.name}", "kibana", "kibana.${module.elk_namespace.name}", "kibana-kibana", "kibana-kibana.${module.elk_namespace.name}", "logstash", "logstash.${module.elk_namespace.name}"] - validity_period_hours = 8760 - early_renewal_hours = 336 -} - resource "kubernetes_secret" "elasticsearch_credentials" { metadata { name = "elastic-credentials" @@ -135,6 +234,7 @@ resource "random_string" "kibana_password" { upper = true } +#tfsec:ignore:aws-s3-enable-versioning tfsec:ignore:aws-s3-enable-bucket-logging resource "aws_s3_bucket" "elastic_stack" { bucket = "${local.name}-elastic-stack" acl = "private" @@ -165,40 +265,19 @@ resource "aws_s3_bucket_public_access_block" "elastic_stack_public_access_block" restrict_public_buckets = true } -module "aws_iam_elastic_stack" { - source = "../modules/aws-iam-user-with-policy" +resource "helm_release" "elk" { + name = "elk" + chart = local.elk.chart + repository = local.elk.repository + version = local.elk.chart_version + namespace = module.elk_namespace.name + wait = false + max_history = var.helm_release_history_size + + values = [ + data.template_file.elk.rendered + ] - name = "${local.name}-elk" - policy = jsonencode({ - "Version" : "2012-10-17", - "Statement" : [ - { - "Effect" : "Allow", - "Action" : [ - "s3:ListBucket", - "s3:GetBucketLocation", - "s3:ListBucketMultipartUploads", - "s3:ListBucketVersions" - ], - "Resource" : [ - "arn:aws:s3:::${aws_s3_bucket.elastic_stack.id}" - ] - }, - { - "Effect" : "Allow", - "Action" : [ - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject", - "s3:AbortMultipartUpload", - "s3:ListMultipartUploadParts" - ], - "Resource" : [ - "arn:aws:s3:::${aws_s3_bucket.elastic_stack.id}/*" - ] - } - ] - }) } output "kibana_domain_name" { diff --git a/terraform/layer2-k8s/examples/eks-gitlab-runner.tf b/terraform/layer2-k8s/examples/eks-gitlab-runner.tf index 3065e8b2..43dd2b85 100644 --- a/terraform/layer2-k8s/examples/eks-gitlab-runner.tf +++ b/terraform/layer2-k8s/examples/eks-gitlab-runner.tf @@ -14,25 +14,53 @@ locals { }) } +#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress module "gitlab_runner_namespace" { source = "../modules/kubernetes-namespace" name = "gitlab-runner" -} - -resource "helm_release" "gitlab_runner" { - name = "gitlab-runner" - chart = local.gitlab-runner.chart - repository = local.gitlab-runner.repository - version = local.gitlab-runner.chart_version - namespace = module.gitlab_runner_namespace.name - wait = false - max_history = var.helm_release_history_size - - values = [ - local.gitlab_runner_template + network_policies = [ + { + name = "default-deny" + policy_types = ["Ingress", "Egress"] + pod_selector = {} + }, + { + name = "allow-this-namespace" + policy_types = ["Ingress"] + pod_selector = {} + ingress = { + from = [ + { + namespace_selector = { + match_labels = { + name = "gitlab-runner" + } + } + } + ] + } + }, + { + name = "allow-egress" + policy_types = ["Egress"] + pod_selector = {} + egress = { + to = [ + { + ip_block = { + cidr = "0.0.0.0/0" + except = [ + "169.254.169.254/32" + ] + } + } + ] + } + } ] } +#tfsec:ignore:aws-s3-enable-versioning tfsec:ignore:aws-s3-enable-bucket-logging resource "aws_s3_bucket" "gitlab_runner_cache" { bucket = "${local.name}-gitlab-runner-cache" acl = "private" @@ -88,7 +116,17 @@ module "aws_iam_gitlab_runner" { { "Effect" : "Allow", "Action" : [ - "ecr:*", + "ecr:GetAuthorizationToken", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:ListTagsForResource", + "ecr:DescribeImageScanFindings", + "ecr:DescribeImages" ], "Resource" : "*" }, @@ -106,6 +144,21 @@ module "aws_iam_gitlab_runner" { }) } +resource "helm_release" "gitlab_runner" { + name = "gitlab-runner" + chart = local.gitlab-runner.chart + repository = local.gitlab-runner.repository + version = local.gitlab-runner.chart_version + namespace = module.gitlab_runner_namespace.name + wait = false + max_history = var.helm_release_history_size + + values = [ + local.gitlab_runner_template + ] + +} + output "gitlab_runner_cache_bucket_name" { value = aws_s3_bucket.gitlab_runner_cache.id description = "Name of the s3 bucket for gitlab-runner cache" diff --git a/terraform/layer2-k8s/examples/eks-istio.tf b/terraform/layer2-k8s/examples/eks-istio.tf index 4dd5fa2e..f4e7651b 100644 --- a/terraform/layer2-k8s/examples/eks-istio.tf +++ b/terraform/layer2-k8s/examples/eks-istio.tf @@ -21,6 +21,16 @@ local { } } +module "istio_system_namespace" { + source = "../modules/kubernetes-namespace" + name = "istio-system" +} + +module "kiali_namespace" { + source = "../modules/kubernetes-namespace" + name = "kiali" +} + resource "helm_release" "istio_operator" { name = "istio-operator" chart = local.istio-operator.chart @@ -86,13 +96,3 @@ resource "helm_release" "kiali" { ] depends_on = [helm_release.istio_operator, helm_release.prometheus_operator] } - -module "istio_system_namespace" { - source = "../modules/kubernetes-namespace" - name = "istio-system" -} - -module "kiali_namespace" { - source = "../modules/kubernetes-namespace" - name = "kiali" -} diff --git a/terraform/modules/kubernetes-namespace/network-policy.tf b/terraform/modules/kubernetes-namespace/network-policy.tf index d34ccae0..df567b3e 100644 --- a/terraform/modules/kubernetes-namespace/network-policy.tf +++ b/terraform/modules/kubernetes-namespace/network-policy.tf @@ -6,16 +6,19 @@ resource "kubernetes_network_policy" "this" { namespace = kubernetes_namespace.this[0].id } spec { - pod_selector { - dynamic "match_expressions" { - for_each = lookup(var.network_policies[count.index], "pod_selector", null) != null ? [var.network_policies[count.index].pod_selector] : [] - content { - key = lookup(pod_selector.value, "key", null) - operator = lookup(pod_selector.value, "operator", null) - values = lookup(pod_selector.value, "values", null) + dynamic "pod_selector" { + for_each = lookup(var.network_policies[count.index], "pod_selector", null) != null ? [var.network_policies[count.index].pod_selector] : [] + content { + dynamic "match_expressions" { + for_each = lookup(pod_selector.value, "match_expressions", null) != null ? [pod_selector.value.match_expressions] : [] + content { + key = lookup(match_expressions.value, "key", null) + operator = lookup(match_expressions.value, "operator", null) + values = lookup(match_expressions.value, "values", null) + } } + match_labels = lookup(pod_selector.value, "match_labels", null) } - match_labels = lookup(var.network_policies[count.index], "pod_selector", null) != null ? lookup(var.network_policies[count.index].pod_selector, "match_labels") : null } dynamic "ingress" {