From b620c16f0a4b82ee65b2d248fd98512a9f7e430e Mon Sep 17 00:00:00 2001 From: Sergiusz Urbaniak Date: Wed, 19 Apr 2017 18:04:07 +0200 Subject: [PATCH] modules/aws: tighten security groups (#264) * modules/aws: tighten security groups Currently masters and workers share a pretty open security group. Furthermore workers expose ingress traffic at critical k8s ports like 10250 and 10255. This fixes it by removing the common cluster default security group and specifying separate ingress/egress rules reflecting settings from the current tectonic installer. It also assigns only one security group for masters and workers. Fixes #248, #243, #227 * Documentation/generic-platform: change flannel port to 4789 ... because that one is configured and recommended since it is the IANA based one. Tools like tcpdump then decode vxlan packets natively. The old port (8472) is retained as the default port in the kernel for backwards compatibility purposes only, see [1]. Other projects also switched to the new IANA assigned port. [1] http://lxr.free-electrons.com/source/drivers/net/vxlan.c#L43 --- Documentation/generic-platform.md | 4 +- modules/aws/etcd/network.tf | 38 ----- modules/aws/etcd/nodes.tf | 2 +- modules/aws/etcd/variables.tf | 9 +- modules/aws/master-asg/elb.tf | 13 +- modules/aws/master-asg/master.tf | 51 +------ modules/aws/master-asg/variables.tf | 21 ++- modules/aws/vpc/outputs.tf | 26 +++- modules/aws/vpc/security-groups.tf | 23 --- modules/aws/vpc/sg-elb.tf | 54 +++++++ modules/aws/vpc/sg-etcd.tf | 52 +++++++ modules/aws/vpc/sg-master.tf | 178 ++++++++++++++++++++++ modules/aws/vpc/sg-worker.tf | 178 ++++++++++++++++++++++ modules/aws/vpc/variables.tf | 5 + modules/aws/worker-asg/security-groups.tf | 58 ------- modules/aws/worker-asg/variables.tf | 5 +- modules/aws/worker-asg/worker.tf | 2 +- platforms/aws/main.tf | 17 ++- 18 files changed, 528 insertions(+), 208 deletions(-) delete mode 100644 modules/aws/etcd/network.tf delete mode 100644 modules/aws/vpc/security-groups.tf create mode 100644 modules/aws/vpc/sg-elb.tf create mode 100644 modules/aws/vpc/sg-etcd.tf create mode 100644 modules/aws/vpc/sg-master.tf create mode 100644 modules/aws/vpc/sg-worker.tf delete mode 100644 modules/aws/worker-asg/security-groups.tf diff --git a/Documentation/generic-platform.md b/Documentation/generic-platform.md index d3cfda8ad9..607c508fd5 100644 --- a/Documentation/generic-platform.md +++ b/Documentation/generic-platform.md @@ -19,7 +19,7 @@ Master nodes run most, if not all, control plane components including the API se - **Network:** - Ingress - MUST allow tcp port 22 [ssh] from user network - - MUST allow port 8472 (UDP) from masters & workers for flannel + - MUST allow port 4789 (UDP) from masters & workers for flannel - MUST allow 32000-32002 from all for: Tectonic ingress (if using node ports for ingress like on AWS, otherwise use host ports on workers) - SHOULD allow port 9100 from masters & workers for: Prometheus Node Exporter metrics - MAY have tcp/udp port 30000-32767 [node port range open] @@ -60,7 +60,7 @@ Worked nodes run all of the user applications. The only component they must run - **Ingress** - MUST allow all ports open to master nodes (TODO: be more specific) - MUST have 30000 to 32767 host port range access open - - MUST allow port 8472 (UDP) from masters & workers for: VXLAN (flannel) + - MUST allow port 4789 (UDP) from masters & workers for: VXLAN (flannel) - SHOULD allow port 10250 from masters for k8s features: port-forward, exec, proxy - SHOULD allow port 9100 from masters & workers for: Prometheus Node Exporter metrics - SHOULD allow port 4194 from masters for: Heapster connections to CAdvisor diff --git a/modules/aws/etcd/network.tf b/modules/aws/etcd/network.tf deleted file mode 100644 index 2cf5323ed4..0000000000 --- a/modules/aws/etcd/network.tf +++ /dev/null @@ -1,38 +0,0 @@ -resource "aws_security_group" "etcd_sec_group" { - vpc_id = "${var.vpc_id}" - count = "${length(var.external_endpoints) == 0 ? 1 : 0}" - - tags = "${merge(map( - "Name", "${var.cluster_name}_etcd_sg", - "KubernetesCluster", "${var.cluster_name}" - ), var.extra_tags)}" - - ingress { - protocol = -1 - self = true - from_port = 0 - to_port = 0 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - to_port = 22 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 2379 - to_port = 2379 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - self = true - cidr_blocks = ["0.0.0.0/0"] - } -} diff --git a/modules/aws/etcd/nodes.tf b/modules/aws/etcd/nodes.tf index 911f64e43e..0dc4c44238 100644 --- a/modules/aws/etcd/nodes.tf +++ b/modules/aws/etcd/nodes.tf @@ -30,7 +30,7 @@ resource "aws_instance" "etcd_node" { subnet_id = "${var.subnets[count.index % var.az_count]}" key_name = "${var.ssh_key}" user_data = "${ignition_config.etcd.*.rendered[count.index]}" - vpc_security_group_ids = ["${aws_security_group.etcd_sec_group.id}"] + vpc_security_group_ids = ["${var.sg_ids}"] tags = "${merge(map( "Name", "${var.cluster_name}-etcd-${count.index}", diff --git a/modules/aws/etcd/variables.tf b/modules/aws/etcd/variables.tf index cbb7361875..1eaf618346 100644 --- a/modules/aws/etcd/variables.tf +++ b/modules/aws/etcd/variables.tf @@ -22,10 +22,6 @@ variable "instance_count" { default = "3" } -variable "vpc_id" { - type = "string" -} - variable "ssh_key" { type = "string" } @@ -66,3 +62,8 @@ variable "root_volume_iops" { type = "string" description = "The amount of provisioned IOPS for the root block device." } + +variable "sg_ids" { + type = "list" + description = "The security group IDs to be applied." +} diff --git a/modules/aws/master-asg/elb.tf b/modules/aws/master-asg/elb.tf index f136ab26ed..27b081ae05 100644 --- a/modules/aws/master-asg/elb.tf +++ b/modules/aws/master-asg/elb.tf @@ -2,7 +2,7 @@ resource "aws_elb" "api-internal" { name = "${var.cluster_name}-api-internal" subnets = ["${var.subnet_ids}"] internal = true - security_groups = ["${aws_security_group.master_sec_group.id}"] + security_groups = ["${var.api_sg_ids}"] listener { instance_port = 443 @@ -11,13 +11,6 @@ resource "aws_elb" "api-internal" { lb_protocol = "tcp" } - listener { - instance_port = 10255 - instance_protocol = "tcp" - lb_port = 10255 - lb_protocol = "tcp" - } - health_check { healthy_threshold = 2 unhealthy_threshold = 2 @@ -49,7 +42,7 @@ resource "aws_elb" "api-external" { name = "${var.custom_dns_name == "" ? var.cluster_name : var.custom_dns_name}-api-external" subnets = ["${var.subnet_ids}"] internal = false - security_groups = ["${aws_security_group.master_sec_group.id}"] + security_groups = ["${var.api_sg_ids}"] listener { instance_port = 22 @@ -96,7 +89,7 @@ resource "aws_elb" "console" { name = "${var.custom_dns_name == "" ? var.cluster_name : var.custom_dns_name}-console" subnets = ["${var.subnet_ids}"] internal = "${var.public_vpc ? false : true}" - security_groups = ["${aws_security_group.master_sec_group.id}"] + security_groups = ["${var.console_sg_ids}"] listener { instance_port = 32001 diff --git a/modules/aws/master-asg/master.tf b/modules/aws/master-asg/master.tf index fff0d335ca..651feee8bc 100644 --- a/modules/aws/master-asg/master.tf +++ b/modules/aws/master-asg/master.tf @@ -22,10 +22,6 @@ data "aws_ami" "coreos_ami" { } } -data "aws_vpc" "cluster_vpc" { - id = "${var.vpc_id}" -} - resource "aws_autoscaling_group" "masters" { name = "${var.cluster_name}-masters" desired_capacity = "${var.instance_count}" @@ -60,7 +56,7 @@ resource "aws_launch_configuration" "master_conf" { image_id = "${data.aws_ami.coreos_ami.image_id}" name_prefix = "${var.cluster_name}-master-" key_name = "${var.ssh_key}" - security_groups = ["${concat(list(aws_security_group.master_sec_group.id), var.extra_sg_ids)}"] + security_groups = ["${var.master_sg_ids}"] iam_instance_profile = "${aws_iam_instance_profile.master_profile.arn}" associate_public_ip_address = "${var.public_vpc}" user_data = "${var.user_data}" @@ -76,51 +72,6 @@ resource "aws_launch_configuration" "master_conf" { } } -resource "aws_security_group" "master_sec_group" { - vpc_id = "${data.aws_vpc.cluster_vpc.id}" - - tags = "${merge(map( - "Name", "${var.cluster_name}_master_sg", - "KubernetesCluster", "${var.cluster_name}" - ), var.extra_tags)}" - - ingress { - protocol = -1 - self = true - from_port = 0 - to_port = 0 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - to_port = 22 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 443 - to_port = 443 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 10255 - to_port = 10255 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - self = true - cidr_blocks = ["0.0.0.0/0"] - } -} - resource "aws_iam_instance_profile" "master_profile" { name = "${var.cluster_name}-master-profile" roles = ["${aws_iam_role.master_role.name}"] diff --git a/modules/aws/master-asg/variables.tf b/modules/aws/master-asg/variables.tf index 9cd23f5065..9c988b44d1 100644 --- a/modules/aws/master-asg/variables.tf +++ b/modules/aws/master-asg/variables.tf @@ -2,10 +2,6 @@ variable "ssh_key" { type = "string" } -variable "vpc_id" { - type = "string" -} - variable "cl_channel" { type = "string" } @@ -26,8 +22,19 @@ variable "subnet_ids" { type = "list" } -variable "extra_sg_ids" { - type = "list" +variable "master_sg_ids" { + type = "list" + description = "The security group IDs to be applied to the master nodes." +} + +variable "api_sg_ids" { + type = "list" + description = "The security group IDs to be applied to the public facing ELB." +} + +variable "console_sg_ids" { + type = "list" + description = "The security group IDs to be applied to the console ELB." } variable "base_domain" { @@ -51,7 +58,7 @@ variable "user_data" { } variable "public_vpc" { - description = "If set to true, public facing ingress resource are created." + description = "If set to true, public facing ingress resources are created." default = true } diff --git a/modules/aws/vpc/outputs.tf b/modules/aws/vpc/outputs.tf index 4837aa2188..2f18b37951 100644 --- a/modules/aws/vpc/outputs.tf +++ b/modules/aws/vpc/outputs.tf @@ -1,9 +1,5 @@ output "vpc_id" { - value = "${length(var.external_vpc_id) > 0 ? var.external_vpc_id : join(" ", aws_vpc.new_vpc.*.id)}" -} - -output "cluster_default_sg" { - value = "${aws_security_group.cluster_default.id}" + value = "${data.aws_vpc.cluster_vpc.id}" } # We have to do this join() & split() 'trick' because null_data_source and @@ -15,3 +11,23 @@ output "master_subnet_ids" { output "worker_subnet_ids" { value = ["${split(",", var.external_vpc_id == "" ? join(",", aws_subnet.worker_subnet.*.id) : join(",", data.aws_subnet.external_worker.*.id))}"] } + +output "etcd_sg_id" { + value = "${aws_security_group.etcd.id}" +} + +output "master_sg_id" { + value = "${aws_security_group.master.id}" +} + +output "worker_sg_id" { + value = "${aws_security_group.worker.id}" +} + +output "api_sg_id" { + value = "${aws_security_group.api.id}" +} + +output "console_sg_id" { + value = "${aws_security_group.console.id}" +} diff --git a/modules/aws/vpc/security-groups.tf b/modules/aws/vpc/security-groups.tf deleted file mode 100644 index fd222a528e..0000000000 --- a/modules/aws/vpc/security-groups.tf +++ /dev/null @@ -1,23 +0,0 @@ -resource "aws_security_group" "cluster_default" { - vpc_id = "${data.aws_vpc.cluster_vpc.id}" - - ingress { - protocol = -1 - self = true - from_port = 0 - to_port = 0 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - self = true - cidr_blocks = ["0.0.0.0/0"] - } - - tags = "${merge(map( - "Name","${var.cluster_name}-sg-cluster_default", - "KubernetesCluster", "${var.cluster_name}" - ), var.extra_tags)}" -} diff --git a/modules/aws/vpc/sg-elb.tf b/modules/aws/vpc/sg-elb.tf new file mode 100644 index 0000000000..ca8239a987 --- /dev/null +++ b/modules/aws/vpc/sg-elb.tf @@ -0,0 +1,54 @@ +resource "aws_security_group" "api" { + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}_api_sg", + "KubernetesCluster", "${var.cluster_name}" + ), var.extra_tags)}" + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + to_port = 443 + } +} + +resource "aws_security_group" "console" { + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}_console_sg", + "KubernetesCluster", "${var.cluster_name}" + ), var.extra_tags)}" + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 80 + to_port = 80 + } + + ingress { + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + to_port = 443 + } +} diff --git a/modules/aws/vpc/sg-etcd.tf b/modules/aws/vpc/sg-etcd.tf new file mode 100644 index 0000000000..0b9491bbef --- /dev/null +++ b/modules/aws/vpc/sg-etcd.tf @@ -0,0 +1,52 @@ +resource "aws_security_group" "etcd" { + count = "${var.enable_etcd_sg}" + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}_etcd_sg", + "KubernetesCluster", "${var.cluster_name}" + ), var.extra_tags)}" + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + protocol = "icmp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + to_port = 0 + } + + ingress { + protocol = "tcp" + from_port = 22 + to_port = 22 + self = true + + security_groups = ["${aws_security_group.master.id}"] + } + + ingress { + protocol = "tcp" + from_port = 2379 + to_port = 2379 + self = true + + security_groups = [ + "${aws_security_group.master.id}", + "${aws_security_group.worker.id}", + ] + } + + ingress { + protocol = "tcp" + from_port = 2380 + to_port = 2380 + self = true + } +} diff --git a/modules/aws/vpc/sg-master.tf b/modules/aws/vpc/sg-master.tf new file mode 100644 index 0000000000..0d767ada0b --- /dev/null +++ b/modules/aws/vpc/sg-master.tf @@ -0,0 +1,178 @@ +resource "aws_security_group" "master" { + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}_master_sg", + "KubernetesCluster", "${var.cluster_name}" + ), var.extra_tags)}" +} + +resource "aws_security_group_rule" "master_egress" { + type = "egress" + security_group_id = "${aws_security_group.master.id}" + + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "master_ingress_icmp" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "icmp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + to_port = 0 +} + +resource "aws_security_group_rule" "master_ingress_ssh" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + to_port = 22 +} + +resource "aws_security_group_rule" "master_ingress_http" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 80 + to_port = 80 +} + +resource "aws_security_group_rule" "master_ingress_https" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + to_port = 443 +} + +resource "aws_security_group_rule" "master_ingress_heapster" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + from_port = 4194 + to_port = 4194 + self = true +} + +resource "aws_security_group_rule" "master_ingress_heapster_from_worker" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + source_security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + from_port = 4194 + to_port = 4194 +} + +resource "aws_security_group_rule" "master_ingress_flannel" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "udp" + from_port = 4789 + to_port = 4789 + self = true +} + +resource "aws_security_group_rule" "master_ingress_flannel_from_worker" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + source_security_group_id = "${aws_security_group.worker.id}" + + protocol = "udp" + from_port = 4789 + to_port = 4789 +} + +resource "aws_security_group_rule" "master_ingress_node_exporter" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + from_port = 9100 + to_port = 9100 + self = true +} + +resource "aws_security_group_rule" "master_ingress_node_exporter_from_worker" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + source_security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + from_port = 9100 + to_port = 9100 +} + +resource "aws_security_group_rule" "master_ingress_kubelet_insecure" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + from_port = 10250 + to_port = 10250 + self = true +} + +resource "aws_security_group_rule" "master_ingress_kubelet_insecure_from_worker" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + source_security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + from_port = 10250 + to_port = 10250 +} + +resource "aws_security_group_rule" "master_ingress_kubelet_secure" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + from_port = 10255 + to_port = 10255 + self = true +} + +resource "aws_security_group_rule" "master_ingress_kubelet_secure_from_worker" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + source_security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + from_port = 10255 + to_port = 10255 +} + +resource "aws_security_group_rule" "master_ingress_services" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + from_port = 32000 + to_port = 32767 + self = true +} + +resource "aws_security_group_rule" "master_ingress_services_from_console" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + source_security_group_id = "${aws_security_group.console.id}" + + protocol = "tcp" + from_port = 32000 + to_port = 32767 +} diff --git a/modules/aws/vpc/sg-worker.tf b/modules/aws/vpc/sg-worker.tf new file mode 100644 index 0000000000..6d15ff3050 --- /dev/null +++ b/modules/aws/vpc/sg-worker.tf @@ -0,0 +1,178 @@ +resource "aws_security_group" "worker" { + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}_worker_sg", + "KubernetesCluster", "${var.cluster_name}" + ), var.extra_tags)}" +} + +resource "aws_security_group_rule" "worker_egress" { + type = "egress" + security_group_id = "${aws_security_group.worker.id}" + + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker_ingress_icmp" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "icmp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + to_port = 0 +} + +resource "aws_security_group_rule" "worker_ingress_ssh" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + to_port = 22 +} + +resource "aws_security_group_rule" "worker_ingress_http" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 80 + to_port = 80 +} + +resource "aws_security_group_rule" "worker_ingress_https" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + to_port = 443 +} + +resource "aws_security_group_rule" "worker_ingress_heapster" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + from_port = 4194 + to_port = 4194 + self = true +} + +resource "aws_security_group_rule" "worker_ingress_heapster_from_master" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + from_port = 4194 + to_port = 4194 +} + +resource "aws_security_group_rule" "worker_ingress_flannel" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "udp" + from_port = 4789 + to_port = 4789 + self = true +} + +resource "aws_security_group_rule" "worker_ingress_flannel_from_master" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = "${aws_security_group.master.id}" + + protocol = "udp" + from_port = 4789 + to_port = 4789 +} + +resource "aws_security_group_rule" "worker_ingress_node_exporter" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + from_port = 9100 + to_port = 9100 + self = true +} + +resource "aws_security_group_rule" "worker_ingress_node_exporter_from_master" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + from_port = 9100 + to_port = 9100 +} + +resource "aws_security_group_rule" "worker_ingress_kubelet_insecure" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + from_port = 10250 + to_port = 10250 + self = true +} + +resource "aws_security_group_rule" "worker_ingress_kubelet_insecure_from_master" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + from_port = 10250 + to_port = 10250 +} + +resource "aws_security_group_rule" "worker_ingress_kubelet_secure" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + from_port = 10255 + to_port = 10255 + self = true +} + +resource "aws_security_group_rule" "worker_ingress_kubelet_secure_from_master" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + from_port = 10255 + to_port = 10255 +} + +resource "aws_security_group_rule" "worker_ingress_services" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + + protocol = "tcp" + from_port = 32000 + to_port = 32767 + self = true +} + +resource "aws_security_group_rule" "worker_ingress_services_from_console" { + type = "ingress" + security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = "${aws_security_group.console.id}" + + protocol = "tcp" + from_port = 32000 + to_port = 32767 +} diff --git a/modules/aws/vpc/variables.tf b/modules/aws/vpc/variables.tf index 11f8d852fe..054c63c556 100644 --- a/modules/aws/vpc/variables.tf +++ b/modules/aws/vpc/variables.tf @@ -27,3 +27,8 @@ variable "extra_tags" { type = "map" default = {} } + +variable "enable_etcd_sg" { + description = "If set to true, security groups for etcd nodes are being created" + default = true +} diff --git a/modules/aws/worker-asg/security-groups.tf b/modules/aws/worker-asg/security-groups.tf deleted file mode 100644 index dcb594cfc3..0000000000 --- a/modules/aws/worker-asg/security-groups.tf +++ /dev/null @@ -1,58 +0,0 @@ -resource "aws_security_group" "worker_sec_group" { - vpc_id = "${var.vpc_id}" - - tags = "${merge(map( - "Name", "${var.cluster_name}_worker_sg", - "KubernetesCluster", "${var.cluster_name}" - ), var.extra_tags)}" - - ingress { - protocol = -1 - self = true - from_port = 0 - to_port = 0 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - to_port = 22 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 443 - to_port = 443 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 10255 - to_port = 10255 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 10250 - to_port = 10250 - } - - ingress { - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 30000 - to_port = 32767 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - self = true - cidr_blocks = ["0.0.0.0/0"] - } -} diff --git a/modules/aws/worker-asg/variables.tf b/modules/aws/worker-asg/variables.tf index 7d80b31918..ce3b6e732a 100644 --- a/modules/aws/worker-asg/variables.tf +++ b/modules/aws/worker-asg/variables.tf @@ -26,8 +26,9 @@ variable "subnet_ids" { type = "list" } -variable "extra_sg_ids" { - type = "list" +variable "sg_ids" { + type = "list" + description = "The security group IDs to be applied." } variable "user_data" { diff --git a/modules/aws/worker-asg/worker.tf b/modules/aws/worker-asg/worker.tf index 73327f0fb3..23bdbbd5a2 100644 --- a/modules/aws/worker-asg/worker.tf +++ b/modules/aws/worker-asg/worker.tf @@ -27,7 +27,7 @@ resource "aws_launch_configuration" "worker_conf" { image_id = "${data.aws_ami.coreos_ami.image_id}" name_prefix = "${var.cluster_name}-worker-" key_name = "${var.ssh_key}" - security_groups = ["${concat(list(aws_security_group.worker_sec_group.id), var.extra_sg_ids)}"] + security_groups = ["${var.sg_ids}"] iam_instance_profile = "${aws_iam_instance_profile.worker_profile.arn}" user_data = "${var.user_data}" diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf index 6983a3929c..57bdbb3135 100644 --- a/platforms/aws/main.tf +++ b/platforms/aws/main.tf @@ -11,6 +11,7 @@ module "vpc" { external_master_subnets = ["${compact(var.tectonic_aws_external_master_subnet_ids)}"] external_worker_subnets = ["${compact(var.tectonic_aws_external_worker_subnet_ids)}"] extra_tags = "${var.tectonic_aws_extra_tags}" + enable_etcd_sg = "${length(compact(var.tectonic_etcd_servers)) == 0 ? 1 : 0}" } module "etcd" { @@ -19,12 +20,12 @@ module "etcd" { instance_count = "${var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : var.tectonic_aws_az_count == 5 ? 5 : 3}" az_count = "${var.tectonic_aws_az_count}" ec2_type = "${var.tectonic_aws_etcd_ec2_type}" + sg_ids = ["${module.vpc.etcd_sg_id}"] ssh_key = "${var.tectonic_aws_ssh_key}" cl_channel = "${var.tectonic_cl_channel}" container_image = "${var.tectonic_container_images["etcd"]}" - vpc_id = "${module.vpc.vpc_id}" subnets = ["${module.vpc.worker_subnet_ids}"] dns_zone_id = "${aws_route53_zone.tectonic-int.zone_id}" @@ -60,9 +61,11 @@ module "masters" { ec2_type = "${var.tectonic_aws_master_ec2_type}" cluster_name = "${var.tectonic_cluster_name}" - vpc_id = "${module.vpc.vpc_id}" - subnet_ids = ["${module.vpc.master_subnet_ids}"] - extra_sg_ids = ["${module.vpc.cluster_default_sg}"] + subnet_ids = ["${module.vpc.master_subnet_ids}"] + + master_sg_ids = ["${module.vpc.master_sg_id}"] + api_sg_ids = ["${module.vpc.api_sg_id}"] + console_sg_ids = ["${module.vpc.console_sg_id}"] ssh_key = "${var.tectonic_aws_ssh_key}" cl_channel = "${var.tectonic_cl_channel}" @@ -101,9 +104,9 @@ module "workers" { ec2_type = "${var.tectonic_aws_worker_ec2_type}" cluster_name = "${var.tectonic_cluster_name}" - vpc_id = "${module.vpc.vpc_id}" - subnet_ids = ["${module.vpc.worker_subnet_ids}"] - extra_sg_ids = ["${module.vpc.cluster_default_sg}"] + vpc_id = "${module.vpc.vpc_id}" + subnet_ids = ["${module.vpc.worker_subnet_ids}"] + sg_ids = ["${module.vpc.worker_sg_id}"] ssh_key = "${var.tectonic_aws_ssh_key}" cl_channel = "${var.tectonic_cl_channel}"