modules/aws: tighten security groups (#264)

* modules/aws: tighten security groups

Currently masters and workers share a pretty open security group.
Furthermore workers expose ingress traffic at critical k8s ports like
10250 and 10255.

This fixes it by removing the common cluster default security group and
specifying separate ingress/egress rules reflecting settings from the
current tectonic installer.

It also assigns only one security group for masters and workers.

Fixes #248, #243, #227

* Documentation/generic-platform: change flannel port to 4789

... because that one is configured and recommended since it is the IANA
based one. Tools like tcpdump then decode vxlan packets natively.

The old port (8472) is retained as the default port in the kernel for
backwards compatibility purposes only, see [1].

Other projects also switched to the new IANA assigned port.

[1] http://lxr.free-electrons.com/source/drivers/net/vxlan.c#L43

Documentation/generic-platform.md

@@ -19,7 +19,7 @@ Master nodes run most, if not all, control plane components including the API se
 - **Network:**
    - Ingress 
       - MUST allow tcp port 22 [ssh] from user network 
-      - MUST allow port 8472 (UDP) from masters & workers for flannel 
+      - MUST allow port 4789 (UDP) from masters & workers for flannel
       - MUST allow 32000-32002 from all for: Tectonic ingress (if using node ports for ingress like on AWS, otherwise use host ports on workers) 
       - SHOULD allow port 9100 from masters & workers for: Prometheus Node Exporter metrics 
       - MAY have tcp/udp port 30000-32767 [node port range open] 
@@ -60,7 +60,7 @@ Worked nodes run all of the user applications. The only component they must run
     - **Ingress**
         - MUST allow all ports open to master nodes (TODO: be more specific) 
         - MUST have 30000 to 32767 host port range access open 
-        - MUST allow port 8472 (UDP) from masters & workers for: VXLAN (flannel) 
+        - MUST allow port 4789 (UDP) from masters & workers for: VXLAN (flannel) 
         - SHOULD allow port 10250 from masters for k8s features: port-forward, exec, proxy 
         - SHOULD allow port 9100 from masters & workers for: Prometheus Node Exporter metrics 
         - SHOULD allow port 4194 from masters for: Heapster connections to CAdvisor 

modules/aws/etcd/nodes.tf

@@ -30,7 +30,7 @@ resource "aws_instance" "etcd_node" {
   subnet_id              = "${var.subnets[count.index % var.az_count]}"
   key_name               = "${var.ssh_key}"
   user_data              = "${ignition_config.etcd.*.rendered[count.index]}"
-  vpc_security_group_ids = ["${aws_security_group.etcd_sec_group.id}"]
+  vpc_security_group_ids = ["${var.sg_ids}"]
 
   tags = "${merge(map(
       "Name", "${var.cluster_name}-etcd-${count.index}",

modules/aws/etcd/variables.tf

@@ -22,10 +22,6 @@ variable "instance_count" {
   default = "3"
 }
 
-variable "vpc_id" {
-  type = "string"
-}
-
 variable "ssh_key" {
   type = "string"
 }
@@ -66,3 +62,8 @@ variable "root_volume_iops" {
   type        = "string"
   description = "The amount of provisioned IOPS for the root block device."
 }
+
+variable "sg_ids" {
+  type        = "list"
+  description = "The security group IDs to be applied."
+}

modules/aws/master-asg/elb.tf

@@ -2,7 +2,7 @@ resource "aws_elb" "api-internal" {
   name            = "${var.cluster_name}-api-internal"
   subnets         = ["${var.subnet_ids}"]
   internal        = true
-  security_groups = ["${aws_security_group.master_sec_group.id}"]
+  security_groups = ["${var.api_sg_ids}"]
 
   listener {
     instance_port     = 443
@@ -11,13 +11,6 @@ resource "aws_elb" "api-internal" {
     lb_protocol       = "tcp"
   }
 
-  listener {
-    instance_port     = 10255
-    instance_protocol = "tcp"
-    lb_port           = 10255
-    lb_protocol       = "tcp"
-  }
-
   health_check {
     healthy_threshold   = 2
     unhealthy_threshold = 2
@@ -49,7 +42,7 @@ resource "aws_elb" "api-external" {
   name            = "${var.custom_dns_name == "" ? var.cluster_name : var.custom_dns_name}-api-external"
   subnets         = ["${var.subnet_ids}"]
   internal        = false
-  security_groups = ["${aws_security_group.master_sec_group.id}"]
+  security_groups = ["${var.api_sg_ids}"]
 
   listener {
     instance_port     = 22
@@ -96,7 +89,7 @@ resource "aws_elb" "console" {
   name            = "${var.custom_dns_name == "" ? var.cluster_name : var.custom_dns_name}-console"
   subnets         = ["${var.subnet_ids}"]
   internal        = "${var.public_vpc ? false : true}"
-  security_groups = ["${aws_security_group.master_sec_group.id}"]
+  security_groups = ["${var.console_sg_ids}"]
 
   listener {
     instance_port     = 32001

modules/aws/master-asg/master.tf

@@ -22,10 +22,6 @@ data "aws_ami" "coreos_ami" {
   }
 }
 
-data "aws_vpc" "cluster_vpc" {
-  id = "${var.vpc_id}"
-}
-
 resource "aws_autoscaling_group" "masters" {
   name                 = "${var.cluster_name}-masters"
   desired_capacity     = "${var.instance_count}"
@@ -60,7 +56,7 @@ resource "aws_launch_configuration" "master_conf" {
   image_id                    = "${data.aws_ami.coreos_ami.image_id}"
   name_prefix                 = "${var.cluster_name}-master-"
   key_name                    = "${var.ssh_key}"
-  security_groups             = ["${concat(list(aws_security_group.master_sec_group.id), var.extra_sg_ids)}"]
+  security_groups             = ["${var.master_sg_ids}"]
   iam_instance_profile        = "${aws_iam_instance_profile.master_profile.arn}"
   associate_public_ip_address = "${var.public_vpc}"
   user_data                   = "${var.user_data}"
@@ -76,51 +72,6 @@ resource "aws_launch_configuration" "master_conf" {
   }
 }
 
-resource "aws_security_group" "master_sec_group" {
-  vpc_id = "${data.aws_vpc.cluster_vpc.id}"
-
-  tags = "${merge(map(
-      "Name", "${var.cluster_name}_master_sg",
-      "KubernetesCluster", "${var.cluster_name}"
-    ), var.extra_tags)}"
-
-  ingress {
-    protocol  = -1
-    self      = true
-    from_port = 0
-    to_port   = 0
-  }
-
-  ingress {
-    protocol    = "tcp"
-    cidr_blocks = ["0.0.0.0/0"]
-    from_port   = 22
-    to_port     = 22
-  }
-
-  ingress {
-    protocol    = "tcp"
-    cidr_blocks = ["0.0.0.0/0"]
-    from_port   = 443
-    to_port     = 443
-  }
-
-  ingress {
-    protocol    = "tcp"
-    cidr_blocks = ["0.0.0.0/0"]
-    from_port   = 10255
-    to_port     = 10255
-  }
-
-  egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    self        = true
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-}
-
 resource "aws_iam_instance_profile" "master_profile" {
   name  = "${var.cluster_name}-master-profile"
   roles = ["${aws_iam_role.master_role.name}"]

modules/aws/master-asg/variables.tf

@@ -2,10 +2,6 @@ variable "ssh_key" {
   type = "string"
 }
 
-variable "vpc_id" {
-  type = "string"
-}
-
 variable "cl_channel" {
   type = "string"
 }
@@ -26,8 +22,19 @@ variable "subnet_ids" {
   type = "list"
 }
 
-variable "extra_sg_ids" {
-  type = "list"
+variable "master_sg_ids" {
+  type        = "list"
+  description = "The security group IDs to be applied to the master nodes."
+}
+
+variable "api_sg_ids" {
+  type        = "list"
+  description = "The security group IDs to be applied to the public facing ELB."
+}
+
+variable "console_sg_ids" {
+  type        = "list"
+  description = "The security group IDs to be applied to the console ELB."
 }
 
 variable "base_domain" {
@@ -51,7 +58,7 @@ variable "user_data" {
 }
 
 variable "public_vpc" {
-  description = "If set to true, public facing ingress resource are created."
+  description = "If set to true, public facing ingress resources are created."
   default     = true
 }
 

modules/aws/vpc/outputs.tf

@@ -1,9 +1,5 @@
 output "vpc_id" {
-  value = "${length(var.external_vpc_id) > 0 ? var.external_vpc_id : join(" ", aws_vpc.new_vpc.*.id)}"
-}
-
-output "cluster_default_sg" {
-  value = "${aws_security_group.cluster_default.id}"
+  value = "${data.aws_vpc.cluster_vpc.id}"
 }
 
 # We have to do this join() & split() 'trick' because null_data_source and 
@@ -15,3 +11,23 @@ output "master_subnet_ids" {
 output "worker_subnet_ids" {
   value = ["${split(",", var.external_vpc_id == "" ? join(",", aws_subnet.worker_subnet.*.id) :  join(",", data.aws_subnet.external_worker.*.id))}"]
 }
+
+output "etcd_sg_id" {
+  value = "${aws_security_group.etcd.id}"
+}
+
+output "master_sg_id" {
+  value = "${aws_security_group.master.id}"
+}
+
+output "worker_sg_id" {
+  value = "${aws_security_group.worker.id}"
+}
+
+output "api_sg_id" {
+  value = "${aws_security_group.api.id}"
+}
+
+output "console_sg_id" {
+  value = "${aws_security_group.console.id}"
+}

modules/aws/vpc/sg-elb.tf

@@ -0,0 +1,54 @@
+resource "aws_security_group" "api" {
+  vpc_id = "${data.aws_vpc.cluster_vpc.id}"
+
+  tags = "${merge(map(
+      "Name", "${var.cluster_name}_api_sg",
+      "KubernetesCluster", "${var.cluster_name}"
+    ), var.extra_tags)}"
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    self        = true
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    from_port   = 443
+    to_port     = 443
+  }
+}
+
+resource "aws_security_group" "console" {
+  vpc_id = "${data.aws_vpc.cluster_vpc.id}"
+
+  tags = "${merge(map(
+      "Name", "${var.cluster_name}_console_sg",
+      "KubernetesCluster", "${var.cluster_name}"
+    ), var.extra_tags)}"
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    self        = true
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    from_port   = 80
+    to_port     = 80
+  }
+
+  ingress {
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    from_port   = 443
+    to_port     = 443
+  }
+}