Merge pull request #3379 from justinsb/honor_nodeport_range

Automatic merge from submit-queue Honor ServiceNodePortRange when opening NodePort access
kubernetes · Sep 15, 2017 · 7cecaa9 · 7cecaa9
2 parents 7127bab + b29f3a7
commit 7cecaa9
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 9 deletions.
diff --git a/pkg/model/context.go b/pkg/model/context.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/blang/semver"
 	"github.com/golang/glog"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/featureflag"
@@ -314,3 +315,19 @@ func VersionGTE(version semver.Version, major uint64, minor uint64) bool {
 func (c *KopsModelContext) WellKnownServiceIP(id int) (net.IP, error) {
 	return components.WellKnownServiceIP(&c.Cluster.Spec, id)
 }
+
+// NodePortRange returns the range of ports allocated to NodePorts
+func (c *KopsModelContext) NodePortRange() (utilnet.PortRange, error) {
+	// defaultServiceNodePortRange is the default port range for NodePort services.
+	defaultServiceNodePortRange := utilnet.PortRange{Base: 30000, Size: 2768}
+
+	kubeApiServer := c.Cluster.Spec.KubeAPIServer
+	if kubeApiServer != nil && kubeApiServer.ServiceNodePortRange != "" {
+		err := defaultServiceNodePortRange.Set(kubeApiServer.ServiceNodePortRange)
+		if err != nil {
+			return utilnet.PortRange{}, fmt.Errorf("error parsing ServiceNodePortRange %q", kubeApiServer.ServiceNodePortRange)
+		}
+	}
+
+	return defaultServiceNodePortRange, nil
+}
diff --git a/pkg/model/external_access.go b/pkg/model/external_access.go
@@ -72,22 +72,27 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	for _, nodePortAccess := range b.Cluster.Spec.NodePortAccess {
+		nodePortRange, err := b.NodePortRange()
+		if err != nil {
+			return err
+		}
+
 		c.AddTask(&awstasks.SecurityGroupRule{
 			Name:          s("nodeport-tcp-external-to-node-" + nodePortAccess),
 			Lifecycle:     b.Lifecycle,
 			SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleNode),
 			Protocol:      s("tcp"),
-			FromPort:      i64(30000),
-			ToPort:        i64(32767),
+			FromPort:      i64(int64(nodePortRange.Base)),
+			ToPort:        i64(int64(nodePortRange.Base + nodePortRange.Size - 1)),
 			CIDR:          s(nodePortAccess),
 		})
 		c.AddTask(&awstasks.SecurityGroupRule{
 			Name:          s("nodeport-udp-external-to-node-" + nodePortAccess),
 			Lifecycle:     b.Lifecycle,
 			SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleNode),
 			Protocol:      s("udp"),
-			FromPort:      i64(30000),
-			ToPort:        i64(32767),
+			FromPort:      i64(int64(nodePortRange.Base)),
+			ToPort:        i64(int64(nodePortRange.Base + nodePortRange.Size - 1)),
 			CIDR:          s(nodePortAccess),
 		})
 	}

diff --git a/pkg/model/gcemodel/external_access.go b/pkg/model/gcemodel/external_access.go
@@ -17,6 +17,7 @@ limitations under the License.
 package gcemodel
 
 import (
+	"fmt"
 	"github.com/golang/glog"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/upup/pkg/fi"
@@ -68,11 +69,17 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		})
 	}
 
+	// NodePort access
+	nodePortRange, err := b.NodePortRange()
+	if err != nil {
+		return err
+	}
+	nodePortRangeString := nodePortRange.String()
 	c.AddTask(&gcetasks.FirewallRule{
 		Name:         s(b.SafeObjectName("nodeport-external-to-node")),
 		Lifecycle:    b.Lifecycle,
 		TargetTags:   []string{b.GCETagForRole(kops.InstanceGroupRoleNode)},
-		Allowed:      []string{"tcp:30000-32767,udp:30000-32767"},
+		Allowed:      []string{fmt.Sprintf("tcp:%s,udp:%s", nodePortRangeString, nodePortRangeString)},
 		SourceRanges: b.Cluster.Spec.NodePortAccess,
 		Network:      b.LinkToNetwork(),
 	})

diff --git a/tests/integration/complex/in-v1alpha2.yaml b/tests/integration/complex/in-v1alpha2.yaml
@@ -21,6 +21,8 @@ spec:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
     name: events
+  kubeAPIServer:
+    serviceNodePortRange: 28000-32767
   kubernetesVersion: v1.4.6
   masterInternalName: api.internal.complex.example.com
   masterPublicName: api.complex.example.com

diff --git a/tests/integration/complex/kubernetes.tf b/tests/integration/complex/kubernetes.tf
@@ -365,7 +365,7 @@ resource "aws_security_group_rule" "node-to-master-udp-1-65535" {
 resource "aws_security_group_rule" "nodeport-tcp-external-to-node-1-2-3-4--32" {
   type              = "ingress"
   security_group_id = "${aws_security_group.nodes-complex-example-com.id}"
-  from_port         = 30000
+  from_port         = 28000
   to_port           = 32767
   protocol          = "tcp"
   cidr_blocks       = ["1.2.3.4/32"]
@@ -374,7 +374,7 @@ resource "aws_security_group_rule" "nodeport-tcp-external-to-node-1-2-3-4--32" {
 resource "aws_security_group_rule" "nodeport-tcp-external-to-node-10-20-30-0--24" {
   type              = "ingress"
   security_group_id = "${aws_security_group.nodes-complex-example-com.id}"
-  from_port         = 30000
+  from_port         = 28000
   to_port           = 32767
   protocol          = "tcp"
   cidr_blocks       = ["10.20.30.0/24"]
@@ -383,7 +383,7 @@ resource "aws_security_group_rule" "nodeport-tcp-external-to-node-10-20-30-0--24
 resource "aws_security_group_rule" "nodeport-udp-external-to-node-1-2-3-4--32" {
   type              = "ingress"
   security_group_id = "${aws_security_group.nodes-complex-example-com.id}"
-  from_port         = 30000
+  from_port         = 28000
   to_port           = 32767
   protocol          = "udp"
   cidr_blocks       = ["1.2.3.4/32"]
@@ -392,7 +392,7 @@ resource "aws_security_group_rule" "nodeport-udp-external-to-node-1-2-3-4--32" {
 resource "aws_security_group_rule" "nodeport-udp-external-to-node-10-20-30-0--24" {
   type              = "ingress"
   security_group_id = "${aws_security_group.nodes-complex-example-com.id}"
-  from_port         = 30000
+  from_port         = 28000
   to_port           = 32767
   protocol          = "udp"
   cidr_blocks       = ["10.20.30.0/24"]