From b29f3a7505d40423e027cfdf6f720d3bab6d085a Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 15 Sep 2017 00:36:42 -0400 Subject: [PATCH] Honor ServiceNodePortRange when opening NodePort access --- pkg/model/context.go | 17 +++++++++++++++++ pkg/model/external_access.go | 13 +++++++++---- pkg/model/gcemodel/external_access.go | 9 ++++++++- tests/integration/complex/in-v1alpha2.yaml | 2 ++ tests/integration/complex/kubernetes.tf | 8 ++++---- 5 files changed, 40 insertions(+), 9 deletions(-) diff --git a/pkg/model/context.go b/pkg/model/context.go index 149bb7b01ab8a..713746901b3ad 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -25,6 +25,7 @@ import ( "github.com/blang/semver" "github.com/golang/glog" + utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/featureflag" @@ -314,3 +315,19 @@ func VersionGTE(version semver.Version, major uint64, minor uint64) bool { func (c *KopsModelContext) WellKnownServiceIP(id int) (net.IP, error) { return components.WellKnownServiceIP(&c.Cluster.Spec, id) } + +// NodePortRange returns the range of ports allocated to NodePorts +func (c *KopsModelContext) NodePortRange() (utilnet.PortRange, error) { + // defaultServiceNodePortRange is the default port range for NodePort services. + defaultServiceNodePortRange := utilnet.PortRange{Base: 30000, Size: 2768} + + kubeApiServer := c.Cluster.Spec.KubeAPIServer + if kubeApiServer != nil && kubeApiServer.ServiceNodePortRange != "" { + err := defaultServiceNodePortRange.Set(kubeApiServer.ServiceNodePortRange) + if err != nil { + return utilnet.PortRange{}, fmt.Errorf("error parsing ServiceNodePortRange %q", kubeApiServer.ServiceNodePortRange) + } + } + + return defaultServiceNodePortRange, nil +} diff --git a/pkg/model/external_access.go b/pkg/model/external_access.go index d9c66a5238991..e413656456151 100644 --- a/pkg/model/external_access.go +++ b/pkg/model/external_access.go @@ -72,13 +72,18 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.ModelBuilderContext) error { } for _, nodePortAccess := range b.Cluster.Spec.NodePortAccess { + nodePortRange, err := b.NodePortRange() + if err != nil { + return err + } + c.AddTask(&awstasks.SecurityGroupRule{ Name: s("nodeport-tcp-external-to-node-" + nodePortAccess), Lifecycle: b.Lifecycle, SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleNode), Protocol: s("tcp"), - FromPort: i64(30000), - ToPort: i64(32767), + FromPort: i64(int64(nodePortRange.Base)), + ToPort: i64(int64(nodePortRange.Base + nodePortRange.Size - 1)), CIDR: s(nodePortAccess), }) c.AddTask(&awstasks.SecurityGroupRule{ @@ -86,8 +91,8 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.ModelBuilderContext) error { Lifecycle: b.Lifecycle, SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleNode), Protocol: s("udp"), - FromPort: i64(30000), - ToPort: i64(32767), + FromPort: i64(int64(nodePortRange.Base)), + ToPort: i64(int64(nodePortRange.Base + nodePortRange.Size - 1)), CIDR: s(nodePortAccess), }) } diff --git a/pkg/model/gcemodel/external_access.go b/pkg/model/gcemodel/external_access.go index a35413c4f1636..17ee2a093a938 100644 --- a/pkg/model/gcemodel/external_access.go +++ b/pkg/model/gcemodel/external_access.go @@ -17,6 +17,7 @@ limitations under the License. package gcemodel import ( + "fmt" "github.com/golang/glog" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" @@ -68,11 +69,17 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.ModelBuilderContext) error { }) } + // NodePort access + nodePortRange, err := b.NodePortRange() + if err != nil { + return err + } + nodePortRangeString := nodePortRange.String() c.AddTask(&gcetasks.FirewallRule{ Name: s(b.SafeObjectName("nodeport-external-to-node")), Lifecycle: b.Lifecycle, TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleNode)}, - Allowed: []string{"tcp:30000-32767,udp:30000-32767"}, + Allowed: []string{fmt.Sprintf("tcp:%s,udp:%s", nodePortRangeString, nodePortRangeString)}, SourceRanges: b.Cluster.Spec.NodePortAccess, Network: b.LinkToNetwork(), }) diff --git a/tests/integration/complex/in-v1alpha2.yaml b/tests/integration/complex/in-v1alpha2.yaml index c67fb5155e844..99f5701c25f60 100644 --- a/tests/integration/complex/in-v1alpha2.yaml +++ b/tests/integration/complex/in-v1alpha2.yaml @@ -21,6 +21,8 @@ spec: - instanceGroup: master-us-test-1a name: us-test-1a name: events + kubeAPIServer: + serviceNodePortRange: 28000-32767 kubernetesVersion: v1.4.6 masterInternalName: api.internal.complex.example.com masterPublicName: api.complex.example.com diff --git a/tests/integration/complex/kubernetes.tf b/tests/integration/complex/kubernetes.tf index e0af8c170fd79..46eccfe4cf2cf 100644 --- a/tests/integration/complex/kubernetes.tf +++ b/tests/integration/complex/kubernetes.tf @@ -365,7 +365,7 @@ resource "aws_security_group_rule" "node-to-master-udp-1-65535" { resource "aws_security_group_rule" "nodeport-tcp-external-to-node-1-2-3-4--32" { type = "ingress" security_group_id = "${aws_security_group.nodes-complex-example-com.id}" - from_port = 30000 + from_port = 28000 to_port = 32767 protocol = "tcp" cidr_blocks = ["1.2.3.4/32"] @@ -374,7 +374,7 @@ resource "aws_security_group_rule" "nodeport-tcp-external-to-node-1-2-3-4--32" { resource "aws_security_group_rule" "nodeport-tcp-external-to-node-10-20-30-0--24" { type = "ingress" security_group_id = "${aws_security_group.nodes-complex-example-com.id}" - from_port = 30000 + from_port = 28000 to_port = 32767 protocol = "tcp" cidr_blocks = ["10.20.30.0/24"] @@ -383,7 +383,7 @@ resource "aws_security_group_rule" "nodeport-tcp-external-to-node-10-20-30-0--24 resource "aws_security_group_rule" "nodeport-udp-external-to-node-1-2-3-4--32" { type = "ingress" security_group_id = "${aws_security_group.nodes-complex-example-com.id}" - from_port = 30000 + from_port = 28000 to_port = 32767 protocol = "udp" cidr_blocks = ["1.2.3.4/32"] @@ -392,7 +392,7 @@ resource "aws_security_group_rule" "nodeport-udp-external-to-node-1-2-3-4--32" { resource "aws_security_group_rule" "nodeport-udp-external-to-node-10-20-30-0--24" { type = "ingress" security_group_id = "${aws_security_group.nodes-complex-example-com.id}" - from_port = 30000 + from_port = 28000 to_port = 32767 protocol = "udp" cidr_blocks = ["10.20.30.0/24"]