Skip to content

Commit

Permalink
Add NATTDiscoveryPort to submarinerconfig spec
Browse files Browse the repository at this point in the history 
Signed-off-by: Vishal Thapar <5137689+vthapar@users.noreply.github.com>
  • Loading branch information
@vthapar
vthapar committed Sep 3, 2021
1 parent 339da65 commit e42ea77
Show file tree
Hide file tree
Showing 13 changed files with 158 additions and 102 deletions.
4 changes: 4 additions & 0 deletions deploy/config/crds/submarineraddon.open-cluster-management.io_submarinerconfigs.yaml
View file
Expand Up @@ -40,6 +40,10 @@ spec:
default: 4500
description: IPSecNATTPort represents IPsec NAT-T port (default 4500).
type: integer
NATTDiscoveryPort:
default: 4900
description: NATTDiscoveryPort represents NAT-T Discovery port (default UDP/4900).
type: integer
NATTEnable:
default: true
description: NATTEnable represents IPsec NAT-T enabled (default true).
Expand Down
3 changes: 3 additions & 0 deletions deploy/olm-catalog/0.3.0/submarineraddon.open-cluster-management.io_submarinerconfigs.yaml
View file
Expand Up @@ -35,6 +35,9 @@ spec:
IPSecNATTPort:
description: IPSecNATTPort represents IPsec NAT-T port (default 4500).
type: integer
NATTDiscoveryPort:
description: NATTDiscoveryPort represents NAT-T Discovery port (default UDP/4900).
type: integer
NATTEnable:
description: NATTEnable represents IPsec NAT-T enabled (default true).
type: boolean
Expand Down
4 changes: 4 additions & 0 deletions ...y/olm-catalog/manifests/submarineraddon.open-cluster-management.io_submarinerconfigs.yaml
View file
Expand Up @@ -38,6 +38,10 @@ spec:
default: 4500
description: IPSecNATTPort represents IPsec NAT-T port (default 4500).
type: integer
NATTDiscoveryPort:
default: 4900
description: NATTDiscoveryPort represents NAT-T Discovery port (default UDP/4900).
type: integer
NATTEnable:
default: true
description: NATTEnable represents IPsec NAT-T enabled (default true).
Expand Down
4 changes: 4 additions & 0 deletions ...ig/v1alpha1/0000_00_submarineraddon.open-cluster-management.io_submarinerconfigs.crd.yaml
View file
Expand Up @@ -40,6 +40,10 @@ spec:
default: 4500
description: IPSecNATTPort represents IPsec NAT-T port (default 4500).
type: integer
NATTDiscoveryPort:
default: 4900
description: NATTDiscoveryPort represents NAT-T Discovery port (default UDP/4900).
type: integer
NATTEnable:
default: true
description: NATTEnable represents IPsec NAT-T enabled (default true).
Expand Down
5 changes: 5 additions & 0 deletions pkg/apis/submarinerconfig/v1alpha1/types.go
View file
Expand Up @@ -42,6 +42,11 @@ type SubmarinerConfigSpec struct {
// +kubebuilder:default=4500
IPSecNATTPort int `json:"IPSecNATTPort,omitempty"`

// NATTDiscoveryPort represents NAT Discovery port (default UDP/4900).
// +optional
// +kubebuilder:default=4900
NATTDiscoveryPort int `json:"NATTDiscoveryPort,omitempty"`

// NATTEnable represents IPsec NAT-T enabled (default true).
// +optional
// +kubebuilder:default=true
Expand Down
1 change: 1 addition & 0 deletions pkg/apis/submarinerconfig/v1alpha1/zz_generated.swagger_doc_generated.go
View file
Expand Up @@ -65,6 +65,7 @@ var map_SubmarinerConfigSpec = map[string]string{
"cableDriver": "CableDriver represents the submariner cable driver implementation. Available options are libreswan (default) strongswan, wireguard, and vxlan.",
"IPSecIKEPort": "IPSecIKEPort represents IPsec IKE port (default 500).",
"IPSecNATTPort": "IPSecNATTPort represents IPsec NAT-T port (default 4500).",
"NATTDiscoveryPort": "NATTDiscoveryPort represents NAT Discovery port (default UDP/4900).",
"NATTEnable": "NATTEnable represents IPsec NAT-T enabled (default true).",
"credentialsSecret": "CredentialsSecret is a reference to the secret with a certain cloud platform credentials, the supported platform includes AWS, GCP, Azure, ROKS and OSD. The submariner-addon will use these credentials to prepare Submariner cluster environment. If the submariner cluster environment requires submariner-addon preparation, this field should be specified.",
"subscriptionConfig": "SubscriptionConfig represents a Submariner subscription. SubscriptionConfig can be used to customize the Submariner subscription.",
Expand Down
77 changes: 47 additions & 30 deletions pkg/cloud/aws/aws.go
View file
Expand Up @@ -56,25 +56,26 @@ type machineSetConfig struct {
}

type awsProvider struct {
workClinet workclient.Interface
awsClinet client.Interface
eventRecorder events.Recorder
region string
infraId string
ikePort int64
nattPort int64
clusterName string
instanceType string
gateways int
cloudPrepare cpapi.Cloud
workClinet workclient.Interface
awsClinet client.Interface
eventRecorder events.Recorder
region string
infraId string
ikePort int64
nattPort int64
nattDiscoveryPort int64
clusterName string
instanceType string
gateways int
cloudPrepare cpapi.Cloud
}

func NewAWSProvider(
kubeClient kubernetes.Interface, workClient workclient.Interface,
eventRecorder events.Recorder,
region, infraId, clusterName, credentialsSecretName string,
instanceType string,
ikePort, nattPort int,
ikePort, nattPort, nattDiscoveryPort int,
gateways int) (*awsProvider, error) {
if region == "" {
return nil, fmt.Errorf("cluster region is empty")
Expand All @@ -96,6 +97,10 @@ func NewAWSProvider(
nattPort = helpers.SubmarinerNatTPort
}

if nattDiscoveryPort == 0 {
nattDiscoveryPort = helpers.SubmarinerNatTDiscoveryPort
}

if instanceType == "" {
instanceType = defaultInstanceType
}
Expand All @@ -106,25 +111,26 @@ func NewAWSProvider(
}

return &awsProvider{
workClinet: workClient,
awsClinet: awsClient,
eventRecorder: eventRecorder,
region: region,
infraId: infraId,
ikePort: int64(ikePort),
nattPort: int64(nattPort),
clusterName: clusterName,
instanceType: instanceType,
gateways: gateways,
cloudPrepare: cpaws.NewCloud(awsClient.FullEC2API(), infraId, region),
workClinet: workClient,
awsClinet: awsClient,
eventRecorder: eventRecorder,
region: region,
infraId: infraId,
ikePort: int64(ikePort),
nattPort: int64(nattPort),
nattDiscoveryPort: int64(nattDiscoveryPort),
clusterName: clusterName,
instanceType: instanceType,
gateways: gateways,
cloudPrepare: cpaws.NewCloud(awsClient.FullEC2API(), infraId, region),
}, nil
}

// PrepareSubmarinerClusterEnv prepares submariner cluster environment on AWS
// The below tasks will be executed
// 1. open submariner route port (4800/UDP) between all master and worker nodes
// 2. open submariner metrics port (8080/TCP) between all master and worker nodes
// 3. open IPsec ports (by default, 4500/UDP and 500/UDP) for submariner gateway instances
// 3. open IPsec ant NATT discovery ports (by default, 4500/UDP, 4900/UDP and 500/UDP) for submariner gateway instances
// 4. find one pulic subnet and tag it with label AWS internal elb label for automatic
// subnet discovery by aws load balancers or ingress controllers
// 5. apply a manifest work to create a MachineSet on managed cluster to create a new AWS
Expand Down Expand Up @@ -160,7 +166,7 @@ func (a *awsProvider) PrepareSubmarinerClusterEnv() error {
return fmt.Errorf("failed to open route port in security group: %v \n", err)
}

// Open IPsec ports (by default, 4500/UDP and 500/UDP) for submariner gateway instances
// Open IPsec and NAT discovery ports (by default, 4500/UDP, 4900/UDP and 500/UDP) for submariner gateway instances
if err := a.openIPsecPorts(vpc); err != nil {
return fmt.Errorf("failed to create security group with infraID %s and vpcID %s: %v \n", a.infraId, *vpc.VpcId, err)
}
Expand Down Expand Up @@ -458,7 +464,7 @@ func (a *awsProvider) revokePort(masterSecurityGroup, workerSecurityGroup *ec2.S
}

func (a *awsProvider) openIPsecPorts(vpc *ec2.Vpc) error {
permissions := getIPsecPortsPermission(a.ikePort, a.nattPort)
permissions := getIPsecPortsPermission(a.ikePort, a.nattPort, a.nattDiscoveryPort)
groupName := fmt.Sprintf("%s-submariner-gw-sg", a.infraId)
sg, err := a.findSecurityGroup(*vpc.VpcId, groupName)
if errors.IsNotFound(err) {
Expand All @@ -469,7 +475,7 @@ func (a *awsProvider) openIPsecPorts(vpc *ec2.Vpc) error {
}

// the rules has been built
if hasIPsecPorts(sg.IpPermissions, a.ikePort, a.nattPort) {
if hasIPsecPorts(sg.IpPermissions, a.ikePort, a.nattPort, a.nattDiscoveryPort) {
klog.V(4).Infof("the IPsec ports has been opened in security group %s on aws", *sg.GroupId)
return nil
}
Expand Down Expand Up @@ -734,7 +740,7 @@ func getRoutePortPermission(
})
}

func getIPsecPortsPermission(ikePort, nattPort int64) []*ec2.IpPermission {
func getIPsecPortsPermission(ikePort, nattPort, nattDiscoveryPort int64) []*ec2.IpPermission {
return []*ec2.IpPermission{
(&ec2.IpPermission{}).
SetIpProtocol("udp").
Expand All @@ -750,22 +756,33 @@ func getIPsecPortsPermission(ikePort, nattPort int64) []*ec2.IpPermission {
SetIpRanges([]*ec2.IpRange{
(&ec2.IpRange{}).SetCidrIp("0.0.0.0/0"),
}),
(&ec2.IpPermission{}).
SetIpProtocol("udp").
SetFromPort(nattDiscoveryPort).
SetToPort(nattDiscoveryPort).
SetIpRanges([]*ec2.IpRange{
(&ec2.IpRange{}).SetCidrIp("0.0.0.0/0"),
}),
}
}

func hasIPsecPorts(permissions []*ec2.IpPermission, expectedIKEPort, expectedNatTPort int64) bool {
if len(permissions) != 2 {
func hasIPsecPorts(permissions []*ec2.IpPermission, expectedIKEPort, expectedNatTPort, expectedNatTDiscoveryPort int64) bool {
if len(permissions) != 3 {
return false
}
ports := make(map[int64]bool)
ports[*permissions[0].FromPort] = true
ports[*permissions[1].FromPort] = true
ports[*permissions[2].FromPort] = true
if _, ok := ports[expectedIKEPort]; !ok {
return false
}
if _, ok := ports[expectedNatTPort]; !ok {
return false
}
if _, ok := ports[expectedNatTDiscoveryPort]; !ok {
return false
}
return true
}

Expand Down
45 changes: 24 additions & 21 deletions pkg/cloud/aws/aws_test.go
View file
Expand Up @@ -135,7 +135,7 @@ func TestPrepareSubmarinerClusterEnv(t *testing.T) {
}).Return(&ec2.CreateSecurityGroupOutput{GroupId: aws.String("sg-3")}, nil)
mock.EXPECT().AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
GroupId: aws.String("sg-3"),
IpPermissions: getIPsecPortsPermission(int64(500), int64(4500)),
IpPermissions: getIPsecPortsPermission(int64(500), int64(4500), int64(4900)),
}).Return(&ec2.AuthorizeSecurityGroupIngressOutput{}, nil)
mock.EXPECT().DescribeSubnets(&ec2.DescribeSubnetsInput{
Filters: []*ec2.Filter{
Expand Down Expand Up @@ -273,7 +273,7 @@ func TestPrepareSubmarinerClusterEnv(t *testing.T) {
}).Return(&ec2.AuthorizeSecurityGroupIngressOutput{}, awserr.New("InvalidPermission.Duplicate", "test", nil))
gwSG := &ec2.SecurityGroup{
GroupId: aws.String("sg-3"),
IpPermissions: getIPsecPortsPermission(int64(500), int64(4500)),
IpPermissions: getIPsecPortsPermission(int64(500), int64(4500), int64(4900)),
}
mock.EXPECT().DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
Filters: []*ec2.Filter{
Expand Down Expand Up @@ -348,16 +348,17 @@ func TestPrepareSubmarinerClusterEnv(t *testing.T) {

workClient := fakeworkclient.NewSimpleClientset(c.works...)
awsProvider := &awsProvider{
workClinet: workClient,
awsClinet: awsClient,
region: "us-east-1",
infraId: "testcluster-a1b1",
ikePort: int64(500),
nattPort: int64(4500),
clusterName: "testcluster",
instanceType: "m5n.large",
gateways: 1,
eventRecorder: eventstesting.NewTestingEventRecorder(t),
workClinet: workClient,
awsClinet: awsClient,
region: "us-east-1",
infraId: "testcluster-a1b1",
ikePort: int64(500),
nattPort: int64(4500),
nattDiscoveryPort: int64(4900),
clusterName: "testcluster",
instanceType: "m5n.large",
gateways: 1,
eventRecorder: eventstesting.NewTestingEventRecorder(t),
}

err := awsProvider.PrepareSubmarinerClusterEnv()
Expand Down Expand Up @@ -437,7 +438,8 @@ func TestCleanUpSubmarinerClusterEnv(t *testing.T) {
},
Resources: []*string{aws.String("sub-1")},
}).Return(&ec2.DeleteTagsOutput{}, nil)
sg := &ec2.SecurityGroup{GroupId: aws.String("sg-3"), IpPermissions: getIPsecPortsPermission(int64(500), int64(4500))}
sg := &ec2.SecurityGroup{GroupId: aws.String("sg-3"), IpPermissions: getIPsecPortsPermission(int64(500),
int64(4500), int64(4900))}
mock.EXPECT().DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
Filters: []*ec2.Filter{
{
Expand Down Expand Up @@ -518,14 +520,15 @@ func TestCleanUpSubmarinerClusterEnv(t *testing.T) {

workClient := fakeworkclient.NewSimpleClientset(c.works...)
awsProvider := &awsProvider{
workClinet: workClient,
awsClinet: awsClient,
region: "us-east-1",
infraId: "testcluster-a1b1",
ikePort: int64(500),
nattPort: int64(4500),
clusterName: "testcluster",
eventRecorder: eventstesting.NewTestingEventRecorder(t),
workClinet: workClient,
awsClinet: awsClient,
region: "us-east-1",
infraId: "testcluster-a1b1",
ikePort: int64(500),
nattPort: int64(4500),
nattDiscoveryPort: int64(4900),
clusterName: "testcluster",
eventRecorder: eventstesting.NewTestingEventRecorder(t),
}

err := awsProvider.CleanUpSubmarinerClusterEnv()
Expand Down
4 changes: 2 additions & 2 deletions pkg/cloud/cloud.go
View file
Expand Up @@ -46,15 +46,15 @@ func GetCloudProvider(
eventsRecorder,
region, infraId, clusterName, config.Spec.CredentialsSecret.Name,
config.Spec.GatewayConfig.AWS.InstanceType,
config.Spec.IPSecIKEPort, config.Spec.IPSecNATTPort,
config.Spec.IPSecIKEPort, config.Spec.IPSecNATTPort, config.Spec.NATTDiscoveryPort,
config.Spec.Gateways,
)
case "GCP":
return gcp.NewGCPProvider(
kubeClient,
eventsRecorder,
infraId, clusterName, config.Spec.CredentialsSecret.Name,
config.Spec.IPSecIKEPort, config.Spec.IPSecNATTPort,
config.Spec.IPSecIKEPort, config.Spec.IPSecNATTPort, config.Spec.NATTDiscoveryPort,
)
}
return nil, fmt.Errorf("unsupported cloud platform %q of cluster %q", platform, clusterName)
Expand Down

0 comments on commit e42ea77

Please sign in to comment.