From 2b40460ddee532890903fe0913efa4956e065a1d Mon Sep 17 00:00:00 2001 From: nikimanoledaki <18622989+nikimanoledaki@users.noreply.github.com> Date: Tue, 5 Oct 2021 13:02:08 +0200 Subject: [PATCH 1/9] Implement default workflow to add IPv6 VPC resources - Add integration test for ipv6 - Renamed VPC files to IPv4 file, and added IPv6 VPC files. - Created new resource set for IPv6 - Added all of the resources to create IPv6 VPC to CF template - Resources being created: VPC, IPv4 CIDR, IPv6 CIDR, EOIGW, NAT GW, IGW, route tables, routes, private and public subnets, route table associations - Added outputs to CF template for VPC and public/private subnets - Added integration and unit tests Co-authored-by: Jake Klein Refactoring VPC template creation - Making addResources and addOutputs private - Creating new CreateTemplate function for both IPv4 and IPv6 --- integration/tests/ipv6/ipv6_test.go | 109 +++ pkg/cfn/builder/builder_suite_test.go | 21 +- pkg/cfn/builder/cluster.go | 21 +- pkg/cfn/builder/cluster_test.go | 42 ++ pkg/cfn/builder/fakes/fake_cfn_template.go | 37 +- pkg/cfn/builder/vpc.go | 669 +----------------- pkg/cfn/builder/vpc_endpoint_test.go | 9 +- pkg/cfn/builder/vpc_ipv4.go | 643 +++++++++++++++++ .../builder/{vpc_test.go => vpc_ipv4_test.go} | 62 +- pkg/cfn/builder/vpc_ipv6.go | 192 +++++ pkg/cfn/builder/vpc_ipv6_test.go | 386 ++++++++++ pkg/vpc/vpc.go | 2 +- 12 files changed, 1483 insertions(+), 710 deletions(-) create mode 100644 integration/tests/ipv6/ipv6_test.go create mode 100644 pkg/cfn/builder/vpc_ipv4.go rename pkg/cfn/builder/{vpc_test.go => vpc_ipv4_test.go} (93%) create mode 100644 pkg/cfn/builder/vpc_ipv6.go create mode 100644 pkg/cfn/builder/vpc_ipv6_test.go diff --git a/integration/tests/ipv6/ipv6_test.go b/integration/tests/ipv6/ipv6_test.go new file mode 100644 index 0000000000..c022ac0ada --- /dev/null +++ b/integration/tests/ipv6/ipv6_test.go @@ -0,0 +1,109 @@ +//go:build integration +// +build integration + +package ipv6 + +import ( + "bytes" + "encoding/json" + "fmt" + "testing" + + "github.com/aws/aws-sdk-go/aws" + cfn "github.com/aws/aws-sdk-go/service/cloudformation" + awsec2 "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/weaveworks/eksctl/integration/matchers" + . "github.com/weaveworks/eksctl/integration/runner" + "github.com/weaveworks/eksctl/integration/tests" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/testutils" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var params *tests.Params + +func init() { + // Call testing.Init() prior to tests.NewParams(), as otherwise -test.* will not be recognised. See also: https://golang.org/doc/go1.13#testing + testing.Init() + params = tests.NewParams("IPv6") +} + +func TestIPv6(t *testing.T) { + testutils.RegisterAndRun(t) +} + +var _ = Describe("(Integration) [EKS IPv6 test]", func() { + + Context("Creating a cluster with IPv6", func() { + clusterName := params.NewClusterName("ipv6") + + BeforeSuite(func() { + clusterConfig := api.NewClusterConfig() + clusterConfig.Metadata.Name = clusterName + clusterConfig.Metadata.Version = "latest" + clusterConfig.Metadata.Region = params.Region + clusterConfig.VPC.IPFamily = aws.String("IPv6") + clusterConfig.IAM.WithOIDC = api.Enabled() + clusterConfig.Addons = []*api.Addon{ + { + Name: "vpc-cni", + }, + { + Name: "kube-proxy", + }, + { + Name: "coredns", + }, + } + + data, err := json.Marshal(clusterConfig) + Expect(err).ToNot(HaveOccurred()) + + cmd := params.EksctlCreateCmd. + WithArgs( + "cluster", + "--config-file", "-", + "--verbose", "4", + ). + WithoutArg("--region", params.Region). + WithStdin(bytes.NewReader(data)) + Expect(cmd).To(RunSuccessfully()) + }) + + AfterSuite(func() { + cmd := params.EksctlDeleteCmd.WithArgs( + "cluster", clusterName, + "--verbose", "2", + ) + Expect(cmd).To(RunSuccessfully()) + }) + + It("should support ipv6", func() { + By("Asserting that the VPC that is created has a IPv6 CIDR") + awsSession := NewSession(params.Region) + cfnSession := cfn.New(awsSession) + + var describeStackOut *cfn.DescribeStacksOutput + describeStackOut, err := cfnSession.DescribeStacks(&cfn.DescribeStacksInput{ + StackName: aws.String(fmt.Sprintf("eksctl-%s-cluster", clusterName)), + }) + Expect(err).NotTo(HaveOccurred()) + + var vpcID string + for _, output := range describeStackOut.Stacks[0].Outputs { + if *output.OutputKey == "VPC" { + vpcID = *output.OutputValue + } + } + + ec2 := awsec2.New(awsSession) + output, err := ec2.DescribeVpcs(&awsec2.DescribeVpcsInput{ + VpcIds: aws.StringSlice([]string{vpcID}), + }) + Expect(err).NotTo(HaveOccurred(), output.GoString()) + Expect(output.Vpcs[0].Ipv6CidrBlockAssociationSet).To(HaveLen(1)) + }) + }) +}) diff --git a/pkg/cfn/builder/builder_suite_test.go b/pkg/cfn/builder/builder_suite_test.go index 880cd90014..ad3860e933 100644 --- a/pkg/cfn/builder/builder_suite_test.go +++ b/pkg/cfn/builder/builder_suite_test.go @@ -17,16 +17,17 @@ func TestCfnBuilder(t *testing.T) { } var ( - azA, azB = "us-west-2a", "us-west-2b" - privateSubnet1, privateSubnet2 = "subnet-0ade11bad78dced9f", "subnet-0f98135715dfcf55a" - publicSubnet1, publicSubnet2 = "subnet-0ade11bad78dced9e", "subnet-0f98135715dfcf55f" - privateSubnetRef1, privateSubnetRef2 = "SubnetPrivateUSWEST2A", "SubnetPrivateUSWEST2B" - publicSubnetRef1, publicSubnetRef2 = "SubnetPublicUSWEST2A", "SubnetPublicUSWEST2B" - vpcResourceKey, igwKey, gaKey = "VPC", "InternetGateway", "VPCGatewayAttachment" - pubRouteTable, pubSubnetRoute = "PublicRouteTable", "PublicSubnetRoute" - privRouteTableA, privRouteTableB = "PrivateRouteTableUSWEST2A", "PrivateRouteTableUSWEST2B" - rtaPublicA, rtaPublicB = "RouteTableAssociationPublicUSWEST2A", "RouteTableAssociationPublicUSWEST2B" - rtaPrivateA, rtaPrivateB = "RouteTableAssociationPrivateUSWEST2A", "RouteTableAssociationPrivateUSWEST2B" + azA, azB, azC = "us-west-2a", "us-west-2b", "us-west-2c" + azAFormatted, azBFormatted, azCFormatted = "USWEST2A", "USWEST2B", "USWEST2C" + privateSubnet1, privateSubnet2 = "subnet-0ade11bad78dced9f", "subnet-0f98135715dfcf55a" + publicSubnet1, publicSubnet2 = "subnet-0ade11bad78dced9e", "subnet-0f98135715dfcf55f" + privateSubnetRef1, privateSubnetRef2 = "SubnetPrivateUSWEST2A", "SubnetPrivateUSWEST2B" + publicSubnetRef1, publicSubnetRef2 = "SubnetPublicUSWEST2A", "SubnetPublicUSWEST2B" + vpcResourceKey, igwKey, gaKey = "VPC", "InternetGateway", "VPCGatewayAttachment" + pubRouteTable, pubSubnetRoute = "PublicRouteTable", "PublicSubnetRoute" + privRouteTableA, privRouteTableB = "PrivateRouteTableUSWEST2A", "PrivateRouteTableUSWEST2B" + rtaPublicA, rtaPublicB = "RouteTableAssociationPublicUSWEST2A", "RouteTableAssociationPublicUSWEST2B" + rtaPrivateA, rtaPrivateB = "RouteTableAssociationPrivateUSWEST2A", "RouteTableAssociationPrivateUSWEST2B" ) func vpcConfig() *api.ClusterVPC { diff --git a/pkg/cfn/builder/cluster.go b/pkg/cfn/builder/cluster.go index e8b9a4d045..47ec6160a1 100644 --- a/pkg/cfn/builder/cluster.go +++ b/pkg/cfn/builder/cluster.go @@ -24,7 +24,7 @@ type ClusterResourceSet struct { ec2API ec2iface.EC2API region string supportsManagedNodes bool - vpcResourceSet *VPCResourceSet + vpcResourceSet VPCResourceSet securityGroups []*gfnt.Value } @@ -34,13 +34,19 @@ func NewClusterResourceSet(ec2API ec2iface.EC2API, region string, spec *api.Clus unsetExistingResources(existingStack, spec) } rs := newResourceSet() + + var vpcResourceSet VPCResourceSet + vpcResourceSet = NewIPv4VPCResourceSet(rs, spec, ec2API) + if spec.VPC.IPFamily != nil && *spec.VPC.IPFamily == string(api.IPV6Family) { + vpcResourceSet = NewIPv6VPCResourceSet(rs, spec, ec2API) + } return &ClusterResourceSet{ rs: rs, spec: spec, ec2API: ec2API, region: region, supportsManagedNodes: supportsManagedNodes, - vpcResourceSet: NewVPCResourceSet(rs, spec, ec2API), + vpcResourceSet: vpcResourceSet, } } @@ -50,16 +56,15 @@ func (c *ClusterResourceSet) AddAllResources() error { return err } - vpcResource, err := c.vpcResourceSet.AddResources() + vpcID, subnetDetails, err := c.vpcResourceSet.CreateTemplate() if err != nil { return errors.Wrap(err, "error adding VPC resources") } - c.vpcResourceSet.AddOutputs() - clusterSG := c.addResourcesForSecurityGroups(vpcResource) + clusterSG := c.addResourcesForSecurityGroups(vpcID) if privateCluster := c.spec.PrivateCluster; privateCluster.Enabled { - vpcEndpointResourceSet := NewVPCEndpointResourceSet(c.ec2API, c.region, c.rs, c.spec, vpcResource.VPC, vpcResource.SubnetDetails.Private, clusterSG.ClusterSharedNode) + vpcEndpointResourceSet := NewVPCEndpointResourceSet(c.ec2API, c.region, c.rs, c.spec, vpcID, subnetDetails.Private, clusterSG.ClusterSharedNode) if err := vpcEndpointResourceSet.AddResources(); err != nil { return errors.Wrap(err, "error adding resources for VPC endpoints") @@ -67,7 +72,7 @@ func (c *ClusterResourceSet) AddAllResources() error { } c.addResourcesForIAM() - c.addResourcesForControlPlane(vpcResource.SubnetDetails) + c.addResourcesForControlPlane(subnetDetails) if len(c.spec.FargateProfiles) > 0 { c.addResourcesForFargate() @@ -132,7 +137,7 @@ func (c *ClusterResourceSet) newResource(name string, resource gfn.Resource) *gf return c.rs.newResource(name, resource) } -func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *subnetDetails) { +func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDetails) { clusterVPC := &gfneks.Cluster_ResourcesVpcConfig{ SecurityGroupIds: gfnt.NewSlice(c.securityGroups...), } diff --git a/pkg/cfn/builder/cluster_test.go b/pkg/cfn/builder/cluster_test.go index 51e25faa7a..16cd3d8ff7 100644 --- a/pkg/cfn/builder/cluster_test.go +++ b/pkg/cfn/builder/cluster_test.go @@ -3,6 +3,7 @@ package builder_test import ( "encoding/json" + "github.com/aws/aws-sdk-go/aws" cfn "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo" @@ -32,6 +33,7 @@ var _ = Describe("Cluster Template Builder", func() { cfg = api.NewClusterConfig() cfg.VPC = vpcConfig() cfg.AvailabilityZones = []string{"us-west-2a", "us-west-2b"} + cfg.VPC.IPFamily = aws.String(string(api.IPV4Family)) }) JustBeforeEach(func() { @@ -80,6 +82,46 @@ var _ = Describe("Cluster Template Builder", func() { Expect(clusterTemplate.Resources).To(HaveKey(privateSubnetRef1)) }) + Context("when ipFamily is set to IPv6", func() { + BeforeEach(func() { + cfg.VPC.IPFamily = aws.String(string(api.IPV6Family)) + }) + + It("should add IPv6 vpc resources", func() { + Expect(clusterTemplate.Resources).To(HaveKey(builder.VPCResourceKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.IPv6CIDRBlockKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.IGWKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.GAKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.EgressOnlyInternetGatewayKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.NATGatewayKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.ElasticIPKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubRouteTableKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubSubRouteKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubSubIPv6RouteKey)) + privateRouteTableA := builder.PrivateRouteTableKey + azAFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteTableA)) + privateRouteTableB := builder.PrivateRouteTableKey + azBFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteTableB)) + privateRouteA := builder.PrivateSubnetRouteKey + azAFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteA)) + privateRouteB := builder.PrivateSubnetRouteKey + azBFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteB)) + privateRouteA = builder.PrivateSubnetIpv6RouteKey + azAFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteA)) + privateRouteB = builder.PrivateSubnetIpv6RouteKey + azBFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteB)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PublicSubnetKey + azAFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PublicSubnetKey + azBFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateSubnetKey + azAFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateSubnetKey + azBFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubRouteTableAssociation + azAFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubRouteTableAssociation + azBFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateRouteTableAssociation + azAFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateRouteTableAssociation + azBFormatted)) + + }) + }) + Context("when AutoAllocateIPv6 is enabled", func() { BeforeEach(func() { autoAllocated := true diff --git a/pkg/cfn/builder/fakes/fake_cfn_template.go b/pkg/cfn/builder/fakes/fake_cfn_template.go index 1462583f9b..49c8e07cb6 100644 --- a/pkg/cfn/builder/fakes/fake_cfn_template.go +++ b/pkg/cfn/builder/fakes/fake_cfn_template.go @@ -1,9 +1,5 @@ package fakes -import ( - cfn "github.com/aws/aws-sdk-go/service/cloudformation" -) - type FakeTemplate struct { Description string Resources map[string]struct { @@ -13,7 +9,7 @@ type FakeTemplate struct { UpdatePolicy map[string]map[string]interface{} } Mappings map[string]interface{} - Outputs map[string]cfn.Output + Outputs interface{} } type Tag struct { @@ -24,13 +20,14 @@ type Tag struct { } type Properties struct { - GroupDescription string - Description string - Tags []Tag - SecurityGroupIngress []SGIngress - GroupID interface{} - SourceSecurityGroupID interface{} - DestinationSecurityGroupID interface{} + EnableDnsHostnames, EnableDnsSupport bool + GroupDescription string + Description string + Tags []Tag + SecurityGroupIngress []SGIngress + GroupID interface{} + SourceSecurityGroupID interface{} + DestinationSecurityGroupID interface{} Path, RoleName string Roles, ManagedPolicyArns []interface{} @@ -62,16 +59,18 @@ type Properties struct { CidrIP, CidrIpv6, IPProtocol string FromPort, ToPort int - VpcID, SubnetID interface{} - RouteTableID, AllocationID interface{} - GatewayID, InternetGatewayID, NatGatewayID interface{} - DestinationCidrBlock interface{} - MapPublicIPOnLaunch bool + VpcID, SubnetID interface{} + EgressOnlyInternetGatewayID, RouteTableID, AllocationID interface{} + GatewayID, InternetGatewayID, NatGatewayID interface{} + DestinationCidrBlock, DestinationIpv6CidrBlock interface{} + MapPublicIPOnLaunch bool + AssignIpv6AddressOnCreation *bool Ipv6CidrBlock map[string][]interface{} + CidrBlock interface{} - AmazonProvidedIpv6CidrBlock bool - AvailabilityZone, Domain, CidrBlock string + AmazonProvidedIpv6CidrBlock bool + AvailabilityZone, Domain string Name, Version string RoleArn interface{} diff --git a/pkg/cfn/builder/vpc.go b/pkg/cfn/builder/vpc.go index 2129f54b8b..7fe4d18b0f 100644 --- a/pkg/cfn/builder/vpc.go +++ b/pkg/cfn/builder/vpc.go @@ -1,655 +1,50 @@ package builder import ( - "fmt" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/pkg/errors" - gfncfn "github.com/weaveworks/goformation/v4/cloudformation/cloudformation" - gfnec2 "github.com/weaveworks/goformation/v4/cloudformation/ec2" gfnt "github.com/weaveworks/goformation/v4/cloudformation/types" - - api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" - "github.com/weaveworks/eksctl/pkg/cfn/outputs" - "github.com/weaveworks/eksctl/pkg/vpc" ) -var internetCIDR = gfnt.NewString("0.0.0.0/0") - const ( - cfnControlPlaneSGResource = "ControlPlaneSecurityGroup" - cfnSharedNodeSGResource = "ClusterSharedNodeSecurityGroup" - cfnIngressClusterToNodeSGResource = "IngressDefaultClusterToNodeSG" -) - -// A VPCResourceSet builds the resources required for the specified VPC -type VPCResourceSet struct { - rs *resourceSet - clusterConfig *api.ClusterConfig - ec2API ec2iface.EC2API - - vpcResource *VPCResource -} - -// VPCResource represents a VPC resource -type VPCResource struct { - VPC *gfnt.Value - SubnetDetails *subnetDetails -} - -type SubnetResource struct { - Subnet *gfnt.Value - RouteTable *gfnt.Value - AvailabilityZone string -} - -type subnetDetails struct { - Private []SubnetResource - Public []SubnetResource -} - -// NewVPCResourceSet creates and returns a new VPCResourceSet -func NewVPCResourceSet(rs *resourceSet, clusterConfig *api.ClusterConfig, ec2API ec2iface.EC2API) *VPCResourceSet { - var vpcRef *gfnt.Value - if clusterConfig.VPC.ID == "" { - vpcRef = rs.newResource("VPC", &gfnec2.VPC{ - CidrBlock: gfnt.NewString(clusterConfig.VPC.CIDR.String()), - EnableDnsSupport: gfnt.True(), - EnableDnsHostnames: gfnt.True(), - }) - } else { - vpcRef = gfnt.NewString(clusterConfig.VPC.ID) - } - - return &VPCResourceSet{ - rs: rs, - clusterConfig: clusterConfig, - ec2API: ec2API, - - vpcResource: &VPCResource{ - VPC: vpcRef, - SubnetDetails: &subnetDetails{}, - }, - } -} - -// AddResources adds all required resources -func (v *VPCResourceSet) AddResources() (*VPCResource, error) { - vpc := v.clusterConfig.VPC - if vpc.ID != "" { // custom VPC has been set - if err := v.importResources(); err != nil { - return nil, errors.Wrap(err, "error importing VPC resources") - } - return v.vpcResource, nil - } - - if api.IsEnabled(vpc.AutoAllocateIPv6) { - v.rs.newResource("AutoAllocatedCIDRv6", &gfnec2.VPCCidrBlock{ - VpcId: v.vpcResource.VPC, - AmazonProvidedIpv6CidrBlock: gfnt.True(), - }) - } - - if v.isFullyPrivate() { - v.noNAT() - v.vpcResource.SubnetDetails.Private = v.addSubnets(nil, api.SubnetTopologyPrivate, vpc.Subnets.Private) - return v.vpcResource, nil - } - - refIG := v.rs.newResource("InternetGateway", &gfnec2.InternetGateway{}) - vpcGA := "VPCGatewayAttachment" - v.rs.newResource(vpcGA, &gfnec2.VPCGatewayAttachment{ - InternetGatewayId: refIG, - VpcId: v.vpcResource.VPC, - }) - - refPublicRT := v.rs.newResource("PublicRouteTable", &gfnec2.RouteTable{ - VpcId: v.vpcResource.VPC, - }) - - v.rs.newResource("PublicSubnetRoute", &gfnec2.Route{ - RouteTableId: refPublicRT, - DestinationCidrBlock: internetCIDR, - GatewayId: refIG, - AWSCloudFormationDependsOn: []string{vpcGA}, - }) - - v.vpcResource.SubnetDetails.Public = v.addSubnets(refPublicRT, api.SubnetTopologyPublic, vpc.Subnets.Public) - - if err := v.addNATGateways(); err != nil { - return nil, err - } - - v.vpcResource.SubnetDetails.Private = v.addSubnets(nil, api.SubnetTopologyPrivate, vpc.Subnets.Private) - return v.vpcResource, nil -} - -func (s *subnetDetails) PublicSubnetRefs() []*gfnt.Value { - var subnetRefs []*gfnt.Value - for _, subnetAZ := range s.Public { - subnetRefs = append(subnetRefs, subnetAZ.Subnet) - } - return subnetRefs -} - -func (s *subnetDetails) PrivateSubnetRefs() []*gfnt.Value { - var subnetRefs []*gfnt.Value - for _, subnetAZ := range s.Private { - subnetRefs = append(subnetRefs, subnetAZ.Subnet) - } - return subnetRefs -} - -// AddOutputs adds VPC resource outputs -func (v *VPCResourceSet) AddOutputs() { - v.rs.defineOutput(outputs.ClusterVPC, v.vpcResource.VPC, true, func(val string) error { - v.clusterConfig.VPC.ID = val - return nil - }) - if v.clusterConfig.VPC.NAT != nil { - v.rs.defineOutputWithoutCollector(outputs.ClusterFeatureNATMode, v.clusterConfig.VPC.NAT.Gateway, false) - } - - addSubnetOutput := func(subnetRefs []*gfnt.Value, topology api.SubnetTopology, outputName string) { - v.rs.defineJoinedOutput(outputName, subnetRefs, true, func(value string) error { - return vpc.ImportSubnetsFromIDList(v.ec2API, v.clusterConfig, topology, strings.Split(value, ",")) - }) - } - - if subnetAZs := v.vpcResource.SubnetDetails.PrivateSubnetRefs(); len(subnetAZs) > 0 { - addSubnetOutput(subnetAZs, api.SubnetTopologyPrivate, outputs.ClusterSubnetsPrivate) - } - - if subnetAZs := v.vpcResource.SubnetDetails.PublicSubnetRefs(); len(subnetAZs) > 0 { - addSubnetOutput(subnetAZs, api.SubnetTopologyPublic, outputs.ClusterSubnetsPublic) - } - - if v.isFullyPrivate() { - v.rs.defineOutputWithoutCollector(outputs.ClusterFullyPrivate, true, true) - } -} - -// RenderJSON returns the rendered JSON -func (v *VPCResourceSet) RenderJSON() ([]byte, error) { - return v.rs.renderJSON() -} - -func (v *VPCResourceSet) addSubnets(refRT *gfnt.Value, topology api.SubnetTopology, subnets map[string]api.AZSubnetSpec) []SubnetResource { - var subnetIndexForIPv6 int - if api.IsEnabled(v.clusterConfig.VPC.AutoAllocateIPv6) { - // this is same kind of indexing we have in vpc.SetSubnets - switch topology { - case api.SubnetTopologyPrivate: - subnetIndexForIPv6 = len(v.clusterConfig.AvailabilityZones) - case api.SubnetTopologyPublic: - subnetIndexForIPv6 = 0 - } - } - - var subnetResources []SubnetResource - - for name, subnet := range subnets { - az := subnet.AZ - nameAlias := strings.ToUpper(strings.Join(strings.Split(name, "-"), "")) - subnet := &gfnec2.Subnet{ - AvailabilityZone: gfnt.NewString(az), - CidrBlock: gfnt.NewString(subnet.CIDR.String()), - VpcId: v.vpcResource.VPC, - } - - switch topology { - case api.SubnetTopologyPrivate: - // Choose the appropriate route table for private subnets - refRT = gfnt.MakeRef("PrivateRouteTable" + nameAlias) - subnet.Tags = []gfncfn.Tag{{ - Key: gfnt.NewString("kubernetes.io/role/internal-elb"), - Value: gfnt.NewString("1"), - }} - case api.SubnetTopologyPublic: - subnet.Tags = []gfncfn.Tag{{ - Key: gfnt.NewString("kubernetes.io/role/elb"), - Value: gfnt.NewString("1"), - }} - subnet.MapPublicIpOnLaunch = gfnt.True() - } - subnetAlias := string(topology) + nameAlias - refSubnet := v.rs.newResource("Subnet"+subnetAlias, subnet) - v.rs.newResource("RouteTableAssociation"+subnetAlias, &gfnec2.SubnetRouteTableAssociation{ - SubnetId: refSubnet, - RouteTableId: refRT, - }) - - if api.IsEnabled(v.clusterConfig.VPC.AutoAllocateIPv6) { - // get 8 of /64 subnets from the auto-allocated IPv6 block, - // and pick one block based on subnetIndexForIPv6 counter; - // NOTE: this is done inside of CloudFormation using Fn::Cidr, - // we don't slice it here, just construct the JSON expression - // that does slicing at runtime. - refAutoAllocateCIDRv6 := gfnt.MakeFnSelect( - gfnt.NewInteger(0), gfnt.MakeFnGetAttString("VPC", "Ipv6CidrBlocks"), - ) - refSubnetSlices := gfnt.MakeFnCIDR( - refAutoAllocateCIDRv6, gfnt.NewInteger(8), gfnt.NewInteger(64), - ) - v.rs.newResource(subnetAlias+"CIDRv6", &gfnec2.SubnetCidrBlock{ - SubnetId: refSubnet, - Ipv6CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(subnetIndexForIPv6), refSubnetSlices), - }) - subnetIndexForIPv6++ - } - - subnetResources = append(subnetResources, SubnetResource{ - AvailabilityZone: az, - RouteTable: refRT, - Subnet: refSubnet, - }) - } - return subnetResources -} - -func (v *VPCResourceSet) addNATGateways() error { - switch *v.clusterConfig.VPC.NAT.Gateway { - case api.ClusterHighlyAvailableNAT: - v.haNAT() - case api.ClusterSingleNAT: - v.singleNAT() - case api.ClusterDisableNAT: - v.noNAT() - default: - // TODO validate this before starting to add resources - return fmt.Errorf("%s is not a valid NAT gateway mode", *v.clusterConfig.VPC.NAT.Gateway) - } - return nil -} - -func (v *VPCResourceSet) importResources() error { - if subnets := v.clusterConfig.VPC.Subnets.Private; subnets != nil { - var ( - subnetRoutes map[string]string - err error - ) - if v.isFullyPrivate() { - subnetRoutes, err = importRouteTables(v.ec2API, v.clusterConfig.VPC.Subnets.Private) - if err != nil { - return err - } - } - - subnetResources, err := makeSubnetResources(subnets, subnetRoutes) - if err != nil { - return err - } - v.vpcResource.SubnetDetails.Private = subnetResources - } - - if subnets := v.clusterConfig.VPC.Subnets.Public; subnets != nil { - subnetResources, err := makeSubnetResources(subnets, nil) - if err != nil { - return err - } - v.vpcResource.SubnetDetails.Public = subnetResources - } - - return nil -} - -func makeSubnetResources(subnets map[string]api.AZSubnetSpec, subnetRoutes map[string]string) ([]SubnetResource, error) { - subnetResources := make([]SubnetResource, len(subnets)) - i := 0 - for _, network := range subnets { - az := network.AZ - sr := SubnetResource{ - AvailabilityZone: az, - Subnet: gfnt.NewString(network.ID), - } - - if subnetRoutes != nil { - rt, ok := subnetRoutes[network.ID] - if !ok { - return nil, errors.Errorf("failed to find an explicit route table associated with subnet %q; "+ - "eksctl does not modify the main route table if a subnet is not associated with an explicit route table", network.ID) - } - sr.RouteTable = gfnt.NewString(rt) - } - subnetResources[i] = sr - i++ - } - return subnetResources, nil -} - -func importRouteTables(ec2API ec2iface.EC2API, subnets map[string]api.AZSubnetSpec) (map[string]string, error) { - var subnetIDs []string - for _, subnet := range subnets { - subnetIDs = append(subnetIDs, subnet.ID) - } - - var routeTables []*ec2.RouteTable - var nextToken *string - - for { - output, err := ec2API.DescribeRouteTables(&ec2.DescribeRouteTablesInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("association.subnet-id"), - Values: aws.StringSlice(subnetIDs), - }, - }, - NextToken: nextToken, - }) - - if err != nil { - return nil, errors.Wrap(err, "error describing route tables") - } - - routeTables = append(routeTables, output.RouteTables...) - - if nextToken = output.NextToken; nextToken == nil { - break - } - } - - subnetRoutes := make(map[string]string) - for _, rt := range routeTables { - for _, rta := range rt.Associations { - if rta.Main != nil && *rta.Main { - return nil, errors.New("subnets must be associated with a non-main route table; eksctl does not modify the main route table") - } - subnetRoutes[*rta.SubnetId] = *rt.RouteTableId - } - } - return subnetRoutes, nil -} - -func (v *VPCResourceSet) isFullyPrivate() bool { - return v.clusterConfig.PrivateCluster.Enabled -} - -var ( - sgProtoTCP = gfnt.NewString("tcp") - sgSourceAnywhereIPv4 = gfnt.NewString("0.0.0.0/0") - sgSourceAnywhereIPv6 = gfnt.NewString("::/0") - - sgPortZero = gfnt.NewInteger(0) - sgMinNodePort = gfnt.NewInteger(1025) - sgMaxNodePort = gfnt.NewInteger(65535) - - sgPortHTTPS = gfnt.NewInteger(443) - sgPortSSH = gfnt.NewInteger(22) + VPCResourceKey, IGWKey, GAKey = "VPC", "InternetGateway", "VPCGatewayAttachment" + IPv6CIDRBlockKey = "IPv6CidrBlock" + EgressOnlyInternetGatewayKey = "EgressOnlyInternetGateway" + ElasticIPKey = "EIP" + InternetCIDR, InternetIPv6CIDR = "0.0.0.0/0", "::/0" + PubRouteTableKey, PrivateRouteTableKey = "PublicRouteTable", "PrivateRouteTable" + PubRouteTableAssociation, PrivateRouteTableAssociation = "RouteTableAssociationPublic", "RouteTableAssociationPrivate" + PubSubRouteKey, PubSubIPv6RouteKey = "PublicSubnetDefaultRoute", "PublicSubnetIPv6DefaultRoute" + PrivateSubnetRouteKey, PrivateSubnetIpv6RouteKey = "PrivateSubnetDefaultRoute", "PrivateSubnetDefaultIpv6Route" + PublicSubnetKey, PrivateSubnetKey = "PublicSubnet", "PrivateSubnet" + NATGatewayKey = "NATGateway" + PublicSubnetsOutputKey, PrivateSubnetsOutputKey = "SubnetsPublic", "SubnetsPrivate" ) -type clusterSecurityGroup struct { - ControlPlane *gfnt.Value - ClusterSharedNode *gfnt.Value +type VPCResourceSet interface { + CreateTemplate() (*gfnt.Value, *SubnetDetails, error) } -// TODO move this -func (c *ClusterResourceSet) addResourcesForSecurityGroups(vpcResource *VPCResource) *clusterSecurityGroup { - var refControlPlaneSG, refClusterSharedNodeSG *gfnt.Value - - if c.spec.VPC.SecurityGroup == "" { - refControlPlaneSG = c.newResource(cfnControlPlaneSGResource, &gfnec2.SecurityGroup{ - GroupDescription: gfnt.NewString("Communication between the control plane and worker nodegroups"), - VpcId: vpcResource.VPC, - }) - - if len(c.spec.VPC.ExtraCIDRs) > 0 { - for i, cidr := range c.spec.VPC.ExtraCIDRs { - c.newResource(fmt.Sprintf("IngressControlPlaneExtraCIDR%d", i), &gfnec2.SecurityGroupIngress{ - GroupId: refControlPlaneSG, - CidrIp: gfnt.NewString(cidr), - Description: gfnt.NewString(fmt.Sprintf("Allow Extra CIDR %d (%s) to communicate to controlplane", i, cidr)), - IpProtocol: gfnt.NewString("tcp"), - FromPort: sgPortHTTPS, - ToPort: sgPortHTTPS, - }) - } - } - } else { - refControlPlaneSG = gfnt.NewString(c.spec.VPC.SecurityGroup) - } - c.securityGroups = []*gfnt.Value{refControlPlaneSG} // only this one SG is passed to EKS API, nodes are isolated - - if c.spec.VPC.SharedNodeSecurityGroup == "" { - refClusterSharedNodeSG = c.newResource(cfnSharedNodeSGResource, &gfnec2.SecurityGroup{ - GroupDescription: gfnt.NewString("Communication between all nodes in the cluster"), - VpcId: vpcResource.VPC, - }) - c.newResource("IngressInterNodeGroupSG", &gfnec2.SecurityGroupIngress{ - GroupId: refClusterSharedNodeSG, - SourceSecurityGroupId: refClusterSharedNodeSG, - Description: gfnt.NewString("Allow nodes to communicate with each other (all ports)"), - IpProtocol: gfnt.NewString("-1"), - FromPort: sgPortZero, - ToPort: sgMaxNodePort, - }) - } else { - refClusterSharedNodeSG = gfnt.NewString(c.spec.VPC.SharedNodeSecurityGroup) - } - - if c.supportsManagedNodes && api.IsEnabled(c.spec.VPC.ManageSharedNodeSecurityGroupRules) { - // To enable communication between both managed and unmanaged nodegroups, this allows ingress traffic from - // the default cluster security group ID that EKS creates by default - // EKS attaches this to Managed Nodegroups by default, but we need to handle this for unmanaged nodegroups - c.newResource(cfnIngressClusterToNodeSGResource, &gfnec2.SecurityGroupIngress{ - GroupId: refClusterSharedNodeSG, - SourceSecurityGroupId: gfnt.MakeFnGetAttString("ControlPlane", outputs.ClusterDefaultSecurityGroup), - Description: gfnt.NewString("Allow managed and unmanaged nodes to communicate with each other (all ports)"), - IpProtocol: gfnt.NewString("-1"), - FromPort: sgPortZero, - ToPort: sgMaxNodePort, - }) - c.newResource("IngressNodeToDefaultClusterSG", &gfnec2.SecurityGroupIngress{ - GroupId: gfnt.MakeFnGetAttString("ControlPlane", outputs.ClusterDefaultSecurityGroup), - SourceSecurityGroupId: refClusterSharedNodeSG, - Description: gfnt.NewString("Allow unmanaged nodes to communicate with control plane (all ports)"), - IpProtocol: gfnt.NewString("-1"), - FromPort: sgPortZero, - ToPort: sgMaxNodePort, - }) - } - - if c.spec.VPC == nil { - c.spec.VPC = &api.ClusterVPC{} - } - c.rs.defineOutput(outputs.ClusterSecurityGroup, refControlPlaneSG, true, func(v string) error { - c.spec.VPC.SecurityGroup = v - return nil - }) - c.rs.defineOutput(outputs.ClusterSharedNodeSecurityGroup, refClusterSharedNodeSG, true, func(v string) error { - c.spec.VPC.SharedNodeSecurityGroup = v - return nil - }) - - return &clusterSecurityGroup{ - ControlPlane: refControlPlaneSG, - ClusterSharedNode: refClusterSharedNodeSG, - } +func formatAZ(az string) string { + return strings.ToUpper(strings.ReplaceAll(az, "-", "")) } -// TODO move this -func (rs *resourceSet) addEFASecurityGroup(vpcID *gfnt.Value, clusterName, desc string) *gfnt.Value { - efaSG := rs.newResource("EFASG", &gfnec2.SecurityGroup{ - VpcId: vpcID, - GroupDescription: gfnt.NewString("EFA-enabled security group"), - Tags: []gfncfn.Tag{{ - Key: gfnt.NewString("kubernetes.io/cluster/" + clusterName), - Value: gfnt.NewString("owned"), - }}, - }) - rs.newResource("EFAIngressSelf", &gfnec2.SecurityGroupIngress{ - GroupId: efaSG, - SourceSecurityGroupId: efaSG, - Description: gfnt.NewString("Allow " + desc + " to communicate to itself (EFA-enabled)"), - IpProtocol: gfnt.NewString("-1"), - }) - rs.newResource("EFAEgressSelf", &gfnec2.SecurityGroupEgress{ - GroupId: efaSG, - DestinationSecurityGroupId: efaSG, - Description: gfnt.NewString("Allow " + desc + " to communicate to itself (EFA-enabled)"), - IpProtocol: gfnt.NewString("-1"), - }) - - return efaSG -} - -// TODO move this -func (n *NodeGroupResourceSet) addResourcesForSecurityGroups() { - for _, id := range n.spec.SecurityGroups.AttachIDs { - n.securityGroups = append(n.securityGroups, gfnt.NewString(id)) - } - - if api.IsEnabled(n.spec.SecurityGroups.WithShared) { - n.securityGroups = append(n.securityGroups, n.vpcImporter.SharedNodeSecurityGroup()) - } - - if api.IsDisabled(n.spec.SecurityGroups.WithLocal) { - return - } - - desc := "worker nodes in group " + n.spec.Name - vpcID := n.vpcImporter.VPC() - refControlPlaneSG := n.vpcImporter.ControlPlaneSecurityGroup() - - refNodeGroupLocalSG := n.newResource("SG", &gfnec2.SecurityGroup{ - VpcId: vpcID, - GroupDescription: gfnt.NewString("Communication between the control plane and " + desc), - Tags: []gfncfn.Tag{{ - Key: gfnt.NewString("kubernetes.io/cluster/" + n.clusterSpec.Metadata.Name), - Value: gfnt.NewString("owned"), - }}, - SecurityGroupIngress: makeNodeIngressRules(n.spec.NodeGroupBase, refControlPlaneSG, n.clusterSpec.VPC.CIDR.String(), desc), - }) - - n.securityGroups = append(n.securityGroups, refNodeGroupLocalSG) - - if api.IsEnabled(n.spec.EFAEnabled) { - efaSG := n.rs.addEFASecurityGroup(vpcID, n.clusterSpec.Metadata.Name, desc) - n.securityGroups = append(n.securityGroups, efaSG) - } - - n.newResource("EgressInterCluster", &gfnec2.SecurityGroupEgress{ - GroupId: refControlPlaneSG, - DestinationSecurityGroupId: refNodeGroupLocalSG, - Description: gfnt.NewString("Allow control plane to communicate with " + desc + " (kubelet and workload TCP ports)"), - IpProtocol: sgProtoTCP, - FromPort: sgMinNodePort, - ToPort: sgMaxNodePort, - }) - n.newResource("EgressInterClusterAPI", &gfnec2.SecurityGroupEgress{ - GroupId: refControlPlaneSG, - DestinationSecurityGroupId: refNodeGroupLocalSG, - Description: gfnt.NewString("Allow control plane to communicate with " + desc + " (workloads using HTTPS port, commonly used with extension API servers)"), - IpProtocol: sgProtoTCP, - FromPort: sgPortHTTPS, - ToPort: sgPortHTTPS, - }) - n.newResource("IngressInterClusterCP", &gfnec2.SecurityGroupIngress{ - GroupId: refControlPlaneSG, - SourceSecurityGroupId: refNodeGroupLocalSG, - Description: gfnt.NewString("Allow control plane to receive API requests from " + desc), - IpProtocol: sgProtoTCP, - FromPort: sgPortHTTPS, - ToPort: sgPortHTTPS, - }) -} - -func makeNodeIngressRules(ng *api.NodeGroupBase, controlPlaneSG *gfnt.Value, vpcCIDR, description string) []gfnec2.SecurityGroup_Ingress { - ingressRules := []gfnec2.SecurityGroup_Ingress{ - { - SourceSecurityGroupId: controlPlaneSG, - Description: gfnt.NewString(fmt.Sprintf("[IngressInterCluster] Allow %s to communicate with control plane (kubelet and workload TCP ports)", description)), - IpProtocol: sgProtoTCP, - FromPort: sgMinNodePort, - ToPort: sgMaxNodePort, - }, - { - SourceSecurityGroupId: controlPlaneSG, - Description: gfnt.NewString(fmt.Sprintf("[IngressInterClusterAPI] Allow %s to communicate with control plane (workloads using HTTPS port, commonly used with extension API servers)", description)), - IpProtocol: sgProtoTCP, - FromPort: sgPortHTTPS, - ToPort: sgPortHTTPS, - }, - } - - return append(ingressRules, makeSSHIngressRules(ng, vpcCIDR, description)...) -} - -func (v *VPCResourceSet) haNAT() { - for _, az := range v.clusterConfig.AvailabilityZones { - alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) - - // Allocate an EIP - v.rs.newResource("NATIP"+alphanumericUpperAZ, &gfnec2.EIP{ - Domain: gfnt.NewString("vpc"), - }) - // Allocate a NAT gateway in the public subnet - refNG := v.rs.newResource("NATGateway"+alphanumericUpperAZ, &gfnec2.NatGateway{ - AllocationId: gfnt.MakeFnGetAttString("NATIP"+alphanumericUpperAZ, "AllocationId"), - SubnetId: gfnt.MakeRef("SubnetPublic" + alphanumericUpperAZ), - }) - - // Allocate a routing table for the private subnet - refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ - VpcId: v.vpcResource.VPC, - }) - // Create a route that sends Internet traffic through the NAT gateway - v.rs.newResource("NATPrivateSubnetRoute"+alphanumericUpperAZ, &gfnec2.Route{ - RouteTableId: refRT, - DestinationCidrBlock: internetCIDR, - NatGatewayId: refNG, - }) - // Associate the routing table with the subnet - v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ - SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), - RouteTableId: refRT, - }) - } -} - -func (v *VPCResourceSet) singleNAT() { - sortedAZs := v.clusterConfig.AvailabilityZones - firstUpperAZ := strings.ToUpper(strings.Join(strings.Split(sortedAZs[0], "-"), "")) - - v.rs.newResource("NATIP", &gfnec2.EIP{ - Domain: gfnt.NewString("vpc"), - }) - refNG := v.rs.newResource("NATGateway", &gfnec2.NatGateway{ - AllocationId: gfnt.MakeFnGetAttString("NATIP", "AllocationId"), - SubnetId: gfnt.MakeRef("SubnetPublic" + firstUpperAZ), - }) - - for _, az := range v.clusterConfig.AvailabilityZones { - alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) - - refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ - VpcId: v.vpcResource.VPC, - }) - - v.rs.newResource("NATPrivateSubnetRoute"+alphanumericUpperAZ, &gfnec2.Route{ - RouteTableId: refRT, - DestinationCidrBlock: internetCIDR, - NatGatewayId: refNG, - }) - v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ - SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), - RouteTableId: refRT, - }) - } +func getSubnetIPv6CIDRBlock(cidrPartitions int) *gfnt.Value { + // get 8 of /64 subnets from the auto-allocated IPv6 block, + // and pick one block based on subnetIndexForIPv6 counter; + // NOTE: this is done inside of CloudFormation using Fn::Cidr, + // we don't slice it here, just construct the JSON expression + // that does slicing at runtime. + refIPv6CIDRv6 := gfnt.MakeFnSelect( + gfnt.NewInteger(0), gfnt.MakeFnGetAttString("VPC", "Ipv6CidrBlocks"), + ) + refSubnetSlices := gfnt.MakeFnCIDR(refIPv6CIDRv6, gfnt.NewInteger(cidrPartitions), gfnt.NewInteger(64)) + return refSubnetSlices } -func (v *VPCResourceSet) noNAT() { - for _, az := range v.clusterConfig.AvailabilityZones { - alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) - - refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ - VpcId: v.vpcResource.VPC, - }) - v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ - SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), - RouteTableId: refRT, - }) - } +func getSubnetIPv4CIDRBlock(cidrPartitions int) *gfnt.Value { + //TODO: should we be doing /19? Should we adjust for the partition size? + desiredMask := 19 + refSubnetSlices := gfnt.MakeFnCIDR(gfnt.MakeFnGetAttString("VPC", "CidrBlock"), gfnt.NewInteger(cidrPartitions), gfnt.NewInteger(32-desiredMask)) + return refSubnetSlices } diff --git a/pkg/cfn/builder/vpc_endpoint_test.go b/pkg/cfn/builder/vpc_endpoint_test.go index 6a2a4152c2..e7028abddd 100644 --- a/pkg/cfn/builder/vpc_endpoint_test.go +++ b/pkg/cfn/builder/vpc_endpoint_test.go @@ -29,7 +29,7 @@ type vpcResourceSetCase struct { var _ = Describe("VPC Endpoint Builder", func() { - DescribeTable("Add resources", func(vc vpcResourceSetCase) { + DescribeTable("Adds resources to template", func(vc vpcResourceSetCase) { api.SetClusterConfigDefaults(vc.clusterConfig) if len(vc.clusterConfig.AvailabilityZones) == 0 { @@ -54,8 +54,8 @@ var _ = Describe("VPC Endpoint Builder", func() { } rs := newResourceSet() - vpcResourceSet := NewVPCResourceSet(rs, vc.clusterConfig, provider.EC2()) - vpcResource, err := vpcResourceSet.AddResources() + vpcResourceSet := NewIPv4VPCResourceSet(rs, vc.clusterConfig, provider.EC2()) + vpcID, subnetDetails, err := vpcResourceSet.CreateTemplate() if vc.err != "" { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("subnets must be associated with a non-main route table")) @@ -64,7 +64,7 @@ var _ = Describe("VPC Endpoint Builder", func() { Expect(err).ToNot(HaveOccurred()) if vc.clusterConfig.PrivateCluster.Enabled { - vpcEndpointResourceSet := NewVPCEndpointResourceSet(provider.EC2(), provider.Region(), rs, vc.clusterConfig, vpcResource.VPC, vpcResource.SubnetDetails.Private, gfnt.NewString("sg-test")) + vpcEndpointResourceSet := NewVPCEndpointResourceSet(provider.EC2(), provider.Region(), rs, vc.clusterConfig, vpcID, subnetDetails.Private, gfnt.NewString("sg-test")) Expect(vpcEndpointResourceSet.AddResources()).To(Succeed()) s3Endpoint := rs.template.Resources["VPCEndpointS3"].(*gfnec2.VPCEndpoint) routeIdsSlice, ok := s3Endpoint.RouteTableIds.Raw().(gfnt.Slice) @@ -77,6 +77,7 @@ var _ = Describe("VPC Endpoint Builder", func() { return } + rs.template.Outputs = nil resourceJSON, err := rs.template.JSON() Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/cfn/builder/vpc_ipv4.go b/pkg/cfn/builder/vpc_ipv4.go new file mode 100644 index 0000000000..801edc5e34 --- /dev/null +++ b/pkg/cfn/builder/vpc_ipv4.go @@ -0,0 +1,643 @@ +package builder + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/pkg/errors" + gfncfn "github.com/weaveworks/goformation/v4/cloudformation/cloudformation" + gfnec2 "github.com/weaveworks/goformation/v4/cloudformation/ec2" + gfnt "github.com/weaveworks/goformation/v4/cloudformation/types" + + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/outputs" + "github.com/weaveworks/eksctl/pkg/vpc" +) + +const ( + cfnControlPlaneSGResource = "ControlPlaneSecurityGroup" + cfnSharedNodeSGResource = "ClusterSharedNodeSecurityGroup" + cfnIngressClusterToNodeSGResource = "IngressDefaultClusterToNodeSG" +) + +// A IPv4VPCResourceSet builds the resources required for the specified VPC +type IPv4VPCResourceSet struct { + rs *resourceSet + clusterConfig *api.ClusterConfig + ec2API ec2iface.EC2API + vpcID *gfnt.Value + subnetDetails *SubnetDetails +} + +type SubnetResource struct { + Subnet *gfnt.Value + RouteTable *gfnt.Value + AvailabilityZone string +} + +type SubnetDetails struct { + Private []SubnetResource + Public []SubnetResource +} + +// NewIPv4VPCResourceSet creates and returns a new VPCResourceSet +func NewIPv4VPCResourceSet(rs *resourceSet, clusterConfig *api.ClusterConfig, ec2API ec2iface.EC2API) *IPv4VPCResourceSet { + var vpcRef *gfnt.Value + if clusterConfig.VPC.ID == "" { + vpcRef = rs.newResource("VPC", &gfnec2.VPC{ + CidrBlock: gfnt.NewString(clusterConfig.VPC.CIDR.String()), + EnableDnsSupport: gfnt.True(), + EnableDnsHostnames: gfnt.True(), + }) + } else { + vpcRef = gfnt.NewString(clusterConfig.VPC.ID) + } + + return &IPv4VPCResourceSet{ + rs: rs, + clusterConfig: clusterConfig, + ec2API: ec2API, + vpcID: vpcRef, + subnetDetails: &SubnetDetails{}, + } +} + +func (v *IPv4VPCResourceSet) CreateTemplate() (*gfnt.Value, *SubnetDetails, error) { + err := v.addResources() + if err != nil { + return nil, nil, err + } + v.addOutputs() + return v.vpcID, v.subnetDetails, nil +} + +// AddResources adds all required resources +func (v *IPv4VPCResourceSet) addResources() error { + vpc := v.clusterConfig.VPC + if vpc.ID != "" { // custom VPC has been set + if err := v.importResources(); err != nil { + return errors.Wrap(err, "error importing VPC resources") + } + return nil + } + + if api.IsEnabled(vpc.AutoAllocateIPv6) { + v.rs.newResource("AutoAllocatedCIDRv6", &gfnec2.VPCCidrBlock{ + VpcId: v.vpcID, + AmazonProvidedIpv6CidrBlock: gfnt.True(), + }) + } + + if v.isFullyPrivate() { + v.noNAT() + v.subnetDetails.Private = v.addSubnets(nil, api.SubnetTopologyPrivate, vpc.Subnets.Private) + return nil + } + + refIG := v.rs.newResource("InternetGateway", &gfnec2.InternetGateway{}) + vpcGA := "VPCGatewayAttachment" + v.rs.newResource(vpcGA, &gfnec2.VPCGatewayAttachment{ + InternetGatewayId: refIG, + VpcId: v.vpcID, + }) + + refPublicRT := v.rs.newResource("PublicRouteTable", &gfnec2.RouteTable{ + VpcId: v.vpcID, + }) + + v.rs.newResource("PublicSubnetRoute", &gfnec2.Route{ + RouteTableId: refPublicRT, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + GatewayId: refIG, + AWSCloudFormationDependsOn: []string{vpcGA}, + }) + + v.subnetDetails.Public = v.addSubnets(refPublicRT, api.SubnetTopologyPublic, vpc.Subnets.Public) + + if err := v.addNATGateways(); err != nil { + return err + } + + v.subnetDetails.Private = v.addSubnets(nil, api.SubnetTopologyPrivate, vpc.Subnets.Private) + return nil +} + +func (s *SubnetDetails) PublicSubnetRefs() []*gfnt.Value { + var subnetRefs []*gfnt.Value + for _, subnetAZ := range s.Public { + subnetRefs = append(subnetRefs, subnetAZ.Subnet) + } + return subnetRefs +} + +func (s *SubnetDetails) PrivateSubnetRefs() []*gfnt.Value { + var subnetRefs []*gfnt.Value + for _, subnetAZ := range s.Private { + subnetRefs = append(subnetRefs, subnetAZ.Subnet) + } + return subnetRefs +} + +// addOutputs adds VPC resource outputs +func (v *IPv4VPCResourceSet) addOutputs() { + v.rs.defineOutput(outputs.ClusterVPC, v.vpcID, true, func(val string) error { + v.clusterConfig.VPC.ID = val + return nil + }) + if v.clusterConfig.VPC.NAT != nil { + v.rs.defineOutputWithoutCollector(outputs.ClusterFeatureNATMode, v.clusterConfig.VPC.NAT.Gateway, false) + } + + addSubnetOutput := func(subnetRefs []*gfnt.Value, topology api.SubnetTopology, outputName string) { + v.rs.defineJoinedOutput(outputName, subnetRefs, true, func(value string) error { + return vpc.ImportSubnetsFromIDList(v.ec2API, v.clusterConfig, topology, strings.Split(value, ",")) + }) + } + + if subnetAZs := v.subnetDetails.PrivateSubnetRefs(); len(subnetAZs) > 0 { + addSubnetOutput(subnetAZs, api.SubnetTopologyPrivate, outputs.ClusterSubnetsPrivate) + } + + if subnetAZs := v.subnetDetails.PublicSubnetRefs(); len(subnetAZs) > 0 { + addSubnetOutput(subnetAZs, api.SubnetTopologyPublic, outputs.ClusterSubnetsPublic) + } + + if v.isFullyPrivate() { + v.rs.defineOutputWithoutCollector(outputs.ClusterFullyPrivate, true, true) + } +} + +// RenderJSON returns the rendered JSON +func (v *IPv4VPCResourceSet) RenderJSON() ([]byte, error) { + return v.rs.renderJSON() +} + +func (v *IPv4VPCResourceSet) addSubnets(refRT *gfnt.Value, topology api.SubnetTopology, subnets map[string]api.AZSubnetSpec) []SubnetResource { + var subnetIndexForIPv6 int + if api.IsEnabled(v.clusterConfig.VPC.AutoAllocateIPv6) { + // this is same kind of indexing we have in vpc.SetSubnets + switch topology { + case api.SubnetTopologyPrivate: + subnetIndexForIPv6 = len(v.clusterConfig.AvailabilityZones) + case api.SubnetTopologyPublic: + subnetIndexForIPv6 = 0 + } + } + + var subnetResources []SubnetResource + + for name, subnet := range subnets { + az := subnet.AZ + nameAlias := strings.ToUpper(strings.Join(strings.Split(name, "-"), "")) + subnet := &gfnec2.Subnet{ + AvailabilityZone: gfnt.NewString(az), + CidrBlock: gfnt.NewString(subnet.CIDR.String()), + VpcId: v.vpcID, + } + + switch topology { + case api.SubnetTopologyPrivate: + // Choose the appropriate route table for private subnets + refRT = gfnt.MakeRef("PrivateRouteTable" + nameAlias) + subnet.Tags = []gfncfn.Tag{{ + Key: gfnt.NewString("kubernetes.io/role/internal-elb"), + Value: gfnt.NewString("1"), + }} + case api.SubnetTopologyPublic: + subnet.Tags = []gfncfn.Tag{{ + Key: gfnt.NewString("kubernetes.io/role/elb"), + Value: gfnt.NewString("1"), + }} + subnet.MapPublicIpOnLaunch = gfnt.True() + } + subnetAlias := string(topology) + nameAlias + refSubnet := v.rs.newResource("Subnet"+subnetAlias, subnet) + v.rs.newResource("RouteTableAssociation"+subnetAlias, &gfnec2.SubnetRouteTableAssociation{ + SubnetId: refSubnet, + RouteTableId: refRT, + }) + + if api.IsEnabled(v.clusterConfig.VPC.AutoAllocateIPv6) { + refSubnetSlices := getSubnetIPv6CIDRBlock((len(v.clusterConfig.AvailabilityZones) * 2) + 2) + v.rs.newResource(subnetAlias+"CIDRv6", &gfnec2.SubnetCidrBlock{ + SubnetId: refSubnet, + Ipv6CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(subnetIndexForIPv6), refSubnetSlices), + }) + subnetIndexForIPv6++ + } + + subnetResources = append(subnetResources, SubnetResource{ + AvailabilityZone: az, + RouteTable: refRT, + Subnet: refSubnet, + }) + } + return subnetResources +} + +func (v *IPv4VPCResourceSet) addNATGateways() error { + switch *v.clusterConfig.VPC.NAT.Gateway { + case api.ClusterHighlyAvailableNAT: + v.haNAT() + case api.ClusterSingleNAT: + v.singleNAT() + case api.ClusterDisableNAT: + v.noNAT() + default: + // TODO validate this before starting to add resources + return fmt.Errorf("%s is not a valid NAT gateway mode", *v.clusterConfig.VPC.NAT.Gateway) + } + return nil +} + +func (v *IPv4VPCResourceSet) importResources() error { + if subnets := v.clusterConfig.VPC.Subnets.Private; subnets != nil { + var ( + subnetRoutes map[string]string + err error + ) + if v.isFullyPrivate() { + subnetRoutes, err = importRouteTables(v.ec2API, v.clusterConfig.VPC.Subnets.Private) + if err != nil { + return err + } + } + + subnetResources, err := makeSubnetResources(subnets, subnetRoutes) + if err != nil { + return err + } + v.subnetDetails.Private = subnetResources + } + + if subnets := v.clusterConfig.VPC.Subnets.Public; subnets != nil { + subnetResources, err := makeSubnetResources(subnets, nil) + if err != nil { + return err + } + v.subnetDetails.Public = subnetResources + } + + return nil +} + +func makeSubnetResources(subnets map[string]api.AZSubnetSpec, subnetRoutes map[string]string) ([]SubnetResource, error) { + subnetResources := make([]SubnetResource, len(subnets)) + i := 0 + for _, network := range subnets { + az := network.AZ + sr := SubnetResource{ + AvailabilityZone: az, + Subnet: gfnt.NewString(network.ID), + } + + if subnetRoutes != nil { + rt, ok := subnetRoutes[network.ID] + if !ok { + return nil, errors.Errorf("failed to find an explicit route table associated with subnet %q; "+ + "eksctl does not modify the main route table if a subnet is not associated with an explicit route table", network.ID) + } + sr.RouteTable = gfnt.NewString(rt) + } + subnetResources[i] = sr + i++ + } + return subnetResources, nil +} + +func importRouteTables(ec2API ec2iface.EC2API, subnets map[string]api.AZSubnetSpec) (map[string]string, error) { + var subnetIDs []string + for _, subnet := range subnets { + subnetIDs = append(subnetIDs, subnet.ID) + } + + var routeTables []*ec2.RouteTable + var nextToken *string + + for { + output, err := ec2API.DescribeRouteTables(&ec2.DescribeRouteTablesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("association.subnet-id"), + Values: aws.StringSlice(subnetIDs), + }, + }, + NextToken: nextToken, + }) + + if err != nil { + return nil, errors.Wrap(err, "error describing route tables") + } + + routeTables = append(routeTables, output.RouteTables...) + + if nextToken = output.NextToken; nextToken == nil { + break + } + } + + subnetRoutes := make(map[string]string) + for _, rt := range routeTables { + for _, rta := range rt.Associations { + if rta.Main != nil && *rta.Main { + return nil, errors.New("subnets must be associated with a non-main route table; eksctl does not modify the main route table") + } + subnetRoutes[*rta.SubnetId] = *rt.RouteTableId + } + } + return subnetRoutes, nil +} + +func (v *IPv4VPCResourceSet) isFullyPrivate() bool { + return v.clusterConfig.PrivateCluster.Enabled +} + +var ( + sgProtoTCP = gfnt.NewString("tcp") + sgSourceAnywhereIPv4 = gfnt.NewString("0.0.0.0/0") + sgSourceAnywhereIPv6 = gfnt.NewString("::/0") + + sgPortZero = gfnt.NewInteger(0) + sgMinNodePort = gfnt.NewInteger(1025) + sgMaxNodePort = gfnt.NewInteger(65535) + + sgPortHTTPS = gfnt.NewInteger(443) + sgPortSSH = gfnt.NewInteger(22) +) + +type clusterSecurityGroup struct { + ControlPlane *gfnt.Value + ClusterSharedNode *gfnt.Value +} + +// TODO move this +func (c *ClusterResourceSet) addResourcesForSecurityGroups(vpcID *gfnt.Value) *clusterSecurityGroup { + var refControlPlaneSG, refClusterSharedNodeSG *gfnt.Value + + if c.spec.VPC.SecurityGroup == "" { + refControlPlaneSG = c.newResource(cfnControlPlaneSGResource, &gfnec2.SecurityGroup{ + GroupDescription: gfnt.NewString("Communication between the control plane and worker nodegroups"), + VpcId: vpcID, + }) + + if len(c.spec.VPC.ExtraCIDRs) > 0 { + for i, cidr := range c.spec.VPC.ExtraCIDRs { + c.newResource(fmt.Sprintf("IngressControlPlaneExtraCIDR%d", i), &gfnec2.SecurityGroupIngress{ + GroupId: refControlPlaneSG, + CidrIp: gfnt.NewString(cidr), + Description: gfnt.NewString(fmt.Sprintf("Allow Extra CIDR %d (%s) to communicate to controlplane", i, cidr)), + IpProtocol: gfnt.NewString("tcp"), + FromPort: sgPortHTTPS, + ToPort: sgPortHTTPS, + }) + } + } + } else { + refControlPlaneSG = gfnt.NewString(c.spec.VPC.SecurityGroup) + } + c.securityGroups = []*gfnt.Value{refControlPlaneSG} // only this one SG is passed to EKS API, nodes are isolated + + if c.spec.VPC.SharedNodeSecurityGroup == "" { + refClusterSharedNodeSG = c.newResource(cfnSharedNodeSGResource, &gfnec2.SecurityGroup{ + GroupDescription: gfnt.NewString("Communication between all nodes in the cluster"), + VpcId: vpcID, + }) + c.newResource("IngressInterNodeGroupSG", &gfnec2.SecurityGroupIngress{ + GroupId: refClusterSharedNodeSG, + SourceSecurityGroupId: refClusterSharedNodeSG, + Description: gfnt.NewString("Allow nodes to communicate with each other (all ports)"), + IpProtocol: gfnt.NewString("-1"), + FromPort: sgPortZero, + ToPort: sgMaxNodePort, + }) + } else { + refClusterSharedNodeSG = gfnt.NewString(c.spec.VPC.SharedNodeSecurityGroup) + } + + if c.supportsManagedNodes && api.IsEnabled(c.spec.VPC.ManageSharedNodeSecurityGroupRules) { + // To enable communication between both managed and unmanaged nodegroups, this allows ingress traffic from + // the default cluster security group ID that EKS creates by default + // EKS attaches this to Managed Nodegroups by default, but we need to handle this for unmanaged nodegroups + c.newResource(cfnIngressClusterToNodeSGResource, &gfnec2.SecurityGroupIngress{ + GroupId: refClusterSharedNodeSG, + SourceSecurityGroupId: gfnt.MakeFnGetAttString("ControlPlane", outputs.ClusterDefaultSecurityGroup), + Description: gfnt.NewString("Allow managed and unmanaged nodes to communicate with each other (all ports)"), + IpProtocol: gfnt.NewString("-1"), + FromPort: sgPortZero, + ToPort: sgMaxNodePort, + }) + c.newResource("IngressNodeToDefaultClusterSG", &gfnec2.SecurityGroupIngress{ + GroupId: gfnt.MakeFnGetAttString("ControlPlane", outputs.ClusterDefaultSecurityGroup), + SourceSecurityGroupId: refClusterSharedNodeSG, + Description: gfnt.NewString("Allow unmanaged nodes to communicate with control plane (all ports)"), + IpProtocol: gfnt.NewString("-1"), + FromPort: sgPortZero, + ToPort: sgMaxNodePort, + }) + } + + if c.spec.VPC == nil { + c.spec.VPC = &api.ClusterVPC{} + } + c.rs.defineOutput(outputs.ClusterSecurityGroup, refControlPlaneSG, true, func(v string) error { + c.spec.VPC.SecurityGroup = v + return nil + }) + c.rs.defineOutput(outputs.ClusterSharedNodeSecurityGroup, refClusterSharedNodeSG, true, func(v string) error { + c.spec.VPC.SharedNodeSecurityGroup = v + return nil + }) + + return &clusterSecurityGroup{ + ControlPlane: refControlPlaneSG, + ClusterSharedNode: refClusterSharedNodeSG, + } +} + +// TODO move this +func (rs *resourceSet) addEFASecurityGroup(vpcID *gfnt.Value, clusterName, desc string) *gfnt.Value { + efaSG := rs.newResource("EFASG", &gfnec2.SecurityGroup{ + VpcId: vpcID, + GroupDescription: gfnt.NewString("EFA-enabled security group"), + Tags: []gfncfn.Tag{{ + Key: gfnt.NewString("kubernetes.io/cluster/" + clusterName), + Value: gfnt.NewString("owned"), + }}, + }) + rs.newResource("EFAIngressSelf", &gfnec2.SecurityGroupIngress{ + GroupId: efaSG, + SourceSecurityGroupId: efaSG, + Description: gfnt.NewString("Allow " + desc + " to communicate to itself (EFA-enabled)"), + IpProtocol: gfnt.NewString("-1"), + }) + rs.newResource("EFAEgressSelf", &gfnec2.SecurityGroupEgress{ + GroupId: efaSG, + DestinationSecurityGroupId: efaSG, + Description: gfnt.NewString("Allow " + desc + " to communicate to itself (EFA-enabled)"), + IpProtocol: gfnt.NewString("-1"), + }) + + return efaSG +} + +// TODO move this +func (n *NodeGroupResourceSet) addResourcesForSecurityGroups() { + for _, id := range n.spec.SecurityGroups.AttachIDs { + n.securityGroups = append(n.securityGroups, gfnt.NewString(id)) + } + + if api.IsEnabled(n.spec.SecurityGroups.WithShared) { + n.securityGroups = append(n.securityGroups, n.vpcImporter.SharedNodeSecurityGroup()) + } + + if api.IsDisabled(n.spec.SecurityGroups.WithLocal) { + return + } + + desc := "worker nodes in group " + n.spec.Name + vpcID := n.vpcImporter.VPC() + refControlPlaneSG := n.vpcImporter.ControlPlaneSecurityGroup() + + refNodeGroupLocalSG := n.newResource("SG", &gfnec2.SecurityGroup{ + VpcId: vpcID, + GroupDescription: gfnt.NewString("Communication between the control plane and " + desc), + Tags: []gfncfn.Tag{{ + Key: gfnt.NewString("kubernetes.io/cluster/" + n.clusterSpec.Metadata.Name), + Value: gfnt.NewString("owned"), + }}, + SecurityGroupIngress: makeNodeIngressRules(n.spec.NodeGroupBase, refControlPlaneSG, n.clusterSpec.VPC.CIDR.String(), desc), + }) + + n.securityGroups = append(n.securityGroups, refNodeGroupLocalSG) + + if api.IsEnabled(n.spec.EFAEnabled) { + efaSG := n.rs.addEFASecurityGroup(vpcID, n.clusterSpec.Metadata.Name, desc) + n.securityGroups = append(n.securityGroups, efaSG) + } + + n.newResource("EgressInterCluster", &gfnec2.SecurityGroupEgress{ + GroupId: refControlPlaneSG, + DestinationSecurityGroupId: refNodeGroupLocalSG, + Description: gfnt.NewString("Allow control plane to communicate with " + desc + " (kubelet and workload TCP ports)"), + IpProtocol: sgProtoTCP, + FromPort: sgMinNodePort, + ToPort: sgMaxNodePort, + }) + n.newResource("EgressInterClusterAPI", &gfnec2.SecurityGroupEgress{ + GroupId: refControlPlaneSG, + DestinationSecurityGroupId: refNodeGroupLocalSG, + Description: gfnt.NewString("Allow control plane to communicate with " + desc + " (workloads using HTTPS port, commonly used with extension API servers)"), + IpProtocol: sgProtoTCP, + FromPort: sgPortHTTPS, + ToPort: sgPortHTTPS, + }) + n.newResource("IngressInterClusterCP", &gfnec2.SecurityGroupIngress{ + GroupId: refControlPlaneSG, + SourceSecurityGroupId: refNodeGroupLocalSG, + Description: gfnt.NewString("Allow control plane to receive API requests from " + desc), + IpProtocol: sgProtoTCP, + FromPort: sgPortHTTPS, + ToPort: sgPortHTTPS, + }) +} + +func makeNodeIngressRules(ng *api.NodeGroupBase, controlPlaneSG *gfnt.Value, vpcCIDR, description string) []gfnec2.SecurityGroup_Ingress { + ingressRules := []gfnec2.SecurityGroup_Ingress{ + { + SourceSecurityGroupId: controlPlaneSG, + Description: gfnt.NewString(fmt.Sprintf("[IngressInterCluster] Allow %s to communicate with control plane (kubelet and workload TCP ports)", description)), + IpProtocol: sgProtoTCP, + FromPort: sgMinNodePort, + ToPort: sgMaxNodePort, + }, + { + SourceSecurityGroupId: controlPlaneSG, + Description: gfnt.NewString(fmt.Sprintf("[IngressInterClusterAPI] Allow %s to communicate with control plane (workloads using HTTPS port, commonly used with extension API servers)", description)), + IpProtocol: sgProtoTCP, + FromPort: sgPortHTTPS, + ToPort: sgPortHTTPS, + }, + } + + return append(ingressRules, makeSSHIngressRules(ng, vpcCIDR, description)...) +} + +func (v *IPv4VPCResourceSet) haNAT() { + for _, az := range v.clusterConfig.AvailabilityZones { + alphanumericUpperAZ := formatAZ(az) + + // Allocate an EIP + v.rs.newResource("NATIP"+alphanumericUpperAZ, &gfnec2.EIP{ + Domain: gfnt.NewString("vpc"), + }) + // Allocate a NAT gateway in the public subnet + refNG := v.rs.newResource("NATGateway"+alphanumericUpperAZ, &gfnec2.NatGateway{ + AllocationId: gfnt.MakeFnGetAttString("NATIP"+alphanumericUpperAZ, "AllocationId"), + SubnetId: gfnt.MakeRef("SubnetPublic" + alphanumericUpperAZ), + }) + + // Allocate a routing table for the private subnet + refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ + VpcId: v.vpcID, + }) + // Create a route that sends Internet traffic through the NAT gateway + v.rs.newResource("NATPrivateSubnetRoute"+alphanumericUpperAZ, &gfnec2.Route{ + RouteTableId: refRT, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + NatGatewayId: refNG, + }) + // Associate the routing table with the subnet + v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ + SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), + RouteTableId: refRT, + }) + } +} + +func (v *IPv4VPCResourceSet) singleNAT() { + sortedAZs := v.clusterConfig.AvailabilityZones + firstUpperAZ := strings.ToUpper(strings.Join(strings.Split(sortedAZs[0], "-"), "")) + + v.rs.newResource("NATIP", &gfnec2.EIP{ + Domain: gfnt.NewString("vpc"), + }) + refNG := v.rs.newResource("NATGateway", &gfnec2.NatGateway{ + AllocationId: gfnt.MakeFnGetAttString("NATIP", "AllocationId"), + SubnetId: gfnt.MakeRef("SubnetPublic" + firstUpperAZ), + }) + + for _, az := range v.clusterConfig.AvailabilityZones { + alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) + + refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ + VpcId: v.vpcID, + }) + + v.rs.newResource("NATPrivateSubnetRoute"+alphanumericUpperAZ, &gfnec2.Route{ + RouteTableId: refRT, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + NatGatewayId: refNG, + }) + v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ + SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), + RouteTableId: refRT, + }) + } +} + +func (v *IPv4VPCResourceSet) noNAT() { + for _, az := range v.clusterConfig.AvailabilityZones { + alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) + + refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ + VpcId: v.vpcID, + }) + v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ + SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), + RouteTableId: refRT, + }) + } +} diff --git a/pkg/cfn/builder/vpc_test.go b/pkg/cfn/builder/vpc_ipv4_test.go similarity index 93% rename from pkg/cfn/builder/vpc_test.go rename to pkg/cfn/builder/vpc_ipv4_test.go index 62835d8fae..8e2b698523 100644 --- a/pkg/cfn/builder/vpc_test.go +++ b/pkg/cfn/builder/vpc_ipv4_test.go @@ -17,7 +17,7 @@ import ( var _ = Describe("VPC Template Builder", func() { var ( - vpcRs *builder.VPCResourceSet + vpcRs *builder.IPv4VPCResourceSet cfg *api.ClusterConfig mockEC2 = &mocks.EC2API{} ) @@ -29,18 +29,19 @@ var _ = Describe("VPC Template Builder", func() { }) JustBeforeEach(func() { - vpcRs = builder.NewVPCResourceSet(builder.NewRS(), cfg, mockEC2) + vpcRs = builder.NewIPv4VPCResourceSet(builder.NewRS(), cfg, mockEC2) }) Describe("AddResources", func() { var ( - addErr error - result *builder.VPCResource - vpcTemplate *fakes.FakeTemplate + addErr error + vpcID *gfnt.Value + subnetDetails *builder.SubnetDetails + vpcTemplate *fakes.FakeTemplate ) JustBeforeEach(func() { - result, addErr = vpcRs.AddResources() + vpcID, subnetDetails, addErr = vpcRs.CreateTemplate() vpcTemplate = &fakes.FakeTemplate{} templateBody, err := vpcRs.RenderJSON() Expect(err).ShouldNot(HaveOccurred()) @@ -52,7 +53,7 @@ var _ = Describe("VPC Template Builder", func() { }) It("returns the VPC resource", func() { - Expect(result.VPC).To(Equal(gfnt.MakeRef(vpcResourceKey))) + Expect(vpcID).To(Equal(gfnt.MakeRef(vpcResourceKey))) }) It("adds the correct gateway resources to the resource set", func() { @@ -73,12 +74,12 @@ var _ = Describe("VPC Template Builder", func() { }) It("returns public subnet settings", func() { - Expect(result.SubnetDetails.Public).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Public).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.MakeRef(publicSubnetRef2), RouteTable: gfnt.MakeRef(pubRouteTable), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Public).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Public).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.MakeRef(publicSubnetRef1), RouteTable: gfnt.MakeRef(pubRouteTable), AvailabilityZone: azA, @@ -119,13 +120,13 @@ var _ = Describe("VPC Template Builder", func() { }) It("returns private subnet settings", func() { - Expect(result.SubnetDetails.Private).To(HaveLen(2)) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(HaveLen(2)) + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.MakeRef(privateSubnetRef2), RouteTable: gfnt.MakeRef(privRouteTableB), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.MakeRef(privateSubnetRef1), RouteTable: gfnt.MakeRef(privRouteTableA), AvailabilityZone: azA, @@ -270,7 +271,7 @@ var _ = Describe("VPC Template Builder", func() { }) It("the correct VPC resource values are loaded into the VPCResource", func() { - Expect(result.VPC).To(Equal(gfnt.NewString("custom-vpc"))) + Expect(vpcID).To(Equal(gfnt.NewString("custom-vpc"))) }) It("no resources are added to the set", func() { @@ -279,12 +280,12 @@ var _ = Describe("VPC Template Builder", func() { Context("private subnets are configured", func() { It("the private subnet resource values are loaded into the VPCResource", func() { - Expect(result.SubnetDetails.Private).To(HaveLen(2)) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(HaveLen(2)) + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(privateSubnet2), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(privateSubnet1), AvailabilityZone: azA, })) @@ -310,13 +311,13 @@ var _ = Describe("VPC Template Builder", func() { }) It("the private subnet resource values are loaded into the VPCResource with route table association", func() { - Expect(result.SubnetDetails.Private).To(HaveLen(2)) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(HaveLen(2)) + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(privateSubnet2), RouteTable: gfnt.NewString("this-is-a-route-table"), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(privateSubnet1), RouteTable: gfnt.NewString("this-is-a-route-table"), AvailabilityZone: azA, @@ -354,12 +355,12 @@ var _ = Describe("VPC Template Builder", func() { Context("public subnets are configured", func() { It("the public subnet resource values are loaded into the VPCResource", func() { - Expect(result.SubnetDetails.Public).To(HaveLen(2)) - Expect(result.SubnetDetails.Public).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Public).To(HaveLen(2)) + Expect(subnetDetails.Public).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(publicSubnet2), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Public).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Public).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(publicSubnet1), AvailabilityZone: azA, })) @@ -372,7 +373,7 @@ var _ = Describe("VPC Template Builder", func() { BeforeEach(func() { autoAllocated := true cfg.VPC.AutoAllocateIPv6 = &autoAllocated - expectedFnCIDR = `{ "Fn::Cidr": [{ "Fn::Select": [ 0, { "Fn::GetAtt": ["VPC", "Ipv6CidrBlocks"] }]}, 8, 64 ]}` + expectedFnCIDR = `{ "Fn::Cidr": [{ "Fn::Select": [ 0, { "Fn::GetAtt": ["VPC", "Ipv6CidrBlocks"] }]}, 6, 64 ]}` }) It("adds the AutoAllocatedCIDRv6 vpc resource to the resource set", func() { @@ -429,7 +430,7 @@ var _ = Describe("VPC Template Builder", func() { }) It("does not set public subnet resources", func() { - Expect(result.SubnetDetails.Public).To(HaveLen(0)) + Expect(subnetDetails.Public).To(HaveLen(0)) Expect(vpcTemplate.Resources).ToNot(HaveKey(pubSubnetRoute)) Expect(vpcTemplate.Resources).ToNot(HaveKey(pubSubnetRoute)) Expect(vpcTemplate.Resources).ToNot(HaveKey(publicSubnetRef1)) @@ -437,7 +438,7 @@ var _ = Describe("VPC Template Builder", func() { Expect(vpcTemplate.Resources).ToNot(HaveKey(rtaPublicA)) Expect(vpcTemplate.Resources).ToNot(HaveKey(rtaPublicB)) - Expect(result.SubnetDetails.Private).To(HaveLen(2)) + Expect(subnetDetails.Private).To(HaveLen(2)) Expect(vpcTemplate.Resources).To(HaveKey(privRouteTableA)) Expect(vpcTemplate.Resources).To(HaveKey(privRouteTableB)) Expect(vpcTemplate.Resources).To(HaveKey(privateSubnetRef1)) @@ -461,9 +462,8 @@ var _ = Describe("VPC Template Builder", func() { ) JustBeforeEach(func() { - _, err := vpcRs.AddResources() + _, _, err := vpcRs.CreateTemplate() Expect(err).NotTo(HaveOccurred()) - vpcRs.AddOutputs() vpcTemplate = &fakes.FakeTemplate{} templateBody, err := vpcRs.RenderJSON() Expect(err).ShouldNot(HaveOccurred()) @@ -508,9 +508,9 @@ var _ = Describe("VPC Template Builder", func() { Describe("PublicSubnetRefs", func() { It("returns the references of public subnets", func() { - result, err := vpcRs.AddResources() + _, subnetDetails, err := vpcRs.CreateTemplate() Expect(err).NotTo(HaveOccurred()) - refs := result.SubnetDetails.PublicSubnetRefs() + refs := subnetDetails.PublicSubnetRefs() Expect(refs).To(HaveLen(2)) Expect(refs).To(ContainElement(makePrimitive(publicSubnetRef1))) Expect(refs).To(ContainElement(makePrimitive(publicSubnetRef2))) @@ -519,9 +519,9 @@ var _ = Describe("VPC Template Builder", func() { Describe("PrivateSubnetRefs", func() { It("returns the references of private subnets", func() { - result, err := vpcRs.AddResources() + _, subnetDetails, err := vpcRs.CreateTemplate() Expect(err).NotTo(HaveOccurred()) - refs := result.SubnetDetails.PrivateSubnetRefs() + refs := subnetDetails.PrivateSubnetRefs() Expect(refs).To(HaveLen(2)) Expect(refs).To(ContainElement(makePrimitive(privateSubnetRef1))) Expect(refs).To(ContainElement(makePrimitive(privateSubnetRef2))) diff --git a/pkg/cfn/builder/vpc_ipv6.go b/pkg/cfn/builder/vpc_ipv6.go new file mode 100644 index 0000000000..9d88870da0 --- /dev/null +++ b/pkg/cfn/builder/vpc_ipv6.go @@ -0,0 +1,192 @@ +package builder + +import ( + "strings" + + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/outputs" + "github.com/weaveworks/eksctl/pkg/vpc" + "github.com/weaveworks/goformation/v4/cloudformation/cloudformation" + gfnec2 "github.com/weaveworks/goformation/v4/cloudformation/ec2" + gfnt "github.com/weaveworks/goformation/v4/cloudformation/types" +) + +// A IPv6VPCResourceSet builds the resources required for the specified VPC +type IPv6VPCResourceSet struct { + rs *resourceSet + clusterConfig *api.ClusterConfig + ec2API ec2iface.EC2API + + // vpcResource *IPv6VPCResource +} + +// // IPv6VPCResource reresents a VPC resource +// type IPv6VPCResource struct { +// VPC *gfnt.Value +// SubnetDetails *subnetDetails +// } + +// NewIPv6VPCResourceSet creates and returns a new VPCResourceSet +func NewIPv6VPCResourceSet(rs *resourceSet, clusterConfig *api.ClusterConfig, ec2API ec2iface.EC2API) *IPv6VPCResourceSet { + return &IPv6VPCResourceSet{ + rs: rs, + clusterConfig: clusterConfig, + ec2API: ec2API, + // vpcResource: &IPv6VPCResource{ + // VPC: vpcRef, + // SubnetDetails: &subnetDetails{}, + // }, + } +} + +func (v *IPv6VPCResourceSet) CreateTemplate() (*gfnt.Value, *SubnetDetails, error) { + var publicSubnetResourceRefs, privateSubnetResourceRefs []*gfnt.Value + vpcResourceRef := v.rs.newResource(VPCResourceKey, &gfnec2.VPC{ + CidrBlock: gfnt.NewString(v.clusterConfig.VPC.CIDR.String()), + EnableDnsSupport: gfnt.True(), + EnableDnsHostnames: gfnt.True(), + }) + + v.rs.newResource(IPv6CIDRBlockKey, &gfnec2.VPCCidrBlock{ + AmazonProvidedIpv6CidrBlock: gfnt.True(), + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + refIGW := v.rs.newResource(IGWKey, &gfnec2.InternetGateway{}) + + v.rs.newResource(GAKey, &gfnec2.VPCGatewayAttachment{ + InternetGatewayId: gfnt.MakeRef(IGWKey), + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + v.rs.newResource(EgressOnlyInternetGatewayKey, &gfnec2.EgressOnlyInternetGateway{ + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + firstPublicSubnet := PublicSubnetKey + formatAZ(v.clusterConfig.AvailabilityZones[0]) + v.rs.newResource(NATGatewayKey, &gfnec2.NatGateway{ + AWSCloudFormationDependsOn: []string{ + ElasticIPKey, + firstPublicSubnet, + GAKey, + }, + AllocationId: gfnt.MakeFnGetAtt(ElasticIPKey, gfnt.NewString("AllocationId")), + SubnetId: gfnt.MakeRef(firstPublicSubnet), + }) + + v.rs.newResource(ElasticIPKey, &gfnec2.EIP{ + Domain: gfnt.NewString("vpc"), + AWSCloudFormationDependsOn: []string{GAKey}, + }) + + v.rs.newResource(PubRouteTableKey, &gfnec2.RouteTable{ + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + v.rs.newResource(PubSubRouteKey, &gfnec2.Route{ + AWSCloudFormationDependsOn: []string{GAKey}, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + GatewayId: refIGW, + RouteTableId: gfnt.MakeRef(PubRouteTableKey), + }) + + v.rs.newResource(PubSubIPv6RouteKey, &gfnec2.Route{ + AWSCloudFormationDependsOn: []string{GAKey}, + DestinationIpv6CidrBlock: gfnt.NewString(InternetIPv6CIDR), + GatewayId: refIGW, + RouteTableId: gfnt.MakeRef(PubRouteTableKey), + }) + + cidrPartitions := (len(v.clusterConfig.AvailabilityZones) * 2) + 2 + for i, az := range v.clusterConfig.AvailabilityZones { + azFormatted := formatAZ(az) + v.rs.newResource(PrivateRouteTableKey+azFormatted, &gfnec2.RouteTable{ + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + v.rs.newResource(PrivateSubnetIpv6RouteKey+azFormatted, &gfnec2.Route{ + DestinationIpv6CidrBlock: gfnt.NewString(InternetIPv6CIDR), + EgressOnlyInternetGatewayId: gfnt.MakeRef(EgressOnlyInternetGatewayKey), + RouteTableId: gfnt.MakeRef(PrivateRouteTableKey + azFormatted), + }) + + v.rs.newResource(PrivateSubnetRouteKey+azFormatted, &gfnec2.Route{ + AWSCloudFormationDependsOn: []string{NATGatewayKey, GAKey}, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + NatGatewayId: gfnt.MakeRef(NATGatewayKey), + RouteTableId: gfnt.MakeRef(PrivateRouteTableKey + azFormatted), + }) + + publicSubnetResourceRefs = append(publicSubnetResourceRefs, v.createSubnet(az, azFormatted, i, cidrPartitions, false)) + privateSubnetResourceRefs = append(privateSubnetResourceRefs, v.createSubnet(az, azFormatted, i+len(v.clusterConfig.AvailabilityZones), cidrPartitions, true)) + + v.rs.newResource(PubRouteTableAssociation+azFormatted, &gfnec2.SubnetRouteTableAssociation{ + RouteTableId: gfnt.MakeRef(PubRouteTableKey), + SubnetId: gfnt.MakeRef(PublicSubnetKey + azFormatted), + }) + + v.rs.newResource(PrivateRouteTableAssociation+azFormatted, &gfnec2.SubnetRouteTableAssociation{ + RouteTableId: gfnt.MakeRef(PrivateRouteTableKey + azFormatted), + SubnetId: gfnt.MakeRef(PrivateSubnetKey + azFormatted), + }) + } + + v.rs.defineOutput(outputs.ClusterVPC, vpcResourceRef, true, func(val string) error { + v.clusterConfig.VPC.ID = val + return nil + }) + + addSubnetOutput := func(subnetRefs []*gfnt.Value, topology api.SubnetTopology, outputName string) { + v.rs.defineJoinedOutput(outputName, subnetRefs, true, func(value string) error { + return vpc.ImportSubnetsFromIDList(v.ec2API, v.clusterConfig, topology, strings.Split(value, ",")) + }) + } + + addSubnetOutput(publicSubnetResourceRefs, api.SubnetTopologyPublic, outputs.ClusterSubnetsPublic) + addSubnetOutput(privateSubnetResourceRefs, api.SubnetTopologyPrivate, outputs.ClusterSubnetsPrivate) + + var publicSubnets, privateSubnets []SubnetResource + for _, s := range publicSubnetResourceRefs { + publicSubnets = append(publicSubnets, SubnetResource{Subnet: s}) + } + for _, s := range privateSubnetResourceRefs { + privateSubnets = append(privateSubnets, SubnetResource{Subnet: s}) + } + return vpcResourceRef, &SubnetDetails{ + Private: privateSubnets, + Public: publicSubnets, + }, nil +} + +func (v *IPv6VPCResourceSet) RenderJSON() ([]byte, error) { + return v.rs.renderJSON() +} + +func (v *IPv6VPCResourceSet) createSubnet(az, azFormatted string, i, cidrPartitions int, private bool) *gfnt.Value { + var assignIpv6AddressOnCreation *gfnt.Value + subnetKey := PublicSubnetKey + azFormatted + mapPublicIpOnLaunch := gfnt.True() + elbTagKey := "kubernetes.io/role/elb" + + if private { + subnetKey = PrivateSubnetKey + azFormatted + mapPublicIpOnLaunch = nil + assignIpv6AddressOnCreation = gfnt.True() + elbTagKey = "kubernetes.io/role/internal-elb" + } + + return v.rs.newResource(subnetKey, &gfnec2.Subnet{ + AWSCloudFormationDependsOn: []string{IPv6CIDRBlockKey}, + AvailabilityZone: gfnt.NewString(az), + CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(i), getSubnetIPv4CIDRBlock(cidrPartitions)), + Ipv6CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(i), getSubnetIPv6CIDRBlock(cidrPartitions)), + MapPublicIpOnLaunch: mapPublicIpOnLaunch, + AssignIpv6AddressOnCreation: assignIpv6AddressOnCreation, + VpcId: gfnt.MakeRef(VPCResourceKey), + Tags: []cloudformation.Tag{{ + Key: gfnt.NewString(elbTagKey), + Value: gfnt.NewString("1"), + }}, + }) +} diff --git a/pkg/cfn/builder/vpc_ipv6_test.go b/pkg/cfn/builder/vpc_ipv6_test.go new file mode 100644 index 0000000000..88099558a1 --- /dev/null +++ b/pkg/cfn/builder/vpc_ipv6_test.go @@ -0,0 +1,386 @@ +package builder_test + +import ( + "encoding/json" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/builder" + "github.com/weaveworks/eksctl/pkg/cfn/builder/fakes" + "github.com/weaveworks/eksctl/pkg/cfn/outputs" +) + +var _ = Describe("IPv6 VPC builder", func() { + var ( + vpcTemplate *fakes.FakeTemplate + vpcRs *builder.IPv6VPCResourceSet + cfg *api.ClusterConfig + ) + + BeforeEach(func() { + cfg = api.NewClusterConfig() + cfg.VPC.IPFamily = aws.String("ipv6") + }) + + It("creates the ipv6 VPC and its resources", func() { + cfg.AvailabilityZones = []string{azA, azB} + vpcRs = builder.NewIPv6VPCResourceSet(builder.NewRS(), cfg, nil) + + _, subnetDetails, err := vpcRs.CreateTemplate() + Expect(err).NotTo(HaveOccurred()) + + By("returning the references of public subnets") + pubRefs := subnetDetails.PublicSubnetRefs() + Expect(pubRefs).To(HaveLen(2)) + Expect(pubRefs).To(ContainElement(makePrimitive(builder.PublicSubnetKey + azAFormatted))) + Expect(pubRefs).To(ContainElement(makePrimitive(builder.PublicSubnetKey + azBFormatted))) + + By("returning the references of private subnets") + privRef := subnetDetails.PrivateSubnetRefs() + Expect(privRef).To(HaveLen(2)) + Expect(privRef).To(ContainElement(makePrimitive(builder.PrivateSubnetKey + azBFormatted))) + Expect(privRef).To(ContainElement(makePrimitive(builder.PrivateSubnetKey + azBFormatted))) + + vpcTemplate = &fakes.FakeTemplate{} + templateBody, err := vpcRs.RenderJSON() + Expect(err).ShouldNot(HaveOccurred()) + Expect(json.Unmarshal(templateBody, vpcTemplate)).To(Succeed()) + + By("creating the VPC resource") + Expect(vpcTemplate.Resources).To(HaveKey(builder.VPCResourceKey)) + Expect(vpcTemplate.Resources[builder.VPCResourceKey].Type).To(Equal("AWS::EC2::VPC")) + defaultCidr := api.DefaultCIDR() + cidr := &defaultCidr + Expect(vpcTemplate.Resources[builder.VPCResourceKey].Properties).To(Equal(fakes.Properties{ + CidrBlock: cidr.String(), + EnableDnsHostnames: true, + EnableDnsSupport: true, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": "${AWS::StackName}/VPC"}, + }, + }, + })) + + By("creating the IPv6 CIDR") + Expect(vpcTemplate.Resources).To(HaveKey(builder.IPv6CIDRBlockKey)) + Expect(vpcTemplate.Resources[builder.IPv6CIDRBlockKey].Type).To(Equal("AWS::EC2::VPCCidrBlock")) + Expect(vpcTemplate.Resources[builder.IPv6CIDRBlockKey].Properties).To(Equal(fakes.Properties{ + AmazonProvidedIpv6CidrBlock: true, + VpcID: map[string]interface{}{"Ref": "VPC"}, + })) + + By("creating the internet gateway") + Expect(vpcTemplate.Resources).To(HaveKey(builder.IGWKey)) + Expect(vpcTemplate.Resources[builder.IGWKey].Type).To(Equal("AWS::EC2::InternetGateway")) + Expect(vpcTemplate.Resources[builder.IGWKey].Properties).To(Equal(fakes.Properties{ + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": "${AWS::StackName}/InternetGateway"}, + }, + }, + })) + + By("creating a VPC gateway attachment to associate the IGW with the VPC") + Expect(vpcTemplate.Resources).To(HaveKey(builder.GAKey)) + Expect(vpcTemplate.Resources[builder.GAKey].Type).To(Equal("AWS::EC2::VPCGatewayAttachment")) + Expect(vpcTemplate.Resources[builder.GAKey].Properties).To(Equal(fakes.Properties{ + InternetGatewayID: map[string]interface{}{"Ref": "InternetGateway"}, + VpcID: map[string]interface{}{"Ref": "VPC"}, + })) + + By("creating a VPC gateway attachment to associate the IGW with the VPC") + Expect(vpcTemplate.Resources).To(HaveKey(builder.EgressOnlyInternetGatewayKey)) + Expect(vpcTemplate.Resources[builder.EgressOnlyInternetGatewayKey].Type).To(Equal("AWS::EC2::EgressOnlyInternetGateway")) + Expect(vpcTemplate.Resources[builder.EgressOnlyInternetGatewayKey].Properties).To(Equal(fakes.Properties{ + VpcID: map[string]interface{}{"Ref": "VPC"}, + })) + + By("creating the NAT gateway") + Expect(vpcTemplate.Resources).To(HaveKey(builder.NATGatewayKey)) + Expect(vpcTemplate.Resources[builder.NATGatewayKey].Type).To(Equal("AWS::EC2::NatGateway")) + Expect(vpcTemplate.Resources[builder.NATGatewayKey].DependsOn).To(ConsistOf(builder.ElasticIPKey, builder.PublicSubnetKey+azAFormatted, builder.GAKey)) + Expect(vpcTemplate.Resources[builder.NATGatewayKey].Properties).To(Equal(fakes.Properties{ + AllocationID: map[string]interface{}{ + "Fn::GetAtt": []interface{}{ + builder.ElasticIPKey, + "AllocationId", + }, + }, + SubnetID: map[string]interface{}{"Ref": builder.PublicSubnetKey + azAFormatted}, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", builder.NATGatewayKey)}, + }, + }, + })) + + By("creating an Elastic IP for the Nat Gateway") + Expect(vpcTemplate.Resources).To(HaveKey(builder.ElasticIPKey)) + Expect(vpcTemplate.Resources[builder.ElasticIPKey].Type).To(Equal("AWS::EC2::EIP")) + Expect(vpcTemplate.Resources[builder.ElasticIPKey].DependsOn).To(ConsistOf(gaKey)) + Expect(vpcTemplate.Resources[builder.ElasticIPKey].Properties).To(Equal(fakes.Properties{ + Domain: "vpc", + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", builder.ElasticIPKey)}, + }, + }, + })) + + By("creating a public Route Table") + Expect(vpcTemplate.Resources).To(HaveKey(builder.PubRouteTableKey)) + Expect(vpcTemplate.Resources[builder.PubRouteTableKey].Type).To(Equal("AWS::EC2::RouteTable")) + Expect(vpcTemplate.Resources[builder.PubRouteTableKey].Properties).To(Equal(fakes.Properties{ + VpcID: map[string]interface{}{"Ref": "VPC"}, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", builder.PubRouteTableKey)}, + }, + }, + })) + + By("creating public subnet route for IPv4 traffic to IPv4 CIDR") + Expect(vpcTemplate.Resources).To(HaveKey(builder.PubSubRouteKey)) + Expect(vpcTemplate.Resources[builder.PubSubRouteKey].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[builder.PubSubRouteKey].DependsOn).To(ConsistOf(builder.GAKey)) + Expect(vpcTemplate.Resources[builder.PubSubRouteKey].Properties).To(Equal(fakes.Properties{ + DestinationCidrBlock: builder.InternetCIDR, + GatewayID: map[string]interface{}{"Ref": builder.IGWKey}, + RouteTableID: map[string]interface{}{"Ref": builder.PubRouteTableKey}, + })) + + By("creating public subnet route for IPv6 traffic to IPv6 CIDR") + Expect(vpcTemplate.Resources).To(HaveKey(builder.PubSubIPv6RouteKey)) + Expect(vpcTemplate.Resources[builder.PubSubIPv6RouteKey].Type).To(Equal("AWS::EC2::Route")) + //TODO: we added this, wasn't in the example template. We think its correct? + Expect(vpcTemplate.Resources[builder.PubSubIPv6RouteKey].DependsOn).To(ConsistOf(builder.GAKey)) + Expect(vpcTemplate.Resources[builder.PubSubIPv6RouteKey].Properties).To(Equal(fakes.Properties{ + DestinationIpv6CidrBlock: builder.InternetIPv6CIDR, + GatewayID: map[string]interface{}{"Ref": builder.IGWKey}, + RouteTableID: map[string]interface{}{"Ref": builder.PubRouteTableKey}, + })) + + By("creating a private route table for each AZ") + privateRouteTableA := builder.PrivateRouteTableKey + azAFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteTableA)) + Expect(vpcTemplate.Resources[privateRouteTableA].Type).To(Equal("AWS::EC2::RouteTable")) + Expect(vpcTemplate.Resources[privateRouteTableA].Properties).To(Equal(fakes.Properties{ + VpcID: map[string]interface{}{"Ref": "VPC"}, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", privateRouteTableA)}, + }, + }, + })) + privateRouteTableB := builder.PrivateRouteTableKey + azBFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteTableB)) + Expect(vpcTemplate.Resources[privateRouteTableB].Type).To(Equal("AWS::EC2::RouteTable")) + Expect(vpcTemplate.Resources[privateRouteTableB].Properties).To(Equal(fakes.Properties{ + VpcID: map[string]interface{}{"Ref": "VPC"}, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", privateRouteTableB)}, + }, + }, + })) + + By("creating a route to the NAT gateway for each private subnet in the AZs") + privateRouteA := builder.PrivateSubnetRouteKey + azAFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteA)) + Expect(vpcTemplate.Resources[privateRouteA].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[privateRouteA].DependsOn).To(ConsistOf(builder.NATGatewayKey, builder.GAKey)) + Expect(vpcTemplate.Resources[privateRouteA].Properties).To(Equal(fakes.Properties{ + DestinationCidrBlock: builder.InternetCIDR, + NatGatewayID: map[string]interface{}{"Ref": builder.NATGatewayKey}, + RouteTableID: map[string]interface{}{"Ref": privateRouteTableA}, + })) + + privateRouteB := builder.PrivateSubnetRouteKey + azBFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteB)) + Expect(vpcTemplate.Resources[privateRouteB].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[privateRouteB].DependsOn).To(ConsistOf(builder.NATGatewayKey, builder.GAKey)) + Expect(vpcTemplate.Resources[privateRouteB].Properties).To(Equal(fakes.Properties{ + DestinationCidrBlock: builder.InternetCIDR, + NatGatewayID: map[string]interface{}{"Ref": builder.NATGatewayKey}, + RouteTableID: map[string]interface{}{"Ref": privateRouteTableB}, + })) + + By("creating a ipv6 route to the ingress only internet gateway for each private subnet in the AZs") + privateRouteA = builder.PrivateSubnetIpv6RouteKey + azAFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteA)) + Expect(vpcTemplate.Resources[privateRouteA].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[privateRouteA].Properties).To(Equal(fakes.Properties{ + DestinationIpv6CidrBlock: builder.InternetIPv6CIDR, + EgressOnlyInternetGatewayID: map[string]interface{}{"Ref": builder.EgressOnlyInternetGatewayKey}, + RouteTableID: map[string]interface{}{"Ref": privateRouteTableA}, + })) + privateRouteB = builder.PrivateSubnetIpv6RouteKey + azBFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteB)) + Expect(vpcTemplate.Resources[privateRouteB].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[privateRouteB].Properties).To(Equal(fakes.Properties{ + DestinationIpv6CidrBlock: builder.InternetIPv6CIDR, + EgressOnlyInternetGatewayID: map[string]interface{}{"Ref": builder.EgressOnlyInternetGatewayKey}, + RouteTableID: map[string]interface{}{"Ref": privateRouteTableB}, + })) + + By("creating a public and private subnet for each AZ") + assertSubnetSet := func(az, subnetKey, kubernetesTag string, cidrBlockIndex float64, mapPublicIpOnLaunch bool) { + Expect(vpcTemplate.Resources).To(HaveKey(subnetKey)) + Expect(vpcTemplate.Resources[subnetKey].Type).To(Equal("AWS::EC2::Subnet")) + Expect(vpcTemplate.Resources[subnetKey].DependsOn).To(ConsistOf(builder.IPv6CIDRBlockKey)) + Expect(vpcTemplate.Resources[subnetKey].Properties.AvailabilityZone).To(Equal(az)) + Expect(vpcTemplate.Resources[subnetKey].Properties.MapPublicIPOnLaunch).To(Equal(mapPublicIpOnLaunch)) + + Expect(vpcTemplate.Resources[subnetKey].Properties.VpcID).To(Equal(map[string]interface{}{"Ref": "VPC"})) + Expect(vpcTemplate.Resources[subnetKey].Properties.Tags).To(ConsistOf( + fakes.Tag{ + Key: kubernetesTag, + Value: "1", + }, + fakes.Tag{ + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", subnetKey)}, + }, + )) + + expectedFnIPv4CIDR := `{ "Fn::Cidr": [{ "Fn::GetAtt": ["VPC", "CidrBlock"]}, 6, 13 ]}` + Expect(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"]).To(HaveLen(2)) + Expect(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"].([]interface{})[0].(float64)).To(Equal(cidrBlockIndex)) + actualFnCIDR, err := json.Marshal(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"].([]interface{})[1]) + Expect(err).NotTo(HaveOccurred()) + Expect(actualFnCIDR).To(MatchJSON([]byte(expectedFnIPv4CIDR))) + + expectedFnIPv6CIDR := `{ "Fn::Cidr": [{ "Fn::Select": [ 0, { "Fn::GetAtt": ["VPC", "Ipv6CidrBlocks"] }]}, 6, 64 ]}` + Expect(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"]).To(HaveLen(2)) + Expect(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"][0].(float64)).To(Equal(cidrBlockIndex)) + actualFnIPv6CIDR, err := json.Marshal(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"][1]) + Expect(err).NotTo(HaveOccurred()) + Expect(actualFnIPv6CIDR).To(MatchJSON([]byte(expectedFnIPv6CIDR))) + } + assertSubnetSet(azA, builder.PublicSubnetKey+azAFormatted, "kubernetes.io/role/elb", float64(0), true) + Expect(vpcTemplate.Resources[builder.PublicSubnetKey+azAFormatted].Properties.AssignIpv6AddressOnCreation).To(BeNil()) + assertSubnetSet(azB, builder.PublicSubnetKey+azBFormatted, "kubernetes.io/role/elb", float64(1), true) + Expect(vpcTemplate.Resources[builder.PublicSubnetKey+azBFormatted].Properties.AssignIpv6AddressOnCreation).To(BeNil()) + + assertSubnetSet(azA, builder.PrivateSubnetKey+azAFormatted, "kubernetes.io/role/internal-elb", float64(2), false) + Expect(*vpcTemplate.Resources[builder.PrivateSubnetKey+azAFormatted].Properties.AssignIpv6AddressOnCreation).To(Equal(true)) + assertSubnetSet(azB, builder.PrivateSubnetKey+azBFormatted, "kubernetes.io/role/internal-elb", float64(3), false) + Expect(*vpcTemplate.Resources[builder.PrivateSubnetKey+azAFormatted].Properties.AssignIpv6AddressOnCreation).To(Equal(true)) + + By("creating route table associations", func() { + assertSubnetRouteTableAssociation := func(routeTableAssociationKey, subnetKey, routeTableKey string) { + Expect(vpcTemplate.Resources).To(HaveKey(routeTableAssociationKey)) + Expect(vpcTemplate.Resources[routeTableAssociationKey].Type).To(Equal("AWS::EC2::SubnetRouteTableAssociation")) + Expect(vpcTemplate.Resources[routeTableAssociationKey].Properties).To(Equal(fakes.Properties{ + RouteTableID: map[string]interface{}{"Ref": routeTableKey}, + SubnetID: map[string]interface{}{"Ref": subnetKey}, + })) + } + + By("associating all public subnets with the public route table", func() { + assertSubnetRouteTableAssociation(builder.PubRouteTableAssociation+azAFormatted, builder.PublicSubnetKey+azAFormatted, builder.PubRouteTableKey) + assertSubnetRouteTableAssociation(builder.PubRouteTableAssociation+azBFormatted, builder.PublicSubnetKey+azBFormatted, builder.PubRouteTableKey) + }) + + By("associating each private subnet with its private route table", func() { + assertSubnetRouteTableAssociation(builder.PrivateRouteTableAssociation+azAFormatted, builder.PrivateSubnetKey+azAFormatted, builder.PrivateRouteTableKey+azAFormatted) + assertSubnetRouteTableAssociation(builder.PrivateRouteTableAssociation+azBFormatted, builder.PrivateSubnetKey+azBFormatted, builder.PrivateRouteTableKey+azBFormatted) + }) + }) + + By("outputting the VPC on the stack") + Expect(vpcTemplate.Outputs).To(HaveKey(builder.VPCResourceKey)) + Expect(vpcTemplate.Outputs.(map[string]interface{})[builder.VPCResourceKey].(map[string]interface{})["Value"]).To(Equal(map[string]interface{}{"Ref": builder.VPCResourceKey})) + Expect(vpcTemplate.Outputs.(map[string]interface{})[builder.VPCResourceKey].(map[string]interface{})["Export"]).To(Equal(map[string]interface{}{ + "Name": map[string]interface{}{ + "Fn::Sub": fmt.Sprintf("${AWS::StackName}::%s", builder.VPCResourceKey), + }, + })) + + By("outputting the public subnets on the stack") + Expect(vpcTemplate.Outputs).To(HaveKey(outputs.ClusterSubnetsPublic)) + Expect(vpcTemplate.Outputs.(map[string]interface{})[outputs.ClusterSubnetsPublic].(map[string]interface{})["Value"]).To(Equal(map[string]interface{}{ + "Fn::Join": []interface{}{ + ",", + []interface{}{ + map[string]interface{}{"Ref": builder.PublicSubnetKey + azAFormatted}, + map[string]interface{}{"Ref": builder.PublicSubnetKey + azBFormatted}, + }, + }, + })) + Expect(vpcTemplate.Outputs.(map[string]interface{})[outputs.ClusterSubnetsPublic].(map[string]interface{})["Export"]).To(Equal(map[string]interface{}{ + "Name": map[string]interface{}{ + "Fn::Sub": fmt.Sprintf("${AWS::StackName}::%s", outputs.ClusterSubnetsPublic), + }, + })) + + By("outputting the private subnets on the stack") + Expect(vpcTemplate.Outputs).To(HaveKey(outputs.ClusterSubnetsPrivate)) + Expect(vpcTemplate.Outputs.(map[string]interface{})[outputs.ClusterSubnetsPrivate].(map[string]interface{})["Value"]).To(Equal(map[string]interface{}{ + "Fn::Join": []interface{}{ + ",", + []interface{}{ + map[string]interface{}{"Ref": builder.PrivateSubnetKey + azAFormatted}, + map[string]interface{}{"Ref": builder.PrivateSubnetKey + azBFormatted}, + }, + }, + })) + Expect(vpcTemplate.Outputs.(map[string]interface{})[outputs.ClusterSubnetsPrivate].(map[string]interface{})["Export"]).To(Equal(map[string]interface{}{ + "Name": map[string]interface{}{ + "Fn::Sub": fmt.Sprintf("${AWS::StackName}::%s", outputs.ClusterSubnetsPrivate), + }, + })) + + }) + + Context("when there are 3 AZs", func() { + It("scales the CIDR blocks accordingly", func() { + cfg.AvailabilityZones = []string{azA, azB, azC} + vpcRs = builder.NewIPv6VPCResourceSet(builder.NewRS(), cfg, nil) + + _, _, err := vpcRs.CreateTemplate() + Expect(err).NotTo(HaveOccurred()) + + vpcTemplate = &fakes.FakeTemplate{} + templateBody, err := vpcRs.RenderJSON() + Expect(err).ShouldNot(HaveOccurred()) + Expect(json.Unmarshal(templateBody, vpcTemplate)).To(Succeed()) + + assertSubnetSet := func(az, subnetKey string, cidrBlockIndex float64) { + Expect(vpcTemplate.Resources).To(HaveKey(subnetKey)) + expectedFnIPv4CIDR := `{ "Fn::Cidr": [{ "Fn::GetAtt": ["VPC", "CidrBlock"]}, 8, 13 ]}` + Expect(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"]).To(HaveLen(2)) + Expect(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"].([]interface{})[0].(float64)).To(Equal(cidrBlockIndex)) + actualFnCIDR, err := json.Marshal(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"].([]interface{})[1]) + Expect(err).NotTo(HaveOccurred()) + Expect(actualFnCIDR).To(MatchJSON([]byte(expectedFnIPv4CIDR))) + + expectedFnIPv6CIDR := `{ "Fn::Cidr": [{ "Fn::Select": [ 0, { "Fn::GetAtt": ["VPC", "Ipv6CidrBlocks"] }]}, 8, 64 ]}` + Expect(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"]).To(HaveLen(2)) + Expect(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"][0].(float64)).To(Equal(cidrBlockIndex)) + actualFnIPv6CIDR, err := json.Marshal(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"][1]) + Expect(err).NotTo(HaveOccurred()) + Expect(actualFnIPv6CIDR).To(MatchJSON([]byte(expectedFnIPv6CIDR))) + } + assertSubnetSet(azA, builder.PublicSubnetKey+azAFormatted, float64(0)) + assertSubnetSet(azB, builder.PublicSubnetKey+azBFormatted, float64(1)) + assertSubnetSet(azC, builder.PublicSubnetKey+azCFormatted, float64(2)) + + assertSubnetSet(azA, builder.PrivateSubnetKey+azAFormatted, float64(3)) + assertSubnetSet(azB, builder.PrivateSubnetKey+azBFormatted, float64(4)) + assertSubnetSet(azC, builder.PrivateSubnetKey+azCFormatted, float64(5)) + + }) + }) +}) diff --git a/pkg/vpc/vpc.go b/pkg/vpc/vpc.go index bb4320f47a..f987474f38 100644 --- a/pkg/vpc/vpc.go +++ b/pkg/vpc/vpc.go @@ -369,7 +369,7 @@ func importSubnetsForTopology(ec2API ec2iface.EC2API, spec *api.ClusterConfig, t return ImportSubnets(ec2API, spec, topology, subnets) } -// ImportSubnetsFromIDList will update spec with subnets _only specified by ID_ +// ImportSubnetsFromIDList will update cluster config with subnets _only specified by ID_ // then pass resulting subnets to ImportSubnets // NOTE: it does respect all fields set in spec.VPC, and will error if // there is a mismatch of local vs remote states From f89020f6a1ca169c2a705fc0da28d891d38cb189 Mon Sep 17 00:00:00 2001 From: Jake Date: Tue, 12 Oct 2021 11:54:07 +0100 Subject: [PATCH 2/9] wait for addons to be healthy only if there are nodegroups --- pkg/actions/addon/tasks.go | 6 ++-- pkg/cfn/builder/fakes/fake_cfn_template.go | 2 +- pkg/cfn/builder/vpc.go | 41 +++++++++++++++------- pkg/cfn/builder/vpc_ipv6.go | 6 ++-- pkg/cfn/builder/vpc_ipv6_test.go | 4 +-- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/pkg/actions/addon/tasks.go b/pkg/actions/addon/tasks.go index e8ce296128..49bb5f3a12 100644 --- a/pkg/actions/addon/tasks.go +++ b/pkg/actions/addon/tasks.go @@ -31,6 +31,7 @@ func CreateAddonTasks(cfg *api.ClusterConfig, clusterProvider *eks.ClusterProvid clusterProvider: clusterProvider, forceAll: forceAll, timeout: timeout, + wait: false, }, ) @@ -42,6 +43,7 @@ func CreateAddonTasks(cfg *api.ClusterConfig, clusterProvider *eks.ClusterProvid clusterProvider: clusterProvider, forceAll: forceAll, timeout: timeout, + wait: len(cfg.NodeGroups) > 0 || len(cfg.ManagedNodeGroups) > 0, }, ) return preTasks, postTasks @@ -52,7 +54,7 @@ type createAddonTask struct { cfg *api.ClusterConfig clusterProvider *eks.ClusterProvider addons []*api.Addon - forceAll bool + forceAll, wait bool timeout time.Duration } @@ -89,7 +91,7 @@ func (t *createAddonTask) Do(errorCh chan error) error { if t.forceAll { a.Force = true } - err := addonManager.Create(a, true) + err := addonManager.Create(a, t.wait) if err != nil { go func() { errorCh <- err diff --git a/pkg/cfn/builder/fakes/fake_cfn_template.go b/pkg/cfn/builder/fakes/fake_cfn_template.go index 49c8e07cb6..81147a403c 100644 --- a/pkg/cfn/builder/fakes/fake_cfn_template.go +++ b/pkg/cfn/builder/fakes/fake_cfn_template.go @@ -20,7 +20,7 @@ type Tag struct { } type Properties struct { - EnableDnsHostnames, EnableDnsSupport bool + EnableDNSHostnames, EnableDNSSupport bool GroupDescription string Description string Tags []Tag diff --git a/pkg/cfn/builder/vpc.go b/pkg/cfn/builder/vpc.go index 7fe4d18b0f..51b76dbfdb 100644 --- a/pkg/cfn/builder/vpc.go +++ b/pkg/cfn/builder/vpc.go @@ -7,18 +7,35 @@ import ( ) const ( - VPCResourceKey, IGWKey, GAKey = "VPC", "InternetGateway", "VPCGatewayAttachment" - IPv6CIDRBlockKey = "IPv6CidrBlock" - EgressOnlyInternetGatewayKey = "EgressOnlyInternetGateway" - ElasticIPKey = "EIP" - InternetCIDR, InternetIPv6CIDR = "0.0.0.0/0", "::/0" - PubRouteTableKey, PrivateRouteTableKey = "PublicRouteTable", "PrivateRouteTable" - PubRouteTableAssociation, PrivateRouteTableAssociation = "RouteTableAssociationPublic", "RouteTableAssociationPrivate" - PubSubRouteKey, PubSubIPv6RouteKey = "PublicSubnetDefaultRoute", "PublicSubnetIPv6DefaultRoute" - PrivateSubnetRouteKey, PrivateSubnetIpv6RouteKey = "PrivateSubnetDefaultRoute", "PrivateSubnetDefaultIpv6Route" - PublicSubnetKey, PrivateSubnetKey = "PublicSubnet", "PrivateSubnet" - NATGatewayKey = "NATGateway" - PublicSubnetsOutputKey, PrivateSubnetsOutputKey = "SubnetsPublic", "SubnetsPrivate" + VPCResourceKey = "VPC" + + // Gateways + IGWKey = "InternetGateway" + GAKey = "VPCGatewayAttachment" + EgressOnlyInternetGatewayKey = "EgressOnlyInternetGateway" + NATGatewayKey = "NATGateway" + ElasticIPKey = "EIP" + + // CIDRs + IPv6CIDRBlockKey = "IPv6CidrBlock" + InternetCIDR = "0.0.0.0/0" + InternetIPv6CIDR = "::/0" + + // Routing + PubRouteTableKey = "PublicRouteTable" + PrivateRouteTableKey = "PrivateRouteTable" + PubRouteTableAssociation = "RouteTableAssociationPublic" + PrivateRouteTableAssociation = "RouteTableAssociationPrivate" + PubSubRouteKey = "PublicSubnetDefaultRoute" + PubSubIPv6RouteKey = "PublicSubnetIPv6DefaultRoute" + PrivateSubnetRouteKey = "PrivateSubnetDefaultRoute" + PrivateSubnetIpv6RouteKey = "PrivateSubnetDefaultIpv6Route" + + // Subnets + PublicSubnetKey = "PublicSubnet" + PrivateSubnetKey = "PrivateSubnet" + PublicSubnetsOutputKey = "SubnetsPublic" + PrivateSubnetsOutputKey = "SubnetsPrivate" ) type VPCResourceSet interface { diff --git a/pkg/cfn/builder/vpc_ipv6.go b/pkg/cfn/builder/vpc_ipv6.go index 9d88870da0..cdb5a691b4 100644 --- a/pkg/cfn/builder/vpc_ipv6.go +++ b/pkg/cfn/builder/vpc_ipv6.go @@ -166,12 +166,12 @@ func (v *IPv6VPCResourceSet) RenderJSON() ([]byte, error) { func (v *IPv6VPCResourceSet) createSubnet(az, azFormatted string, i, cidrPartitions int, private bool) *gfnt.Value { var assignIpv6AddressOnCreation *gfnt.Value subnetKey := PublicSubnetKey + azFormatted - mapPublicIpOnLaunch := gfnt.True() + mapPublicIPOnLaunch := gfnt.True() elbTagKey := "kubernetes.io/role/elb" if private { subnetKey = PrivateSubnetKey + azFormatted - mapPublicIpOnLaunch = nil + mapPublicIPOnLaunch = nil assignIpv6AddressOnCreation = gfnt.True() elbTagKey = "kubernetes.io/role/internal-elb" } @@ -181,7 +181,7 @@ func (v *IPv6VPCResourceSet) createSubnet(az, azFormatted string, i, cidrPartiti AvailabilityZone: gfnt.NewString(az), CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(i), getSubnetIPv4CIDRBlock(cidrPartitions)), Ipv6CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(i), getSubnetIPv6CIDRBlock(cidrPartitions)), - MapPublicIpOnLaunch: mapPublicIpOnLaunch, + MapPublicIpOnLaunch: mapPublicIPOnLaunch, AssignIpv6AddressOnCreation: assignIpv6AddressOnCreation, VpcId: gfnt.MakeRef(VPCResourceKey), Tags: []cloudformation.Tag{{ diff --git a/pkg/cfn/builder/vpc_ipv6_test.go b/pkg/cfn/builder/vpc_ipv6_test.go index 88099558a1..22aac5ce9d 100644 --- a/pkg/cfn/builder/vpc_ipv6_test.go +++ b/pkg/cfn/builder/vpc_ipv6_test.go @@ -56,8 +56,8 @@ var _ = Describe("IPv6 VPC builder", func() { cidr := &defaultCidr Expect(vpcTemplate.Resources[builder.VPCResourceKey].Properties).To(Equal(fakes.Properties{ CidrBlock: cidr.String(), - EnableDnsHostnames: true, - EnableDnsSupport: true, + EnableDNSHostnames: true, + EnableDNSSupport: true, Tags: []fakes.Tag{ { Key: "Name", From 54c262a16443e77c8913d1844b8dee255f6c7762 Mon Sep 17 00:00:00 2001 From: nikimanoledaki <18622989+nikimanoledaki@users.noreply.github.com> Date: Wed, 13 Oct 2021 17:51:08 +0200 Subject: [PATCH 3/9] Passing IpFamily field to Control Plane CF template * Adding integration test set AssignIpv6AddressOnCreation to true for public subnets after the rest of the template. --- go.mod | 4 +- go.sum | 6 +- integration/tests/ipv6/ipv6_test.go | 72 ++++++++++++++++++++-- pkg/cfn/builder/cluster.go | 14 ++++- pkg/cfn/builder/cluster_test.go | 30 +++++---- pkg/cfn/builder/fakes/fake_cfn_template.go | 11 +++- pkg/cfn/builder/vpc_ipv6_test.go | 1 - 7 files changed, 112 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 1a6c73e5a3..1ccb98f7f1 100644 --- a/go.mod +++ b/go.mod @@ -295,7 +295,7 @@ require ( github.com/uudashr/gocognit v1.0.5 // indirect github.com/vektra/mockery v1.1.2 github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 // indirect - github.com/weaveworks/goformation/v4 v4.10.2-0.20210609082249-532b27315cf1 + github.com/weaveworks/goformation/v4 v4.10.2-0.20211012141859-cd360fb1f843 github.com/weaveworks/launcher v0.0.2-0.20200715141516-1ca323f1de15 github.com/weaveworks/schemer v0.0.0-20210802122110-338b258ad2ca github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0 @@ -372,6 +372,8 @@ require ( sigs.k8s.io/yaml v1.2.0 ) +require github.com/xgfone/netaddr v0.5.1 + require ( github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cncf/udpa/go v0.0.0-20210322005330-6414d713912e // indirect diff --git a/go.sum b/go.sum index 7bdadb34d6..1f240763ce 100644 --- a/go.sum +++ b/go.sum @@ -1669,8 +1669,8 @@ github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 h1:txplJASvd6b github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2/go.mod h1:DGCIhurYgnLz8J9ga1fMV/fbLDyUvTyrWXVWUIyJon4= github.com/weaveworks/aws-sdk-go v0.0.0-20211026093156-d6e6822f58db h1:K6lacvb3qzF/bHvx2RsPDw8cYA8VccOecn9e6xDEBY0= github.com/weaveworks/aws-sdk-go v0.0.0-20211026093156-d6e6822f58db/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/weaveworks/goformation/v4 v4.10.2-0.20210609082249-532b27315cf1 h1:yX/flUgj/znRvfhEZ3mC8RdKV+GNwq/9j+FkWN7ve+o= -github.com/weaveworks/goformation/v4 v4.10.2-0.20210609082249-532b27315cf1/go.mod h1:x92o12+Azh6DQ4yoXT5oEuE7dhQHR5V2vy/fmZ6pO7k= +github.com/weaveworks/goformation/v4 v4.10.2-0.20211012141859-cd360fb1f843 h1:9v19OzMM+kFcm0r2yZeoMMAvT71H/apnNWeoMKMxUz0= +github.com/weaveworks/goformation/v4 v4.10.2-0.20211012141859-cd360fb1f843/go.mod h1:x92o12+Azh6DQ4yoXT5oEuE7dhQHR5V2vy/fmZ6pO7k= github.com/weaveworks/launcher v0.0.2-0.20200715141516-1ca323f1de15 h1:i/RhLevywqC6cuUWtGdoaNrsJd+/zWh3PXbkXZIyZsU= github.com/weaveworks/launcher v0.0.2-0.20200715141516-1ca323f1de15/go.mod h1:w9Z1vnQmPobkEZ0F3oyiqRYP+62qDqTGnK6t5uhe1kg= github.com/weaveworks/mesh v0.0.0-20170419100114-1f158d31de55/go.mod h1:mcON9Ws1aW0crSErpXWp7U1ErCDEKliDX2OhVlbWRKk= @@ -1695,6 +1695,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180816142147-da425ebb7609/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56 h1:yhqBHs09SmmUoNOHc9jgK4a60T3XFRtPAkYxVnqgY50= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xgfone/netaddr v0.5.1 h1:87DhCyyR6XUr0p63JHTDT5juGDhH49Ak2ePZNBmSL5I= +github.com/xgfone/netaddr v0.5.1/go.mod h1:QDEYI/4nQfAtNj7TB4RhYQY1B4U31Edj+SOoDEuIfsQ= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= diff --git a/integration/tests/ipv6/ipv6_test.go b/integration/tests/ipv6/ipv6_test.go index c022ac0ada..8568cea6a4 100644 --- a/integration/tests/ipv6/ipv6_test.go +++ b/integration/tests/ipv6/ipv6_test.go @@ -5,8 +5,10 @@ package ipv6 import ( "bytes" + "context" "encoding/json" "fmt" + "strings" "testing" "github.com/aws/aws-sdk-go/aws" @@ -16,10 +18,16 @@ import ( . "github.com/weaveworks/eksctl/integration/runner" "github.com/weaveworks/eksctl/integration/tests" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/builder" + "github.com/weaveworks/eksctl/pkg/eks" "github.com/weaveworks/eksctl/pkg/testutils" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/xgfone/netaddr" ) var params *tests.Params @@ -35,12 +43,15 @@ func TestIPv6(t *testing.T) { } var _ = Describe("(Integration) [EKS IPv6 test]", func() { + var ( + clusterConfig *api.ClusterConfig + ) Context("Creating a cluster with IPv6", func() { clusterName := params.NewClusterName("ipv6") BeforeSuite(func() { - clusterConfig := api.NewClusterConfig() + clusterConfig = api.NewClusterConfig() clusterConfig.Metadata.Name = clusterName clusterConfig.Metadata.Version = "latest" clusterConfig.Metadata.Region = params.Region @@ -81,7 +92,7 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { }) It("should support ipv6", func() { - By("Asserting that the VPC that is created has a IPv6 CIDR") + By("creating a VPC that has an IPv6 CIDR") awsSession := NewSession(params.Region) cfnSession := cfn.New(awsSession) @@ -93,17 +104,66 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { var vpcID string for _, output := range describeStackOut.Stacks[0].Outputs { - if *output.OutputKey == "VPC" { + if *output.OutputKey == builder.VPCResourceKey { vpcID = *output.OutputValue } } ec2 := awsec2.New(awsSession) - output, err := ec2.DescribeVpcs(&awsec2.DescribeVpcsInput{ + vpcOutput, err := ec2.DescribeVpcs(&awsec2.DescribeVpcsInput{ VpcIds: aws.StringSlice([]string{vpcID}), }) - Expect(err).NotTo(HaveOccurred(), output.GoString()) - Expect(output.Vpcs[0].Ipv6CidrBlockAssociationSet).To(HaveLen(1)) + Expect(err).NotTo(HaveOccurred(), vpcOutput.GoString()) + Expect(vpcOutput.Vpcs[0].Ipv6CidrBlockAssociationSet).To(HaveLen(1)) + + // TODO: get rid of this once CF bug is fixed + By("setting AssignIpv6AddressOnCreation to true for each public subnet") + var publicSubnets string + for _, output := range describeStackOut.Stacks[0].Outputs { + if *output.OutputKey == builder.PublicSubnetsOutputKey { + publicSubnets = *output.OutputValue + } + } + + subnetsOutput, err := ec2.DescribeSubnets(&awsec2.DescribeSubnetsInput{ + SubnetIds: aws.StringSlice(strings.Split(publicSubnets, ",")), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(len(subnetsOutput.Subnets)).To(BeNumerically(">", 0)) + for _, s := range subnetsOutput.Subnets { + Expect(s.AssignIpv6AddressOnCreation).NotTo(BeNil()) + Expect(*s.AssignIpv6AddressOnCreation).To(BeTrue()) + } + + By("the k8s cluster's having an IP family of IPv6") + var clientSet *kubernetes.Clientset + ctl, err := eks.New(&api.ProviderConfig{Region: params.Region}, clusterConfig) + Expect(err).NotTo(HaveOccurred()) + err = ctl.RefreshClusterStatus(clusterConfig) + Expect(err).ShouldNot(HaveOccurred()) + clientSet, err = ctl.NewStdClientSet(clusterConfig) + Expect(err).ShouldNot(HaveOccurred()) + + svcName := "IPv6Service" + _, err = clientSet.CoreV1().Services("default").Create(context.TODO(), &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + }, + Spec: corev1.ServiceSpec{ + IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol}, + Selector: map[string]string{"app": "IPv6App"}, + Ports: []corev1.ServicePort{corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + Port: 80, + }}, + }, + }, metav1.CreateOptions{}) + Expect(err).ShouldNot(HaveOccurred()) + + svc, err := clientSet.CoreV1().Services("default").Get(context.TODO(), svcName, metav1.GetOptions{}) + svcIP, err := netaddr.NewIPAddress(svc.Spec.ClusterIP) + Expect(err).NotTo(HaveOccurred()) + Expect(svcIP.Version()).To(Equal(6)) }) }) }) diff --git a/pkg/cfn/builder/cluster.go b/pkg/cfn/builder/cluster.go index 47ec6160a1..52ab8eeffb 100644 --- a/pkg/cfn/builder/cluster.go +++ b/pkg/cfn/builder/cluster.go @@ -3,6 +3,7 @@ package builder import ( "encoding/base64" "fmt" + "strings" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" @@ -168,10 +169,17 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDe ResourcesVpcConfig: clusterVPC, EncryptionConfig: encryptionConfigs, } + + cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ + IpFamily: gfnt.NewString(strings.ToLower(string(api.IPV4Family))), + } + + if c.spec.VPC.IPFamily != nil && *c.spec.VPC.IPFamily == string(api.IPV6Family) { + cluster.KubernetesNetworkConfig.IpFamily = gfnt.NewString(strings.ToLower(string(api.IPV6Family))) + } + if c.spec.KubernetesNetworkConfig != nil && c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR != "" { - cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ - ServiceIpv4Cidr: gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR), - } + cluster.KubernetesNetworkConfig.ServiceIpv4Cidr = gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR) } c.newResource("ControlPlane", &cluster) diff --git a/pkg/cfn/builder/cluster_test.go b/pkg/cfn/builder/cluster_test.go index 16cd3d8ff7..84cbf7ee6c 100644 --- a/pkg/cfn/builder/cluster_test.go +++ b/pkg/cfn/builder/cluster_test.go @@ -34,6 +34,9 @@ var _ = Describe("Cluster Template Builder", func() { cfg.VPC = vpcConfig() cfg.AvailabilityZones = []string{"us-west-2a", "us-west-2b"} cfg.VPC.IPFamily = aws.String(string(api.IPV4Family)) + cfg.KubernetesNetworkConfig = &api.KubernetesNetworkConfig{ + ServiceIPv4CIDR: "131.10.55.70/18", + } }) JustBeforeEach(func() { @@ -62,6 +65,18 @@ var _ = Describe("Cluster Template Builder", func() { Expect(clusterTemplate.Description).To(Equal("EKS cluster (dedicated VPC: true, dedicated IAM: true) [created and managed by eksctl]")) }) + It("should add control plane resources", func() { + Expect(clusterTemplate.Resources).To(HaveKey("ControlPlane")) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.Name).To(Equal(cfg.Metadata.Name)) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.Version).To(Equal(cfg.Metadata.Version)) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SecurityGroupIds[0]).To(ContainElement("ControlPlaneSecurityGroup")) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SubnetIds).To(HaveLen(4)) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.RoleArn).To(ContainElement([]interface{}{"ServiceRole", "Arn"})) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.EncryptionConfig).To(BeNil()) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.ServiceIPv4CIDR).To(Equal("131.10.55.70/18")) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv4")) + }) + It("should add vpc resources", func() { Expect(clusterTemplate.Resources).To(HaveKey(vpcResourceKey)) Expect(clusterTemplate.Resources).To(HaveKey(igwKey)) @@ -87,6 +102,10 @@ var _ = Describe("Cluster Template Builder", func() { cfg.VPC.IPFamily = aws.String(string(api.IPV6Family)) }) + It("should add control plane resources", func() { + Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv6")) + }) + It("should add IPv6 vpc resources", func() { Expect(clusterTemplate.Resources).To(HaveKey(builder.VPCResourceKey)) Expect(clusterTemplate.Resources).To(HaveKey(builder.IPv6CIDRBlockKey)) @@ -118,7 +137,6 @@ var _ = Describe("Cluster Template Builder", func() { Expect(clusterTemplate.Resources).To(HaveKey(builder.PubRouteTableAssociation + azBFormatted)) Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateRouteTableAssociation + azAFormatted)) Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateRouteTableAssociation + azBFormatted)) - }) }) @@ -280,16 +298,6 @@ var _ = Describe("Cluster Template Builder", func() { }) }) - It("should add control plane resources", func() { - Expect(clusterTemplate.Resources).To(HaveKey("ControlPlane")) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.Name).To(Equal(cfg.Metadata.Name)) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.Version).To(Equal(cfg.Metadata.Version)) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SecurityGroupIds[0]).To(ContainElement("ControlPlaneSecurityGroup")) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SubnetIds).To(HaveLen(4)) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.RoleArn).To(ContainElement([]interface{}{"ServiceRole", "Arn"})) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.EncryptionConfig).To(BeNil()) - }) - When("SecretsEncryption is configured", func() { BeforeEach(func() { cfg.SecretsEncryption = &api.SecretsEncryption{ diff --git a/pkg/cfn/builder/fakes/fake_cfn_template.go b/pkg/cfn/builder/fakes/fake_cfn_template.go index 81147a403c..d01fb8f9f4 100644 --- a/pkg/cfn/builder/fakes/fake_cfn_template.go +++ b/pkg/cfn/builder/fakes/fake_cfn_template.go @@ -66,8 +66,9 @@ type Properties struct { MapPublicIPOnLaunch bool AssignIpv6AddressOnCreation *bool - Ipv6CidrBlock map[string][]interface{} - CidrBlock interface{} + Ipv6CidrBlock map[string][]interface{} + CidrBlock interface{} + KubernetesNetworkConfig KubernetesNetworkConfig AmazonProvidedIpv6CidrBlock bool AvailabilityZone, Domain string @@ -111,6 +112,12 @@ type Properties struct { } } +type KubernetesNetworkConfig struct { + ServiceIPv4CIDR string + ServiceIPv6CIDR interface{} + IPFamily string +} + type SGIngress struct { SourceSecurityGroupID interface{} FromPort float64 diff --git a/pkg/cfn/builder/vpc_ipv6_test.go b/pkg/cfn/builder/vpc_ipv6_test.go index 22aac5ce9d..be26bfa518 100644 --- a/pkg/cfn/builder/vpc_ipv6_test.go +++ b/pkg/cfn/builder/vpc_ipv6_test.go @@ -341,7 +341,6 @@ var _ = Describe("IPv6 VPC builder", func() { "Fn::Sub": fmt.Sprintf("${AWS::StackName}::%s", outputs.ClusterSubnetsPrivate), }, })) - }) Context("when there are 3 AZs", func() { From e787a7266ef95a69ebb251882f471efa2c74c995 Mon Sep 17 00:00:00 2001 From: nikimanoledaki <18622989+nikimanoledaki@users.noreply.github.com> Date: Thu, 14 Oct 2021 13:16:08 +0200 Subject: [PATCH 4/9] Commenting ipFamily test and code due to CF bug - Revert me! --- integration/tests/ipv6/ipv6_test.go | 65 +++++++++++++---------------- pkg/cfn/builder/cluster.go | 24 +++++++---- pkg/cfn/builder/cluster_test.go | 6 ++- 3 files changed, 49 insertions(+), 46 deletions(-) diff --git a/integration/tests/ipv6/ipv6_test.go b/integration/tests/ipv6/ipv6_test.go index 8568cea6a4..4bff5df063 100644 --- a/integration/tests/ipv6/ipv6_test.go +++ b/integration/tests/ipv6/ipv6_test.go @@ -5,7 +5,6 @@ package ipv6 import ( "bytes" - "context" "encoding/json" "fmt" "strings" @@ -19,15 +18,10 @@ import ( "github.com/weaveworks/eksctl/integration/tests" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/cfn/builder" - "github.com/weaveworks/eksctl/pkg/eks" "github.com/weaveworks/eksctl/pkg/testutils" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/xgfone/netaddr" ) var params *tests.Params @@ -135,35 +129,36 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { Expect(*s.AssignIpv6AddressOnCreation).To(BeTrue()) } - By("the k8s cluster's having an IP family of IPv6") - var clientSet *kubernetes.Clientset - ctl, err := eks.New(&api.ProviderConfig{Region: params.Region}, clusterConfig) - Expect(err).NotTo(HaveOccurred()) - err = ctl.RefreshClusterStatus(clusterConfig) - Expect(err).ShouldNot(HaveOccurred()) - clientSet, err = ctl.NewStdClientSet(clusterConfig) - Expect(err).ShouldNot(HaveOccurred()) - - svcName := "IPv6Service" - _, err = clientSet.CoreV1().Services("default").Create(context.TODO(), &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: svcName, - }, - Spec: corev1.ServiceSpec{ - IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol}, - Selector: map[string]string{"app": "IPv6App"}, - Ports: []corev1.ServicePort{corev1.ServicePort{ - Protocol: corev1.ProtocolTCP, - Port: 80, - }}, - }, - }, metav1.CreateOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - - svc, err := clientSet.CoreV1().Services("default").Get(context.TODO(), svcName, metav1.GetOptions{}) - svcIP, err := netaddr.NewIPAddress(svc.Spec.ClusterIP) - Expect(err).NotTo(HaveOccurred()) - Expect(svcIP.Version()).To(Equal(6)) + // TODO: Uncomment and run the following test once CF handler bug is fixed from EKS side. + // By("the k8s cluster's having an IP family of IPv6") + // var clientSet *kubernetes.Clientset + // ctl, err := eks.New(&api.ProviderConfig{Region: params.Region}, clusterConfig) + // Expect(err).NotTo(HaveOccurred()) + // err = ctl.RefreshClusterStatus(clusterConfig) + // Expect(err).ShouldNot(HaveOccurred()) + // clientSet, err = ctl.NewStdClientSet(clusterConfig) + // Expect(err).ShouldNot(HaveOccurred()) + + // svcName := "IPv6Service" + // _, err = clientSet.CoreV1().Services("default").Create(context.TODO(), &corev1.Service{ + // ObjectMeta: metav1.ObjectMeta{ + // Name: svcName, + // }, + // Spec: corev1.ServiceSpec{ + // IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol}, + // Selector: map[string]string{"app": "IPv6App"}, + // Ports: []corev1.ServicePort{corev1.ServicePort{ + // Protocol: corev1.ProtocolTCP, + // Port: 80, + // }}, + // }, + // }, metav1.CreateOptions{}) + // Expect(err).ShouldNot(HaveOccurred()) + + // svc, err := clientSet.CoreV1().Services("default").Get(context.TODO(), svcName, metav1.GetOptions{}) + // svcIP, err := netaddr.NewIPAddress(svc.Spec.ClusterIP) + // Expect(err).NotTo(HaveOccurred()) + // Expect(svcIP.Version()).To(Equal(6)) }) }) }) diff --git a/pkg/cfn/builder/cluster.go b/pkg/cfn/builder/cluster.go index 52ab8eeffb..f1d993ad5e 100644 --- a/pkg/cfn/builder/cluster.go +++ b/pkg/cfn/builder/cluster.go @@ -3,7 +3,6 @@ package builder import ( "encoding/base64" "fmt" - "strings" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" @@ -170,17 +169,24 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDe EncryptionConfig: encryptionConfigs, } - cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ - IpFamily: gfnt.NewString(strings.ToLower(string(api.IPV4Family))), + //TODO: delete this below If statement, and uncomment the code below once we can create ipv6 clusters + if c.spec.KubernetesNetworkConfig != nil && c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR != "" { + cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ + ServiceIpv4Cidr: gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR), + } } - if c.spec.VPC.IPFamily != nil && *c.spec.VPC.IPFamily == string(api.IPV6Family) { - cluster.KubernetesNetworkConfig.IpFamily = gfnt.NewString(strings.ToLower(string(api.IPV6Family))) - } + // cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ + // IpFamily: gfnt.NewString(strings.ToLower(string(api.IPV4Family))), + // } - if c.spec.KubernetesNetworkConfig != nil && c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR != "" { - cluster.KubernetesNetworkConfig.ServiceIpv4Cidr = gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR) - } + // if c.spec.VPC.IPFamily != nil && *c.spec.VPC.IPFamily == string(api.IPV6Family) { + // cluster.KubernetesNetworkConfig.IpFamily = gfnt.NewString(strings.ToLower(string(api.IPV6Family))) + // } + + // if c.spec.KubernetesNetworkConfig != nil && c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR != "" { + // cluster.KubernetesNetworkConfig.ServiceIpv4Cidr = gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR) + // } c.newResource("ControlPlane", &cluster) diff --git a/pkg/cfn/builder/cluster_test.go b/pkg/cfn/builder/cluster_test.go index 84cbf7ee6c..18d7856441 100644 --- a/pkg/cfn/builder/cluster_test.go +++ b/pkg/cfn/builder/cluster_test.go @@ -74,7 +74,8 @@ var _ = Describe("Cluster Template Builder", func() { Expect(clusterTemplate.Resources["ControlPlane"].Properties.RoleArn).To(ContainElement([]interface{}{"ServiceRole", "Arn"})) Expect(clusterTemplate.Resources["ControlPlane"].Properties.EncryptionConfig).To(BeNil()) Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.ServiceIPv4CIDR).To(Equal("131.10.55.70/18")) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv4")) + // TODO uncomment once CF handler bug is fixed + // Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv4")) }) It("should add vpc resources", func() { @@ -102,7 +103,8 @@ var _ = Describe("Cluster Template Builder", func() { cfg.VPC.IPFamily = aws.String(string(api.IPV6Family)) }) - It("should add control plane resources", func() { + // TODO: Unpend once CF handler bug is fixed on EKS side. + PIt("should add control plane resources", func() { Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv6")) }) From 827c47e4ca054a969ee8b4659e762a7af1b655d1 Mon Sep 17 00:00:00 2001 From: nikimanoledaki <18622989+nikimanoledaki@users.noreply.github.com> Date: Thu, 14 Oct 2021 13:17:51 +0200 Subject: [PATCH 5/9] Adding AssignIpv6AddressOnCreation task after cluster creation due to CF bug - AssignIpv6AddressOnCreation also needs to be set on public subnets, but due to a current bug in CF, this cannot be set alongside MapPublicIpOnLaunch at create time. This means we need to add it "manually" by hitting the VPC API to update each public subnet after launch. - Added extra validation that NAT is nil --- integration/tests/ipv6/ipv6_test.go | 3 +- pkg/cfn/manager/create_tasks.go | 9 ++ pkg/cfn/manager/create_tasks_test.go | 90 ++++++++++++++++ pkg/cfn/manager/tasks.go | 28 +++++ pkg/cfn/manager/tasks_test.go | 153 ++++++++++++++------------- pkg/ctl/cmdutils/configfile.go | 6 +- pkg/ctl/cmdutils/configfile_test.go | 16 +++ 7 files changed, 228 insertions(+), 77 deletions(-) create mode 100644 pkg/cfn/manager/create_tasks_test.go diff --git a/integration/tests/ipv6/ipv6_test.go b/integration/tests/ipv6/ipv6_test.go index 4bff5df063..91dbb20256 100644 --- a/integration/tests/ipv6/ipv6_test.go +++ b/integration/tests/ipv6/ipv6_test.go @@ -47,9 +47,10 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { BeforeSuite(func() { clusterConfig = api.NewClusterConfig() clusterConfig.Metadata.Name = clusterName - clusterConfig.Metadata.Version = "latest" + clusterConfig.Metadata.Version = "1.21" clusterConfig.Metadata.Region = params.Region clusterConfig.VPC.IPFamily = aws.String("IPv6") + clusterConfig.VPC.NAT = nil clusterConfig.IAM.WithOIDC = api.Enabled() clusterConfig.Addons = []*api.Addon{ { diff --git a/pkg/cfn/manager/create_tasks.go b/pkg/cfn/manager/create_tasks.go index a038e31aea..72c2dfb661 100644 --- a/pkg/cfn/manager/create_tasks.go +++ b/pkg/cfn/manager/create_tasks.go @@ -33,6 +33,15 @@ func (c *StackCollection) NewTasksToCreateClusterWithNodeGroups(nodeGroups []*ap }, ) + if c.spec.VPC.IPFamily != nil && *c.spec.VPC.IPFamily == string(api.IPV6Family) { + taskTree.Append( + &AssignIpv6AddressOnCreationTask{ + ClusterConfig: c.spec, + EC2API: c.ec2API, + }, + ) + } + appendNodeGroupTasksTo := func(taskTree *tasks.TaskTree) { vpcImporter := vpc.NewStackConfigImporter(c.MakeClusterStackName()) nodeGroupTasks := c.NewUnmanagedNodeGroupTask(nodeGroups, false, vpcImporter) diff --git a/pkg/cfn/manager/create_tasks_test.go b/pkg/cfn/manager/create_tasks_test.go new file mode 100644 index 0000000000..d3b70b749b --- /dev/null +++ b/pkg/cfn/manager/create_tasks_test.go @@ -0,0 +1,90 @@ +package manager_test + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/manager" + "github.com/weaveworks/eksctl/pkg/testutils/mockprovider" +) + +var _ = Describe("CreateTasks", func() { + var subnetIDs = []string{"123", "456"} + var clusterConfig *api.ClusterConfig + Context("AssignIpv6AddressOnCreationTask", func() { + BeforeEach(func() { + clusterConfig = api.NewClusterConfig() + clusterConfig.VPC.Subnets = &api.ClusterSubnets{} + clusterConfig.VPC.Subnets.Public = map[string]api.AZSubnetSpec{ + "0": {ID: subnetIDs[0]}, + "1": {ID: subnetIDs[1]}, + } + }) + + It("sets AssignIpv6AddressOnCreation to true for all public subnets", func() { + modifySubnetAttributeCallCount := 0 + p := mockprovider.NewMockProvider() + mockCall1 := p.MockEC2().On("ModifySubnetAttribute", &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String(subnetIDs[0]), + }).Return(&ec2.ModifySubnetAttributeOutput{}, nil) + + mockCall1.RunFn = func(_ mock.Arguments) { + modifySubnetAttributeCallCount++ + } + + mockCall2 := p.MockEC2().On("ModifySubnetAttribute", &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String(subnetIDs[1]), + }).Return(&ec2.ModifySubnetAttributeOutput{}, nil) + + mockCall2.RunFn = func(_ mock.Arguments) { + modifySubnetAttributeCallCount++ + } + + task := manager.AssignIpv6AddressOnCreationTask{ + EC2API: p.EC2(), + ClusterConfig: clusterConfig, + } + errorCh := make(chan error) + err := task.Do(errorCh) + Expect(err).NotTo(HaveOccurred()) + Expect(modifySubnetAttributeCallCount).To(Equal(2)) + + By("closing the error channel") + Eventually(errorCh).Should(BeClosed()) + }) + + When("the API call errors", func() { + It("errors", func() { + p := mockprovider.NewMockProvider() + p.MockEC2().On("ModifySubnetAttribute", &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String(subnetIDs[0]), + }).Return(&ec2.ModifySubnetAttributeOutput{}, fmt.Errorf("foo")) + + task := manager.AssignIpv6AddressOnCreationTask{ + EC2API: p.EC2(), + ClusterConfig: clusterConfig, + } + errorCh := make(chan error) + err := task.Do(errorCh) + Expect(err).To(MatchError("failed to update subnet \"123\": foo")) + + By("closing the error channel") + Eventually(errorCh).Should(BeClosed()) + }) + }) + }) +}) diff --git a/pkg/cfn/manager/tasks.go b/pkg/cfn/manager/tasks.go index a4f1a36c03..e0ca39cf75 100644 --- a/pkg/cfn/manager/tasks.go +++ b/pkg/cfn/manager/tasks.go @@ -6,6 +6,9 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" iamoidc "github.com/weaveworks/eksctl/pkg/iam/oidc" kubewrapper "github.com/weaveworks/eksctl/pkg/kubernetes" @@ -131,3 +134,28 @@ func (t *kubernetesTask) Do(errs chan error) error { close(errs) return err } + +type AssignIpv6AddressOnCreationTask struct { + EC2API ec2iface.EC2API + ClusterConfig *api.ClusterConfig +} + +func (t *AssignIpv6AddressOnCreationTask) Describe() string { + return "set AssignIpv6AddressOnCreation to true for public subnets" +} + +func (t *AssignIpv6AddressOnCreationTask) Do(errs chan error) error { + defer close(errs) + for _, subnet := range t.ClusterConfig.VPC.Subnets.Public.WithIDs() { + _, err := t.EC2API.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String(subnet), + }) + if err != nil { + return fmt.Errorf("failed to update subnet %q: %v", subnet, err) + } + } + return nil +} diff --git a/pkg/cfn/manager/tasks_test.go b/pkg/cfn/manager/tasks_test.go index 37b6481398..496ff5afee 100644 --- a/pkg/cfn/manager/tasks_test.go +++ b/pkg/cfn/manager/tasks_test.go @@ -3,6 +3,7 @@ package manager import ( "fmt" + "github.com/aws/aws-sdk-go/aws" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" @@ -59,83 +60,67 @@ var _ = Describe("StackCollection Tasks", func() { } Describe("TaskTree", func() { - Context("With real tasks", func() { - - BeforeEach(func() { - - p = mockprovider.NewMockProvider() + BeforeEach(func() { + p = mockprovider.NewMockProvider() + cfg = newClusterConfig("test-cluster") + stackManager = NewStackCollection(p, cfg) + }) - cfg = newClusterConfig("test-cluster") + It("should have nice description", func() { + fakeVPCImporter := new(vpcfakes.FakeImporter) + // TODO use DescribeTable + + // The supportsManagedNodes argument has no effect on the Describe call, so the values are alternated + // in these tests + { + tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("bar", "foo"), false, fakeVPCImporter) + Expect(tasks.Describe()).To(Equal(`2 parallel tasks: { create nodegroup "bar", create nodegroup "foo" }`)) + } + { + tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("bar"), false, fakeVPCImporter) + Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "bar" }`)) + } + { + tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("foo"), false, fakeVPCImporter) + Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "foo" }`)) + } + { + tasks := stackManager.NewUnmanagedNodeGroupTask(nil, false, fakeVPCImporter) + Expect(tasks.Describe()).To(Equal(`no tasks`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), nil, true) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo" } }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar"), nil, false) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", create nodegroup "bar" }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(nil, nil, true) + Expect(tasks.Describe()).To(Equal(`1 task: { create cluster control plane "test-cluster" }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), makeManagedNodeGroups("m1", "m2"), false) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 4 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo", create managed nodegroup "m1", create managed nodegroup "m2" } }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("foo"), makeManagedNodeGroups("m1"), true) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 parallel sub-tasks: { create nodegroup "foo", create managed nodegroup "m1" } }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar"), nil, false, &task{id: 1}) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 sequential sub-tasks: { task 1, create nodegroup "bar" } }`)) + } + }) - stackManager = NewStackCollection(p, cfg) + When("IPFamily is set to ipv6", func() { + BeforeEach(func() { + cfg.VPC.IPFamily = aws.String(string(api.IPV6Family)) }) - - It("should have nice description", func() { - makeNodeGroups := func(names ...string) []*api.NodeGroup { - var nodeGroups []*api.NodeGroup - for _, name := range names { - ng := api.NewNodeGroup() - ng.Name = name - nodeGroups = append(nodeGroups, ng) - } - return nodeGroups - } - - makeManagedNodeGroups := func(names ...string) []*api.ManagedNodeGroup { - var managedNodeGroups []*api.ManagedNodeGroup - for _, name := range names { - ng := api.NewManagedNodeGroup() - ng.Name = name - managedNodeGroups = append(managedNodeGroups, ng) - } - return managedNodeGroups - } - - fakeVPCImporter := new(vpcfakes.FakeImporter) - // TODO use DescribeTable - - // The supportsManagedNodes argument has no effect on the Describe call, so the values are alternated - // in these tests - { - tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("bar", "foo"), false, fakeVPCImporter) - Expect(tasks.Describe()).To(Equal(`2 parallel tasks: { create nodegroup "bar", create nodegroup "foo" }`)) - } - { - tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("bar"), false, fakeVPCImporter) - Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "bar" }`)) - } - { - tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("foo"), false, fakeVPCImporter) - Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "foo" }`)) - } - { - tasks := stackManager.NewUnmanagedNodeGroupTask(nil, false, fakeVPCImporter) - Expect(tasks.Describe()).To(Equal(`no tasks`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), nil, true) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo" } }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar"), nil, false) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", create nodegroup "bar" }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(nil, nil, true) - Expect(tasks.Describe()).To(Equal(`1 task: { create cluster control plane "test-cluster" }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), makeManagedNodeGroups("m1", "m2"), false) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 4 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo", create managed nodegroup "m1", create managed nodegroup "m2" } }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("foo"), makeManagedNodeGroups("m1"), true) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 parallel sub-tasks: { create nodegroup "foo", create managed nodegroup "m1" } }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar"), nil, false, &task{id: 1}) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 sequential sub-tasks: { task 1, create nodegroup "bar" } }`)) - } + It("appends the AssignIpv6AddressOnCreation task to occur after the cluster creation", func() { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), nil, true) + Expect(tasks.Describe()).To(Equal(`3 sequential tasks: { create cluster control plane "test-cluster", set AssignIpv6AddressOnCreation to true for public subnets, 2 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo" } }`)) }) }) }) @@ -175,3 +160,23 @@ var _ = Describe("StackCollection Tasks", func() { }) }) }) + +func makeNodeGroups(names ...string) []*api.NodeGroup { + var nodeGroups []*api.NodeGroup + for _, name := range names { + ng := api.NewNodeGroup() + ng.Name = name + nodeGroups = append(nodeGroups, ng) + } + return nodeGroups +} + +func makeManagedNodeGroups(names ...string) []*api.ManagedNodeGroup { + var managedNodeGroups []*api.ManagedNodeGroup + for _, name := range names { + ng := api.NewManagedNodeGroup() + ng.Name = name + managedNodeGroups = append(managedNodeGroups, ng) + } + return managedNodeGroups +} diff --git a/pkg/ctl/cmdutils/configfile.go b/pkg/ctl/cmdutils/configfile.go index 9b0dd42c71..854b8c158f 100644 --- a/pkg/ctl/cmdutils/configfile.go +++ b/pkg/ctl/cmdutils/configfile.go @@ -238,10 +238,12 @@ func NewCreateClusterLoader(cmd *Cmd, ngFilter *filter.NodeGroupFilter, ng *api. } if clusterConfig.VPC.NAT == nil { - clusterConfig.VPC.NAT = api.DefaultClusterNAT() + if clusterConfig.VPC.IPFamily == nil || *clusterConfig.VPC.IPFamily != string(api.IPV6Family) { + clusterConfig.VPC.NAT = api.DefaultClusterNAT() + } } - if !api.IsSetAndNonEmptyString(clusterConfig.VPC.NAT.Gateway) { + if clusterConfig.VPC.NAT != nil && !api.IsSetAndNonEmptyString(clusterConfig.VPC.NAT.Gateway) { *clusterConfig.VPC.NAT.Gateway = api.ClusterSingleNAT } diff --git a/pkg/ctl/cmdutils/configfile_test.go b/pkg/ctl/cmdutils/configfile_test.go index 93281c3542..bd33dcafae 100644 --- a/pkg/ctl/cmdutils/configfile_test.go +++ b/pkg/ctl/cmdutils/configfile_test.go @@ -200,6 +200,22 @@ var _ = Describe("cmdutils configfile", func() { } }) + When("using ipv6", func() { + It("should default VPC.NAT to nil", func() { + cmd := &Cmd{ + CobraCommand: newCmd(), + ClusterConfigFile: filepath.Join(examplesDir, "29-vpc-with-ip-family.yaml"), + ClusterConfig: api.NewClusterConfig(), + ProviderConfig: api.ProviderConfig{}, + } + params := &CreateClusterCmdParams{WithoutNodeGroup: true, CreateManagedNGOptions: CreateManagedNGOptions{ + Managed: false, + }} + Expect(NewCreateClusterLoader(cmd, filter.NewNodeGroupFilter(), nil, params).Load()).To(Succeed()) + Expect(cmd.ClusterConfig.VPC.NAT).To(BeNil()) + }) + }) + It("loader should handle named and unnamed nodegroups without config file", func() { unnamedNG := api.NewNodeGroup() From c68641b3c71e0df8954c228e5f7e9bf7a5eaa215 Mon Sep 17 00:00:00 2001 From: nikimanoledaki <18622989+nikimanoledaki@users.noreply.github.com> Date: Fri, 15 Oct 2021 13:45:43 +0200 Subject: [PATCH 6/9] Revert "Commenting ipFamily test and code due to CF bug" This reverts commit dc2c50a86b22f11bea015c33bb8f77f0919736e8. --- integration/tests/ipv6/ipv6_test.go | 65 ++++++++++++++++------------- pkg/cfn/builder/cluster.go | 24 ++++------- pkg/cfn/builder/cluster_test.go | 6 +-- 3 files changed, 46 insertions(+), 49 deletions(-) diff --git a/integration/tests/ipv6/ipv6_test.go b/integration/tests/ipv6/ipv6_test.go index 91dbb20256..d71f12b16f 100644 --- a/integration/tests/ipv6/ipv6_test.go +++ b/integration/tests/ipv6/ipv6_test.go @@ -5,6 +5,7 @@ package ipv6 import ( "bytes" + "context" "encoding/json" "fmt" "strings" @@ -18,10 +19,15 @@ import ( "github.com/weaveworks/eksctl/integration/tests" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/cfn/builder" + "github.com/weaveworks/eksctl/pkg/eks" "github.com/weaveworks/eksctl/pkg/testutils" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/xgfone/netaddr" ) var params *tests.Params @@ -130,36 +136,35 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { Expect(*s.AssignIpv6AddressOnCreation).To(BeTrue()) } - // TODO: Uncomment and run the following test once CF handler bug is fixed from EKS side. - // By("the k8s cluster's having an IP family of IPv6") - // var clientSet *kubernetes.Clientset - // ctl, err := eks.New(&api.ProviderConfig{Region: params.Region}, clusterConfig) - // Expect(err).NotTo(HaveOccurred()) - // err = ctl.RefreshClusterStatus(clusterConfig) - // Expect(err).ShouldNot(HaveOccurred()) - // clientSet, err = ctl.NewStdClientSet(clusterConfig) - // Expect(err).ShouldNot(HaveOccurred()) - - // svcName := "IPv6Service" - // _, err = clientSet.CoreV1().Services("default").Create(context.TODO(), &corev1.Service{ - // ObjectMeta: metav1.ObjectMeta{ - // Name: svcName, - // }, - // Spec: corev1.ServiceSpec{ - // IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol}, - // Selector: map[string]string{"app": "IPv6App"}, - // Ports: []corev1.ServicePort{corev1.ServicePort{ - // Protocol: corev1.ProtocolTCP, - // Port: 80, - // }}, - // }, - // }, metav1.CreateOptions{}) - // Expect(err).ShouldNot(HaveOccurred()) - - // svc, err := clientSet.CoreV1().Services("default").Get(context.TODO(), svcName, metav1.GetOptions{}) - // svcIP, err := netaddr.NewIPAddress(svc.Spec.ClusterIP) - // Expect(err).NotTo(HaveOccurred()) - // Expect(svcIP.Version()).To(Equal(6)) + By("the k8s cluster's having an IP family of IPv6") + var clientSet *kubernetes.Clientset + ctl, err := eks.New(&api.ProviderConfig{Region: params.Region}, clusterConfig) + Expect(err).NotTo(HaveOccurred()) + err = ctl.RefreshClusterStatus(clusterConfig) + Expect(err).ShouldNot(HaveOccurred()) + clientSet, err = ctl.NewStdClientSet(clusterConfig) + Expect(err).ShouldNot(HaveOccurred()) + + svcName := "IPv6Service" + _, err = clientSet.CoreV1().Services("default").Create(context.TODO(), &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + }, + Spec: corev1.ServiceSpec{ + IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol}, + Selector: map[string]string{"app": "IPv6App"}, + Ports: []corev1.ServicePort{corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + Port: 80, + }}, + }, + }, metav1.CreateOptions{}) + Expect(err).ShouldNot(HaveOccurred()) + + svc, err := clientSet.CoreV1().Services("default").Get(context.TODO(), svcName, metav1.GetOptions{}) + svcIP, err := netaddr.NewIPAddress(svc.Spec.ClusterIP) + Expect(err).NotTo(HaveOccurred()) + Expect(svcIP.Version()).To(Equal(6)) }) }) }) diff --git a/pkg/cfn/builder/cluster.go b/pkg/cfn/builder/cluster.go index f1d993ad5e..52ab8eeffb 100644 --- a/pkg/cfn/builder/cluster.go +++ b/pkg/cfn/builder/cluster.go @@ -3,6 +3,7 @@ package builder import ( "encoding/base64" "fmt" + "strings" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" @@ -169,24 +170,17 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDe EncryptionConfig: encryptionConfigs, } - //TODO: delete this below If statement, and uncomment the code below once we can create ipv6 clusters - if c.spec.KubernetesNetworkConfig != nil && c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR != "" { - cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ - ServiceIpv4Cidr: gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR), - } + cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ + IpFamily: gfnt.NewString(strings.ToLower(string(api.IPV4Family))), } - // cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ - // IpFamily: gfnt.NewString(strings.ToLower(string(api.IPV4Family))), - // } - - // if c.spec.VPC.IPFamily != nil && *c.spec.VPC.IPFamily == string(api.IPV6Family) { - // cluster.KubernetesNetworkConfig.IpFamily = gfnt.NewString(strings.ToLower(string(api.IPV6Family))) - // } + if c.spec.VPC.IPFamily != nil && *c.spec.VPC.IPFamily == string(api.IPV6Family) { + cluster.KubernetesNetworkConfig.IpFamily = gfnt.NewString(strings.ToLower(string(api.IPV6Family))) + } - // if c.spec.KubernetesNetworkConfig != nil && c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR != "" { - // cluster.KubernetesNetworkConfig.ServiceIpv4Cidr = gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR) - // } + if c.spec.KubernetesNetworkConfig != nil && c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR != "" { + cluster.KubernetesNetworkConfig.ServiceIpv4Cidr = gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR) + } c.newResource("ControlPlane", &cluster) diff --git a/pkg/cfn/builder/cluster_test.go b/pkg/cfn/builder/cluster_test.go index 18d7856441..84cbf7ee6c 100644 --- a/pkg/cfn/builder/cluster_test.go +++ b/pkg/cfn/builder/cluster_test.go @@ -74,8 +74,7 @@ var _ = Describe("Cluster Template Builder", func() { Expect(clusterTemplate.Resources["ControlPlane"].Properties.RoleArn).To(ContainElement([]interface{}{"ServiceRole", "Arn"})) Expect(clusterTemplate.Resources["ControlPlane"].Properties.EncryptionConfig).To(BeNil()) Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.ServiceIPv4CIDR).To(Equal("131.10.55.70/18")) - // TODO uncomment once CF handler bug is fixed - // Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv4")) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv4")) }) It("should add vpc resources", func() { @@ -103,8 +102,7 @@ var _ = Describe("Cluster Template Builder", func() { cfg.VPC.IPFamily = aws.String(string(api.IPV6Family)) }) - // TODO: Unpend once CF handler bug is fixed on EKS side. - PIt("should add control plane resources", func() { + It("should add control plane resources", func() { Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv6")) }) From 02efa0d918ca634c26930f17983b5afccbc2afbf Mon Sep 17 00:00:00 2001 From: nikimanoledaki <18622989+nikimanoledaki@users.noreply.github.com> Date: Fri, 15 Oct 2021 17:02:51 +0200 Subject: [PATCH 7/9] Refactoring integration test to wait for Service to eventually exist --- integration/tests/ipv6/ipv6_test.go | 19 +++++++++----- pkg/cfn/manager/create_tasks_test.go | 37 +++++++++------------------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/integration/tests/ipv6/ipv6_test.go b/integration/tests/ipv6/ipv6_test.go index d71f12b16f..4897351676 100644 --- a/integration/tests/ipv6/ipv6_test.go +++ b/integration/tests/ipv6/ipv6_test.go @@ -10,6 +10,7 @@ import ( "fmt" "strings" "testing" + "time" "github.com/aws/aws-sdk-go/aws" cfn "github.com/aws/aws-sdk-go/service/cloudformation" @@ -145,14 +146,14 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { clientSet, err = ctl.NewStdClientSet(clusterConfig) Expect(err).ShouldNot(HaveOccurred()) - svcName := "IPv6Service" + svcName := "ipv6-service" _, err = clientSet.CoreV1().Services("default").Create(context.TODO(), &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: svcName, }, Spec: corev1.ServiceSpec{ IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol}, - Selector: map[string]string{"app": "IPv6App"}, + Selector: map[string]string{"app": "ipv6"}, Ports: []corev1.ServicePort{corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, @@ -161,10 +162,16 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { }, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) - svc, err := clientSet.CoreV1().Services("default").Get(context.TODO(), svcName, metav1.GetOptions{}) - svcIP, err := netaddr.NewIPAddress(svc.Spec.ClusterIP) - Expect(err).NotTo(HaveOccurred()) - Expect(svcIP.Version()).To(Equal(6)) + Eventually(func() int { + svc, err := clientSet.CoreV1().Services("default").Get(context.TODO(), svcName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + svcIP, err := netaddr.NewIPAddress(svc.Spec.ClusterIP) + if err != nil { + return 0 + } + return svcIP.Version() + }, 5*time.Second, time.Minute).Should(Equal(6)) }) }) }) diff --git a/pkg/cfn/manager/create_tasks_test.go b/pkg/cfn/manager/create_tasks_test.go index d3b70b749b..d0b9827efd 100644 --- a/pkg/cfn/manager/create_tasks_test.go +++ b/pkg/cfn/manager/create_tasks_test.go @@ -3,7 +3,6 @@ package manager_test import ( "fmt" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -29,28 +28,15 @@ var _ = Describe("CreateTasks", func() { It("sets AssignIpv6AddressOnCreation to true for all public subnets", func() { modifySubnetAttributeCallCount := 0 p := mockprovider.NewMockProvider() - mockCall1 := p.MockEC2().On("ModifySubnetAttribute", &ec2.ModifySubnetAttributeInput{ - AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ - Value: aws.Bool(true), - }, - SubnetId: aws.String(subnetIDs[0]), - }).Return(&ec2.ModifySubnetAttributeOutput{}, nil) - - mockCall1.RunFn = func(_ mock.Arguments) { + p.MockEC2().On("ModifySubnetAttribute", mock.Anything).Run(func(args mock.Arguments) { + Expect(args).To(HaveLen(1)) + Expect(args[0]).To(BeAssignableToTypeOf(&ec2.ModifySubnetAttributeInput{})) + modifySubnetAttributeInput := args[0].(*ec2.ModifySubnetAttributeInput) + Expect(*modifySubnetAttributeInput.AssignIpv6AddressOnCreation.Value).To(BeTrue()) + Expect(subnetIDs).To(ContainElement(*modifySubnetAttributeInput.SubnetId)) modifySubnetAttributeCallCount++ - } - - mockCall2 := p.MockEC2().On("ModifySubnetAttribute", &ec2.ModifySubnetAttributeInput{ - AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ - Value: aws.Bool(true), - }, - SubnetId: aws.String(subnetIDs[1]), }).Return(&ec2.ModifySubnetAttributeOutput{}, nil) - mockCall2.RunFn = func(_ mock.Arguments) { - modifySubnetAttributeCallCount++ - } - task := manager.AssignIpv6AddressOnCreationTask{ EC2API: p.EC2(), ClusterConfig: clusterConfig, @@ -67,11 +53,12 @@ var _ = Describe("CreateTasks", func() { When("the API call errors", func() { It("errors", func() { p := mockprovider.NewMockProvider() - p.MockEC2().On("ModifySubnetAttribute", &ec2.ModifySubnetAttributeInput{ - AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ - Value: aws.Bool(true), - }, - SubnetId: aws.String(subnetIDs[0]), + p.MockEC2().On("ModifySubnetAttribute", mock.Anything).Run(func(args mock.Arguments) { + Expect(args).To(HaveLen(1)) + Expect(args[0]).To(BeAssignableToTypeOf(&ec2.ModifySubnetAttributeInput{})) + modifySubnetAttributeInput := args[0].(*ec2.ModifySubnetAttributeInput) + Expect(*modifySubnetAttributeInput.AssignIpv6AddressOnCreation.Value).To(BeTrue()) + Expect(subnetIDs).To(ContainElement(*modifySubnetAttributeInput.SubnetId)) }).Return(&ec2.ModifySubnetAttributeOutput{}, fmt.Errorf("foo")) task := manager.AssignIpv6AddressOnCreationTask{ From 67f8226d6ecbe12bc4c44e8a3c80dd5d80fc45d4 Mon Sep 17 00:00:00 2001 From: nikimanoledaki <18622989+nikimanoledaki@users.noreply.github.com> Date: Mon, 18 Oct 2021 11:08:28 +0200 Subject: [PATCH 8/9] Updating goformation with ipFamily changes - Also pass unit tests and remove commented code --- go.mod | 2 +- go.sum | 2 ++ pkg/cfn/builder/vpc_ipv6.go | 12 ------------ pkg/cfn/manager/create_tasks_test.go | 9 ++++++--- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 1ccb98f7f1..397f3fb01f 100644 --- a/go.mod +++ b/go.mod @@ -295,7 +295,7 @@ require ( github.com/uudashr/gocognit v1.0.5 // indirect github.com/vektra/mockery v1.1.2 github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 // indirect - github.com/weaveworks/goformation/v4 v4.10.2-0.20211012141859-cd360fb1f843 + github.com/weaveworks/goformation/v4 v4.10.2-0.20211018090247-36559b6b4f71 github.com/weaveworks/launcher v0.0.2-0.20200715141516-1ca323f1de15 github.com/weaveworks/schemer v0.0.0-20210802122110-338b258ad2ca github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0 diff --git a/go.sum b/go.sum index 1f240763ce..33fdfd78d0 100644 --- a/go.sum +++ b/go.sum @@ -1671,6 +1671,8 @@ github.com/weaveworks/aws-sdk-go v0.0.0-20211026093156-d6e6822f58db h1:K6lacvb3q github.com/weaveworks/aws-sdk-go v0.0.0-20211026093156-d6e6822f58db/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/weaveworks/goformation/v4 v4.10.2-0.20211012141859-cd360fb1f843 h1:9v19OzMM+kFcm0r2yZeoMMAvT71H/apnNWeoMKMxUz0= github.com/weaveworks/goformation/v4 v4.10.2-0.20211012141859-cd360fb1f843/go.mod h1:x92o12+Azh6DQ4yoXT5oEuE7dhQHR5V2vy/fmZ6pO7k= +github.com/weaveworks/goformation/v4 v4.10.2-0.20211018090247-36559b6b4f71 h1:r0uEFnXNXamKxelHxLL7quo7R70JznL2WMyENyUHAZw= +github.com/weaveworks/goformation/v4 v4.10.2-0.20211018090247-36559b6b4f71/go.mod h1:x92o12+Azh6DQ4yoXT5oEuE7dhQHR5V2vy/fmZ6pO7k= github.com/weaveworks/launcher v0.0.2-0.20200715141516-1ca323f1de15 h1:i/RhLevywqC6cuUWtGdoaNrsJd+/zWh3PXbkXZIyZsU= github.com/weaveworks/launcher v0.0.2-0.20200715141516-1ca323f1de15/go.mod h1:w9Z1vnQmPobkEZ0F3oyiqRYP+62qDqTGnK6t5uhe1kg= github.com/weaveworks/mesh v0.0.0-20170419100114-1f158d31de55/go.mod h1:mcON9Ws1aW0crSErpXWp7U1ErCDEKliDX2OhVlbWRKk= diff --git a/pkg/cfn/builder/vpc_ipv6.go b/pkg/cfn/builder/vpc_ipv6.go index cdb5a691b4..3d5245cd31 100644 --- a/pkg/cfn/builder/vpc_ipv6.go +++ b/pkg/cfn/builder/vpc_ipv6.go @@ -17,26 +17,14 @@ type IPv6VPCResourceSet struct { rs *resourceSet clusterConfig *api.ClusterConfig ec2API ec2iface.EC2API - - // vpcResource *IPv6VPCResource } -// // IPv6VPCResource reresents a VPC resource -// type IPv6VPCResource struct { -// VPC *gfnt.Value -// SubnetDetails *subnetDetails -// } - // NewIPv6VPCResourceSet creates and returns a new VPCResourceSet func NewIPv6VPCResourceSet(rs *resourceSet, clusterConfig *api.ClusterConfig, ec2API ec2iface.EC2API) *IPv6VPCResourceSet { return &IPv6VPCResourceSet{ rs: rs, clusterConfig: clusterConfig, ec2API: ec2API, - // vpcResource: &IPv6VPCResource{ - // VPC: vpcRef, - // SubnetDetails: &subnetDetails{}, - // }, } } diff --git a/pkg/cfn/manager/create_tasks_test.go b/pkg/cfn/manager/create_tasks_test.go index d0b9827efd..7a4886b30f 100644 --- a/pkg/cfn/manager/create_tasks_test.go +++ b/pkg/cfn/manager/create_tasks_test.go @@ -19,13 +19,13 @@ var _ = Describe("CreateTasks", func() { BeforeEach(func() { clusterConfig = api.NewClusterConfig() clusterConfig.VPC.Subnets = &api.ClusterSubnets{} + }) + + It("sets AssignIpv6AddressOnCreation to true for all public subnets", func() { clusterConfig.VPC.Subnets.Public = map[string]api.AZSubnetSpec{ "0": {ID: subnetIDs[0]}, "1": {ID: subnetIDs[1]}, } - }) - - It("sets AssignIpv6AddressOnCreation to true for all public subnets", func() { modifySubnetAttributeCallCount := 0 p := mockprovider.NewMockProvider() p.MockEC2().On("ModifySubnetAttribute", mock.Anything).Run(func(args mock.Arguments) { @@ -52,6 +52,9 @@ var _ = Describe("CreateTasks", func() { When("the API call errors", func() { It("errors", func() { + clusterConfig.VPC.Subnets.Public = map[string]api.AZSubnetSpec{ + "0": {ID: subnetIDs[0]}, + } p := mockprovider.NewMockProvider() p.MockEC2().On("ModifySubnetAttribute", mock.Anything).Run(func(args mock.Arguments) { Expect(args).To(HaveLen(1)) From 3237ad2c49ba03db3d252e406d9d6fcdbfaad915 Mon Sep 17 00:00:00 2001 From: nikimanoledaki <18622989+nikimanoledaki@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:12:42 +0200 Subject: [PATCH 9/9] Implement review feedback Update integration/tests/ipv6/ipv6_test.go Co-authored-by: Chetan Patwal set example ipv6 region to us-west-2 --- examples/29-vpc-with-ip-family.yaml | 2 +- go.mod | 3 +-- integration/tests/ipv6/ipv6_test.go | 13 ++++++------- pkg/apis/eksctl.io/v1alpha5/types.go | 3 +++ pkg/cfn/builder/cluster.go | 8 ++++---- pkg/cfn/builder/cluster_test.go | 1 + pkg/cfn/builder/vpc.go | 4 +++- pkg/cfn/builder/vpc_ipv4.go | 1 - pkg/cfn/manager/create_tasks.go | 3 ++- pkg/cfn/manager/tasks.go | 20 +++++++++++--------- pkg/ctl/cmdutils/configfile.go | 5 +++-- 11 files changed, 35 insertions(+), 28 deletions(-) diff --git a/examples/29-vpc-with-ip-family.yaml b/examples/29-vpc-with-ip-family.yaml index b1824ffa25..a52f4a8710 100644 --- a/examples/29-vpc-with-ip-family.yaml +++ b/examples/29-vpc-with-ip-family.yaml @@ -6,7 +6,7 @@ kind: ClusterConfig metadata: name: cluster-2 - region: eu-north-1 + region: us-west-2 version: "1.21" vpc: diff --git a/go.mod b/go.mod index 397f3fb01f..0744b4680c 100644 --- a/go.mod +++ b/go.mod @@ -304,6 +304,7 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56 // indirect + github.com/xgfone/netaddr v0.5.1 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect github.com/yeya24/promlinter v0.1.0 // indirect @@ -372,8 +373,6 @@ require ( sigs.k8s.io/yaml v1.2.0 ) -require github.com/xgfone/netaddr v0.5.1 - require ( github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cncf/udpa/go v0.0.0-20210322005330-6414d713912e // indirect diff --git a/integration/tests/ipv6/ipv6_test.go b/integration/tests/ipv6/ipv6_test.go index 4897351676..847cdefd30 100644 --- a/integration/tests/ipv6/ipv6_test.go +++ b/integration/tests/ipv6/ipv6_test.go @@ -22,13 +22,13 @@ import ( "github.com/weaveworks/eksctl/pkg/cfn/builder" "github.com/weaveworks/eksctl/pkg/eks" "github.com/weaveworks/eksctl/pkg/testutils" + "github.com/xgfone/netaddr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/xgfone/netaddr" ) var params *tests.Params @@ -61,13 +61,13 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { clusterConfig.IAM.WithOIDC = api.Enabled() clusterConfig.Addons = []*api.Addon{ { - Name: "vpc-cni", + Name: api.VPCCNIAddon, }, { - Name: "kube-proxy", + Name: api.KubeProxyAddon, }, { - Name: "coredns", + Name: api.CoreDNSAddon, }, } @@ -118,7 +118,7 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { Expect(err).NotTo(HaveOccurred(), vpcOutput.GoString()) Expect(vpcOutput.Vpcs[0].Ipv6CidrBlockAssociationSet).To(HaveLen(1)) - // TODO: get rid of this once CF bug is fixed + // TODO: get rid of this once CF bug is fixed https://github.com/weaveworks/eksctl/issues/4363 By("setting AssignIpv6AddressOnCreation to true for each public subnet") var publicSubnets string for _, output := range describeStackOut.Stacks[0].Outputs { @@ -133,11 +133,10 @@ var _ = Describe("(Integration) [EKS IPv6 test]", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(subnetsOutput.Subnets)).To(BeNumerically(">", 0)) for _, s := range subnetsOutput.Subnets { - Expect(s.AssignIpv6AddressOnCreation).NotTo(BeNil()) Expect(*s.AssignIpv6AddressOnCreation).To(BeTrue()) } - By("the k8s cluster's having an IP family of IPv6") + By("ensuring the K8s cluster has IPv6 enabled") var clientSet *kubernetes.Clientset ctl, err := eks.New(&api.ProviderConfig{Region: params.Region}, clusterConfig) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/apis/eksctl.io/v1alpha5/types.go b/pkg/apis/eksctl.io/v1alpha5/types.go index 5292579cba..b130aebd37 100644 --- a/pkg/apis/eksctl.io/v1alpha5/types.go +++ b/pkg/apis/eksctl.io/v1alpha5/types.go @@ -380,6 +380,9 @@ func IsDisabled(v *bool) bool { return v != nil && !*v } // IsSetAndNonEmptyString will only return true if s is not nil and not empty func IsSetAndNonEmptyString(s *string) bool { return s != nil && *s != "" } +// IsSetAndNonEmptyString will only return true if s is not nil and not empty +func IsEmpty(s *string) bool { return !IsSetAndNonEmptyString(s) } + // SupportedRegions are the regions where EKS is available func SupportedRegions() []string { return []string{ diff --git a/pkg/cfn/builder/cluster.go b/pkg/cfn/builder/cluster.go index 52ab8eeffb..ebc256015b 100644 --- a/pkg/cfn/builder/cluster.go +++ b/pkg/cfn/builder/cluster.go @@ -16,6 +16,7 @@ import ( api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/cfn/outputs" + utilsstrings "github.com/weaveworks/eksctl/pkg/utils/strings" ) // ClusterResourceSet stores the resource information of the cluster @@ -36,9 +37,8 @@ func NewClusterResourceSet(ec2API ec2iface.EC2API, region string, spec *api.Clus } rs := newResourceSet() - var vpcResourceSet VPCResourceSet - vpcResourceSet = NewIPv4VPCResourceSet(rs, spec, ec2API) - if spec.VPC.IPFamily != nil && *spec.VPC.IPFamily == string(api.IPV6Family) { + var vpcResourceSet VPCResourceSet = NewIPv4VPCResourceSet(rs, spec, ec2API) + if utilsstrings.Value(spec.VPC.IPFamily) == string(api.IPV6Family) { vpcResourceSet = NewIPv6VPCResourceSet(rs, spec, ec2API) } return &ClusterResourceSet{ @@ -174,7 +174,7 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDe IpFamily: gfnt.NewString(strings.ToLower(string(api.IPV4Family))), } - if c.spec.VPC.IPFamily != nil && *c.spec.VPC.IPFamily == string(api.IPV6Family) { + if utilsstrings.Value(c.spec.VPC.IPFamily) == string(api.IPV6Family) { cluster.KubernetesNetworkConfig.IpFamily = gfnt.NewString(strings.ToLower(string(api.IPV6Family))) } diff --git a/pkg/cfn/builder/cluster_test.go b/pkg/cfn/builder/cluster_test.go index 84cbf7ee6c..60ec7f98cc 100644 --- a/pkg/cfn/builder/cluster_test.go +++ b/pkg/cfn/builder/cluster_test.go @@ -100,6 +100,7 @@ var _ = Describe("Cluster Template Builder", func() { Context("when ipFamily is set to IPv6", func() { BeforeEach(func() { cfg.VPC.IPFamily = aws.String(string(api.IPV6Family)) + cfg.KubernetesNetworkConfig = nil }) It("should add control plane resources", func() { diff --git a/pkg/cfn/builder/vpc.go b/pkg/cfn/builder/vpc.go index 51b76dbfdb..01e062eee8 100644 --- a/pkg/cfn/builder/vpc.go +++ b/pkg/cfn/builder/vpc.go @@ -38,8 +38,10 @@ const ( PrivateSubnetsOutputKey = "SubnetsPrivate" ) +//VPCResourceSet interface for creating cloudformation resource sets for generating VPC resources type VPCResourceSet interface { - CreateTemplate() (*gfnt.Value, *SubnetDetails, error) + //CreateTemplate generates all of the resources & outputs required for the VPC. Returns the + CreateTemplate() (vpcID *gfnt.Value, subnetDetails *SubnetDetails, err error) } func formatAZ(az string) string { diff --git a/pkg/cfn/builder/vpc_ipv4.go b/pkg/cfn/builder/vpc_ipv4.go index 801edc5e34..6a3dbb77d1 100644 --- a/pkg/cfn/builder/vpc_ipv4.go +++ b/pkg/cfn/builder/vpc_ipv4.go @@ -373,7 +373,6 @@ type clusterSecurityGroup struct { ClusterSharedNode *gfnt.Value } -// TODO move this func (c *ClusterResourceSet) addResourcesForSecurityGroups(vpcID *gfnt.Value) *clusterSecurityGroup { var refControlPlaneSG, refClusterSharedNodeSG *gfnt.Value diff --git a/pkg/cfn/manager/create_tasks.go b/pkg/cfn/manager/create_tasks.go index 72c2dfb661..36eb6fa411 100644 --- a/pkg/cfn/manager/create_tasks.go +++ b/pkg/cfn/manager/create_tasks.go @@ -8,6 +8,7 @@ import ( api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" iamoidc "github.com/weaveworks/eksctl/pkg/iam/oidc" "github.com/weaveworks/eksctl/pkg/kubernetes" + utilsstrings "github.com/weaveworks/eksctl/pkg/utils/strings" "github.com/weaveworks/eksctl/pkg/utils/tasks" "github.com/weaveworks/eksctl/pkg/vpc" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,7 +34,7 @@ func (c *StackCollection) NewTasksToCreateClusterWithNodeGroups(nodeGroups []*ap }, ) - if c.spec.VPC.IPFamily != nil && *c.spec.VPC.IPFamily == string(api.IPV6Family) { + if utilsstrings.Value(c.spec.VPC.IPFamily) == string(api.IPV6Family) { taskTree.Append( &AssignIpv6AddressOnCreationTask{ ClusterConfig: c.spec, diff --git a/pkg/cfn/manager/tasks.go b/pkg/cfn/manager/tasks.go index e0ca39cf75..11da02282a 100644 --- a/pkg/cfn/manager/tasks.go +++ b/pkg/cfn/manager/tasks.go @@ -146,15 +146,17 @@ func (t *AssignIpv6AddressOnCreationTask) Describe() string { func (t *AssignIpv6AddressOnCreationTask) Do(errs chan error) error { defer close(errs) - for _, subnet := range t.ClusterConfig.VPC.Subnets.Public.WithIDs() { - _, err := t.EC2API.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ - AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ - Value: aws.Bool(true), - }, - SubnetId: aws.String(subnet), - }) - if err != nil { - return fmt.Errorf("failed to update subnet %q: %v", subnet, err) + if t.ClusterConfig.VPC.Subnets.Public != nil { + for _, subnet := range t.ClusterConfig.VPC.Subnets.Public.WithIDs() { + _, err := t.EC2API.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String(subnet), + }) + if err != nil { + return fmt.Errorf("failed to update subnet %q: %v", subnet, err) + } } } return nil diff --git a/pkg/ctl/cmdutils/configfile.go b/pkg/ctl/cmdutils/configfile.go index 854b8c158f..afe3ae4e93 100644 --- a/pkg/ctl/cmdutils/configfile.go +++ b/pkg/ctl/cmdutils/configfile.go @@ -17,6 +17,7 @@ import ( "github.com/weaveworks/eksctl/pkg/ctl/cmdutils/filter" "github.com/weaveworks/eksctl/pkg/eks" "github.com/weaveworks/eksctl/pkg/utils/names" + utilsstrings "github.com/weaveworks/eksctl/pkg/utils/strings" ) // AddConfigFileFlag adds common --config-file flag @@ -238,12 +239,12 @@ func NewCreateClusterLoader(cmd *Cmd, ngFilter *filter.NodeGroupFilter, ng *api. } if clusterConfig.VPC.NAT == nil { - if clusterConfig.VPC.IPFamily == nil || *clusterConfig.VPC.IPFamily != string(api.IPV6Family) { + if utilsstrings.Value(clusterConfig.VPC.IPFamily) != string(api.IPV6Family) { clusterConfig.VPC.NAT = api.DefaultClusterNAT() } } - if clusterConfig.VPC.NAT != nil && !api.IsSetAndNonEmptyString(clusterConfig.VPC.NAT.Gateway) { + if clusterConfig.VPC.NAT != nil && api.IsEmpty(clusterConfig.VPC.NAT.Gateway) { *clusterConfig.VPC.NAT.Gateway = api.ClusterSingleNAT }