diff --git a/examples/29-vpc-with-ip-family.yaml b/examples/29-vpc-with-ip-family.yaml index b1824ffa25..a52f4a8710 100644 --- a/examples/29-vpc-with-ip-family.yaml +++ b/examples/29-vpc-with-ip-family.yaml @@ -6,7 +6,7 @@ kind: ClusterConfig metadata: name: cluster-2 - region: eu-north-1 + region: us-west-2 version: "1.21" vpc: diff --git a/go.mod b/go.mod index 1a6c73e5a3..0744b4680c 100644 --- a/go.mod +++ b/go.mod @@ -295,7 +295,7 @@ require ( github.com/uudashr/gocognit v1.0.5 // indirect github.com/vektra/mockery v1.1.2 github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 // indirect - github.com/weaveworks/goformation/v4 v4.10.2-0.20210609082249-532b27315cf1 + github.com/weaveworks/goformation/v4 v4.10.2-0.20211018090247-36559b6b4f71 github.com/weaveworks/launcher v0.0.2-0.20200715141516-1ca323f1de15 github.com/weaveworks/schemer v0.0.0-20210802122110-338b258ad2ca github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0 @@ -304,6 +304,7 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56 // indirect + github.com/xgfone/netaddr v0.5.1 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect github.com/yeya24/promlinter v0.1.0 // indirect diff --git a/go.sum b/go.sum index 7bdadb34d6..33fdfd78d0 100644 --- a/go.sum +++ b/go.sum @@ -1669,8 +1669,10 @@ github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 h1:txplJASvd6b github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2/go.mod h1:DGCIhurYgnLz8J9ga1fMV/fbLDyUvTyrWXVWUIyJon4= github.com/weaveworks/aws-sdk-go v0.0.0-20211026093156-d6e6822f58db h1:K6lacvb3qzF/bHvx2RsPDw8cYA8VccOecn9e6xDEBY0= github.com/weaveworks/aws-sdk-go v0.0.0-20211026093156-d6e6822f58db/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/weaveworks/goformation/v4 v4.10.2-0.20210609082249-532b27315cf1 h1:yX/flUgj/znRvfhEZ3mC8RdKV+GNwq/9j+FkWN7ve+o= -github.com/weaveworks/goformation/v4 v4.10.2-0.20210609082249-532b27315cf1/go.mod h1:x92o12+Azh6DQ4yoXT5oEuE7dhQHR5V2vy/fmZ6pO7k= +github.com/weaveworks/goformation/v4 v4.10.2-0.20211012141859-cd360fb1f843 h1:9v19OzMM+kFcm0r2yZeoMMAvT71H/apnNWeoMKMxUz0= +github.com/weaveworks/goformation/v4 v4.10.2-0.20211012141859-cd360fb1f843/go.mod h1:x92o12+Azh6DQ4yoXT5oEuE7dhQHR5V2vy/fmZ6pO7k= +github.com/weaveworks/goformation/v4 v4.10.2-0.20211018090247-36559b6b4f71 h1:r0uEFnXNXamKxelHxLL7quo7R70JznL2WMyENyUHAZw= +github.com/weaveworks/goformation/v4 v4.10.2-0.20211018090247-36559b6b4f71/go.mod h1:x92o12+Azh6DQ4yoXT5oEuE7dhQHR5V2vy/fmZ6pO7k= github.com/weaveworks/launcher v0.0.2-0.20200715141516-1ca323f1de15 h1:i/RhLevywqC6cuUWtGdoaNrsJd+/zWh3PXbkXZIyZsU= github.com/weaveworks/launcher v0.0.2-0.20200715141516-1ca323f1de15/go.mod h1:w9Z1vnQmPobkEZ0F3oyiqRYP+62qDqTGnK6t5uhe1kg= github.com/weaveworks/mesh v0.0.0-20170419100114-1f158d31de55/go.mod h1:mcON9Ws1aW0crSErpXWp7U1ErCDEKliDX2OhVlbWRKk= @@ -1695,6 +1697,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180816142147-da425ebb7609/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56 h1:yhqBHs09SmmUoNOHc9jgK4a60T3XFRtPAkYxVnqgY50= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xgfone/netaddr v0.5.1 h1:87DhCyyR6XUr0p63JHTDT5juGDhH49Ak2ePZNBmSL5I= +github.com/xgfone/netaddr v0.5.1/go.mod h1:QDEYI/4nQfAtNj7TB4RhYQY1B4U31Edj+SOoDEuIfsQ= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= diff --git a/integration/tests/ipv6/ipv6_test.go b/integration/tests/ipv6/ipv6_test.go new file mode 100644 index 0000000000..847cdefd30 --- /dev/null +++ b/integration/tests/ipv6/ipv6_test.go @@ -0,0 +1,176 @@ +//go:build integration +// +build integration + +package ipv6 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + cfn "github.com/aws/aws-sdk-go/service/cloudformation" + awsec2 "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/weaveworks/eksctl/integration/matchers" + . "github.com/weaveworks/eksctl/integration/runner" + "github.com/weaveworks/eksctl/integration/tests" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/builder" + "github.com/weaveworks/eksctl/pkg/eks" + "github.com/weaveworks/eksctl/pkg/testutils" + "github.com/xgfone/netaddr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var params *tests.Params + +func init() { + // Call testing.Init() prior to tests.NewParams(), as otherwise -test.* will not be recognised. See also: https://golang.org/doc/go1.13#testing + testing.Init() + params = tests.NewParams("IPv6") +} + +func TestIPv6(t *testing.T) { + testutils.RegisterAndRun(t) +} + +var _ = Describe("(Integration) [EKS IPv6 test]", func() { + var ( + clusterConfig *api.ClusterConfig + ) + + Context("Creating a cluster with IPv6", func() { + clusterName := params.NewClusterName("ipv6") + + BeforeSuite(func() { + clusterConfig = api.NewClusterConfig() + clusterConfig.Metadata.Name = clusterName + clusterConfig.Metadata.Version = "1.21" + clusterConfig.Metadata.Region = params.Region + clusterConfig.VPC.IPFamily = aws.String("IPv6") + clusterConfig.VPC.NAT = nil + clusterConfig.IAM.WithOIDC = api.Enabled() + clusterConfig.Addons = []*api.Addon{ + { + Name: api.VPCCNIAddon, + }, + { + Name: api.KubeProxyAddon, + }, + { + Name: api.CoreDNSAddon, + }, + } + + data, err := json.Marshal(clusterConfig) + Expect(err).ToNot(HaveOccurred()) + + cmd := params.EksctlCreateCmd. + WithArgs( + "cluster", + "--config-file", "-", + "--verbose", "4", + ). + WithoutArg("--region", params.Region). + WithStdin(bytes.NewReader(data)) + Expect(cmd).To(RunSuccessfully()) + }) + + AfterSuite(func() { + cmd := params.EksctlDeleteCmd.WithArgs( + "cluster", clusterName, + "--verbose", "2", + ) + Expect(cmd).To(RunSuccessfully()) + }) + + It("should support ipv6", func() { + By("creating a VPC that has an IPv6 CIDR") + awsSession := NewSession(params.Region) + cfnSession := cfn.New(awsSession) + + var describeStackOut *cfn.DescribeStacksOutput + describeStackOut, err := cfnSession.DescribeStacks(&cfn.DescribeStacksInput{ + StackName: aws.String(fmt.Sprintf("eksctl-%s-cluster", clusterName)), + }) + Expect(err).NotTo(HaveOccurred()) + + var vpcID string + for _, output := range describeStackOut.Stacks[0].Outputs { + if *output.OutputKey == builder.VPCResourceKey { + vpcID = *output.OutputValue + } + } + + ec2 := awsec2.New(awsSession) + vpcOutput, err := ec2.DescribeVpcs(&awsec2.DescribeVpcsInput{ + VpcIds: aws.StringSlice([]string{vpcID}), + }) + Expect(err).NotTo(HaveOccurred(), vpcOutput.GoString()) + Expect(vpcOutput.Vpcs[0].Ipv6CidrBlockAssociationSet).To(HaveLen(1)) + + // TODO: get rid of this once CF bug is fixed https://github.com/weaveworks/eksctl/issues/4363 + By("setting AssignIpv6AddressOnCreation to true for each public subnet") + var publicSubnets string + for _, output := range describeStackOut.Stacks[0].Outputs { + if *output.OutputKey == builder.PublicSubnetsOutputKey { + publicSubnets = *output.OutputValue + } + } + + subnetsOutput, err := ec2.DescribeSubnets(&awsec2.DescribeSubnetsInput{ + SubnetIds: aws.StringSlice(strings.Split(publicSubnets, ",")), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(len(subnetsOutput.Subnets)).To(BeNumerically(">", 0)) + for _, s := range subnetsOutput.Subnets { + Expect(*s.AssignIpv6AddressOnCreation).To(BeTrue()) + } + + By("ensuring the K8s cluster has IPv6 enabled") + var clientSet *kubernetes.Clientset + ctl, err := eks.New(&api.ProviderConfig{Region: params.Region}, clusterConfig) + Expect(err).NotTo(HaveOccurred()) + err = ctl.RefreshClusterStatus(clusterConfig) + Expect(err).ShouldNot(HaveOccurred()) + clientSet, err = ctl.NewStdClientSet(clusterConfig) + Expect(err).ShouldNot(HaveOccurred()) + + svcName := "ipv6-service" + _, err = clientSet.CoreV1().Services("default").Create(context.TODO(), &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + }, + Spec: corev1.ServiceSpec{ + IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol}, + Selector: map[string]string{"app": "ipv6"}, + Ports: []corev1.ServicePort{corev1.ServicePort{ + Protocol: corev1.ProtocolTCP, + Port: 80, + }}, + }, + }, metav1.CreateOptions{}) + Expect(err).ShouldNot(HaveOccurred()) + + Eventually(func() int { + svc, err := clientSet.CoreV1().Services("default").Get(context.TODO(), svcName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + svcIP, err := netaddr.NewIPAddress(svc.Spec.ClusterIP) + if err != nil { + return 0 + } + return svcIP.Version() + }, 5*time.Second, time.Minute).Should(Equal(6)) + }) + }) +}) diff --git a/pkg/actions/addon/tasks.go b/pkg/actions/addon/tasks.go index e8ce296128..49bb5f3a12 100644 --- a/pkg/actions/addon/tasks.go +++ b/pkg/actions/addon/tasks.go @@ -31,6 +31,7 @@ func CreateAddonTasks(cfg *api.ClusterConfig, clusterProvider *eks.ClusterProvid clusterProvider: clusterProvider, forceAll: forceAll, timeout: timeout, + wait: false, }, ) @@ -42,6 +43,7 @@ func CreateAddonTasks(cfg *api.ClusterConfig, clusterProvider *eks.ClusterProvid clusterProvider: clusterProvider, forceAll: forceAll, timeout: timeout, + wait: len(cfg.NodeGroups) > 0 || len(cfg.ManagedNodeGroups) > 0, }, ) return preTasks, postTasks @@ -52,7 +54,7 @@ type createAddonTask struct { cfg *api.ClusterConfig clusterProvider *eks.ClusterProvider addons []*api.Addon - forceAll bool + forceAll, wait bool timeout time.Duration } @@ -89,7 +91,7 @@ func (t *createAddonTask) Do(errorCh chan error) error { if t.forceAll { a.Force = true } - err := addonManager.Create(a, true) + err := addonManager.Create(a, t.wait) if err != nil { go func() { errorCh <- err diff --git a/pkg/apis/eksctl.io/v1alpha5/types.go b/pkg/apis/eksctl.io/v1alpha5/types.go index 5292579cba..b130aebd37 100644 --- a/pkg/apis/eksctl.io/v1alpha5/types.go +++ b/pkg/apis/eksctl.io/v1alpha5/types.go @@ -380,6 +380,9 @@ func IsDisabled(v *bool) bool { return v != nil && !*v } // IsSetAndNonEmptyString will only return true if s is not nil and not empty func IsSetAndNonEmptyString(s *string) bool { return s != nil && *s != "" } +// IsSetAndNonEmptyString will only return true if s is not nil and not empty +func IsEmpty(s *string) bool { return !IsSetAndNonEmptyString(s) } + // SupportedRegions are the regions where EKS is available func SupportedRegions() []string { return []string{ diff --git a/pkg/cfn/builder/builder_suite_test.go b/pkg/cfn/builder/builder_suite_test.go index 880cd90014..ad3860e933 100644 --- a/pkg/cfn/builder/builder_suite_test.go +++ b/pkg/cfn/builder/builder_suite_test.go @@ -17,16 +17,17 @@ func TestCfnBuilder(t *testing.T) { } var ( - azA, azB = "us-west-2a", "us-west-2b" - privateSubnet1, privateSubnet2 = "subnet-0ade11bad78dced9f", "subnet-0f98135715dfcf55a" - publicSubnet1, publicSubnet2 = "subnet-0ade11bad78dced9e", "subnet-0f98135715dfcf55f" - privateSubnetRef1, privateSubnetRef2 = "SubnetPrivateUSWEST2A", "SubnetPrivateUSWEST2B" - publicSubnetRef1, publicSubnetRef2 = "SubnetPublicUSWEST2A", "SubnetPublicUSWEST2B" - vpcResourceKey, igwKey, gaKey = "VPC", "InternetGateway", "VPCGatewayAttachment" - pubRouteTable, pubSubnetRoute = "PublicRouteTable", "PublicSubnetRoute" - privRouteTableA, privRouteTableB = "PrivateRouteTableUSWEST2A", "PrivateRouteTableUSWEST2B" - rtaPublicA, rtaPublicB = "RouteTableAssociationPublicUSWEST2A", "RouteTableAssociationPublicUSWEST2B" - rtaPrivateA, rtaPrivateB = "RouteTableAssociationPrivateUSWEST2A", "RouteTableAssociationPrivateUSWEST2B" + azA, azB, azC = "us-west-2a", "us-west-2b", "us-west-2c" + azAFormatted, azBFormatted, azCFormatted = "USWEST2A", "USWEST2B", "USWEST2C" + privateSubnet1, privateSubnet2 = "subnet-0ade11bad78dced9f", "subnet-0f98135715dfcf55a" + publicSubnet1, publicSubnet2 = "subnet-0ade11bad78dced9e", "subnet-0f98135715dfcf55f" + privateSubnetRef1, privateSubnetRef2 = "SubnetPrivateUSWEST2A", "SubnetPrivateUSWEST2B" + publicSubnetRef1, publicSubnetRef2 = "SubnetPublicUSWEST2A", "SubnetPublicUSWEST2B" + vpcResourceKey, igwKey, gaKey = "VPC", "InternetGateway", "VPCGatewayAttachment" + pubRouteTable, pubSubnetRoute = "PublicRouteTable", "PublicSubnetRoute" + privRouteTableA, privRouteTableB = "PrivateRouteTableUSWEST2A", "PrivateRouteTableUSWEST2B" + rtaPublicA, rtaPublicB = "RouteTableAssociationPublicUSWEST2A", "RouteTableAssociationPublicUSWEST2B" + rtaPrivateA, rtaPrivateB = "RouteTableAssociationPrivateUSWEST2A", "RouteTableAssociationPrivateUSWEST2B" ) func vpcConfig() *api.ClusterVPC { diff --git a/pkg/cfn/builder/cluster.go b/pkg/cfn/builder/cluster.go index e8b9a4d045..ebc256015b 100644 --- a/pkg/cfn/builder/cluster.go +++ b/pkg/cfn/builder/cluster.go @@ -3,6 +3,7 @@ package builder import ( "encoding/base64" "fmt" + "strings" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" @@ -15,6 +16,7 @@ import ( api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" "github.com/weaveworks/eksctl/pkg/cfn/outputs" + utilsstrings "github.com/weaveworks/eksctl/pkg/utils/strings" ) // ClusterResourceSet stores the resource information of the cluster @@ -24,7 +26,7 @@ type ClusterResourceSet struct { ec2API ec2iface.EC2API region string supportsManagedNodes bool - vpcResourceSet *VPCResourceSet + vpcResourceSet VPCResourceSet securityGroups []*gfnt.Value } @@ -34,13 +36,18 @@ func NewClusterResourceSet(ec2API ec2iface.EC2API, region string, spec *api.Clus unsetExistingResources(existingStack, spec) } rs := newResourceSet() + + var vpcResourceSet VPCResourceSet = NewIPv4VPCResourceSet(rs, spec, ec2API) + if utilsstrings.Value(spec.VPC.IPFamily) == string(api.IPV6Family) { + vpcResourceSet = NewIPv6VPCResourceSet(rs, spec, ec2API) + } return &ClusterResourceSet{ rs: rs, spec: spec, ec2API: ec2API, region: region, supportsManagedNodes: supportsManagedNodes, - vpcResourceSet: NewVPCResourceSet(rs, spec, ec2API), + vpcResourceSet: vpcResourceSet, } } @@ -50,16 +57,15 @@ func (c *ClusterResourceSet) AddAllResources() error { return err } - vpcResource, err := c.vpcResourceSet.AddResources() + vpcID, subnetDetails, err := c.vpcResourceSet.CreateTemplate() if err != nil { return errors.Wrap(err, "error adding VPC resources") } - c.vpcResourceSet.AddOutputs() - clusterSG := c.addResourcesForSecurityGroups(vpcResource) + clusterSG := c.addResourcesForSecurityGroups(vpcID) if privateCluster := c.spec.PrivateCluster; privateCluster.Enabled { - vpcEndpointResourceSet := NewVPCEndpointResourceSet(c.ec2API, c.region, c.rs, c.spec, vpcResource.VPC, vpcResource.SubnetDetails.Private, clusterSG.ClusterSharedNode) + vpcEndpointResourceSet := NewVPCEndpointResourceSet(c.ec2API, c.region, c.rs, c.spec, vpcID, subnetDetails.Private, clusterSG.ClusterSharedNode) if err := vpcEndpointResourceSet.AddResources(); err != nil { return errors.Wrap(err, "error adding resources for VPC endpoints") @@ -67,7 +73,7 @@ func (c *ClusterResourceSet) AddAllResources() error { } c.addResourcesForIAM() - c.addResourcesForControlPlane(vpcResource.SubnetDetails) + c.addResourcesForControlPlane(subnetDetails) if len(c.spec.FargateProfiles) > 0 { c.addResourcesForFargate() @@ -132,7 +138,7 @@ func (c *ClusterResourceSet) newResource(name string, resource gfn.Resource) *gf return c.rs.newResource(name, resource) } -func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *subnetDetails) { +func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDetails) { clusterVPC := &gfneks.Cluster_ResourcesVpcConfig{ SecurityGroupIds: gfnt.NewSlice(c.securityGroups...), } @@ -163,10 +169,17 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *subnetDe ResourcesVpcConfig: clusterVPC, EncryptionConfig: encryptionConfigs, } + + cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ + IpFamily: gfnt.NewString(strings.ToLower(string(api.IPV4Family))), + } + + if utilsstrings.Value(c.spec.VPC.IPFamily) == string(api.IPV6Family) { + cluster.KubernetesNetworkConfig.IpFamily = gfnt.NewString(strings.ToLower(string(api.IPV6Family))) + } + if c.spec.KubernetesNetworkConfig != nil && c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR != "" { - cluster.KubernetesNetworkConfig = &gfneks.Cluster_KubernetesNetworkConfig{ - ServiceIpv4Cidr: gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR), - } + cluster.KubernetesNetworkConfig.ServiceIpv4Cidr = gfnt.NewString(c.spec.KubernetesNetworkConfig.ServiceIPv4CIDR) } c.newResource("ControlPlane", &cluster) diff --git a/pkg/cfn/builder/cluster_test.go b/pkg/cfn/builder/cluster_test.go index 51e25faa7a..60ec7f98cc 100644 --- a/pkg/cfn/builder/cluster_test.go +++ b/pkg/cfn/builder/cluster_test.go @@ -3,6 +3,7 @@ package builder_test import ( "encoding/json" + "github.com/aws/aws-sdk-go/aws" cfn "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo" @@ -32,6 +33,10 @@ var _ = Describe("Cluster Template Builder", func() { cfg = api.NewClusterConfig() cfg.VPC = vpcConfig() cfg.AvailabilityZones = []string{"us-west-2a", "us-west-2b"} + cfg.VPC.IPFamily = aws.String(string(api.IPV4Family)) + cfg.KubernetesNetworkConfig = &api.KubernetesNetworkConfig{ + ServiceIPv4CIDR: "131.10.55.70/18", + } }) JustBeforeEach(func() { @@ -60,6 +65,18 @@ var _ = Describe("Cluster Template Builder", func() { Expect(clusterTemplate.Description).To(Equal("EKS cluster (dedicated VPC: true, dedicated IAM: true) [created and managed by eksctl]")) }) + It("should add control plane resources", func() { + Expect(clusterTemplate.Resources).To(HaveKey("ControlPlane")) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.Name).To(Equal(cfg.Metadata.Name)) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.Version).To(Equal(cfg.Metadata.Version)) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SecurityGroupIds[0]).To(ContainElement("ControlPlaneSecurityGroup")) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SubnetIds).To(HaveLen(4)) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.RoleArn).To(ContainElement([]interface{}{"ServiceRole", "Arn"})) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.EncryptionConfig).To(BeNil()) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.ServiceIPv4CIDR).To(Equal("131.10.55.70/18")) + Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv4")) + }) + It("should add vpc resources", func() { Expect(clusterTemplate.Resources).To(HaveKey(vpcResourceKey)) Expect(clusterTemplate.Resources).To(HaveKey(igwKey)) @@ -80,6 +97,50 @@ var _ = Describe("Cluster Template Builder", func() { Expect(clusterTemplate.Resources).To(HaveKey(privateSubnetRef1)) }) + Context("when ipFamily is set to IPv6", func() { + BeforeEach(func() { + cfg.VPC.IPFamily = aws.String(string(api.IPV6Family)) + cfg.KubernetesNetworkConfig = nil + }) + + It("should add control plane resources", func() { + Expect(clusterTemplate.Resources["ControlPlane"].Properties.KubernetesNetworkConfig.IPFamily).To(Equal("ipv6")) + }) + + It("should add IPv6 vpc resources", func() { + Expect(clusterTemplate.Resources).To(HaveKey(builder.VPCResourceKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.IPv6CIDRBlockKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.IGWKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.GAKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.EgressOnlyInternetGatewayKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.NATGatewayKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.ElasticIPKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubRouteTableKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubSubRouteKey)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubSubIPv6RouteKey)) + privateRouteTableA := builder.PrivateRouteTableKey + azAFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteTableA)) + privateRouteTableB := builder.PrivateRouteTableKey + azBFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteTableB)) + privateRouteA := builder.PrivateSubnetRouteKey + azAFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteA)) + privateRouteB := builder.PrivateSubnetRouteKey + azBFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteB)) + privateRouteA = builder.PrivateSubnetIpv6RouteKey + azAFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteA)) + privateRouteB = builder.PrivateSubnetIpv6RouteKey + azBFormatted + Expect(clusterTemplate.Resources).To(HaveKey(privateRouteB)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PublicSubnetKey + azAFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PublicSubnetKey + azBFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateSubnetKey + azAFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateSubnetKey + azBFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubRouteTableAssociation + azAFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PubRouteTableAssociation + azBFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateRouteTableAssociation + azAFormatted)) + Expect(clusterTemplate.Resources).To(HaveKey(builder.PrivateRouteTableAssociation + azBFormatted)) + }) + }) + Context("when AutoAllocateIPv6 is enabled", func() { BeforeEach(func() { autoAllocated := true @@ -238,16 +299,6 @@ var _ = Describe("Cluster Template Builder", func() { }) }) - It("should add control plane resources", func() { - Expect(clusterTemplate.Resources).To(HaveKey("ControlPlane")) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.Name).To(Equal(cfg.Metadata.Name)) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.Version).To(Equal(cfg.Metadata.Version)) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SecurityGroupIds[0]).To(ContainElement("ControlPlaneSecurityGroup")) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.ResourcesVpcConfig.SubnetIds).To(HaveLen(4)) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.RoleArn).To(ContainElement([]interface{}{"ServiceRole", "Arn"})) - Expect(clusterTemplate.Resources["ControlPlane"].Properties.EncryptionConfig).To(BeNil()) - }) - When("SecretsEncryption is configured", func() { BeforeEach(func() { cfg.SecretsEncryption = &api.SecretsEncryption{ diff --git a/pkg/cfn/builder/fakes/fake_cfn_template.go b/pkg/cfn/builder/fakes/fake_cfn_template.go index 1462583f9b..d01fb8f9f4 100644 --- a/pkg/cfn/builder/fakes/fake_cfn_template.go +++ b/pkg/cfn/builder/fakes/fake_cfn_template.go @@ -1,9 +1,5 @@ package fakes -import ( - cfn "github.com/aws/aws-sdk-go/service/cloudformation" -) - type FakeTemplate struct { Description string Resources map[string]struct { @@ -13,7 +9,7 @@ type FakeTemplate struct { UpdatePolicy map[string]map[string]interface{} } Mappings map[string]interface{} - Outputs map[string]cfn.Output + Outputs interface{} } type Tag struct { @@ -24,13 +20,14 @@ type Tag struct { } type Properties struct { - GroupDescription string - Description string - Tags []Tag - SecurityGroupIngress []SGIngress - GroupID interface{} - SourceSecurityGroupID interface{} - DestinationSecurityGroupID interface{} + EnableDNSHostnames, EnableDNSSupport bool + GroupDescription string + Description string + Tags []Tag + SecurityGroupIngress []SGIngress + GroupID interface{} + SourceSecurityGroupID interface{} + DestinationSecurityGroupID interface{} Path, RoleName string Roles, ManagedPolicyArns []interface{} @@ -62,16 +59,19 @@ type Properties struct { CidrIP, CidrIpv6, IPProtocol string FromPort, ToPort int - VpcID, SubnetID interface{} - RouteTableID, AllocationID interface{} - GatewayID, InternetGatewayID, NatGatewayID interface{} - DestinationCidrBlock interface{} - MapPublicIPOnLaunch bool + VpcID, SubnetID interface{} + EgressOnlyInternetGatewayID, RouteTableID, AllocationID interface{} + GatewayID, InternetGatewayID, NatGatewayID interface{} + DestinationCidrBlock, DestinationIpv6CidrBlock interface{} + MapPublicIPOnLaunch bool + AssignIpv6AddressOnCreation *bool - Ipv6CidrBlock map[string][]interface{} + Ipv6CidrBlock map[string][]interface{} + CidrBlock interface{} + KubernetesNetworkConfig KubernetesNetworkConfig - AmazonProvidedIpv6CidrBlock bool - AvailabilityZone, Domain, CidrBlock string + AmazonProvidedIpv6CidrBlock bool + AvailabilityZone, Domain string Name, Version string RoleArn interface{} @@ -112,6 +112,12 @@ type Properties struct { } } +type KubernetesNetworkConfig struct { + ServiceIPv4CIDR string + ServiceIPv6CIDR interface{} + IPFamily string +} + type SGIngress struct { SourceSecurityGroupID interface{} FromPort float64 diff --git a/pkg/cfn/builder/vpc.go b/pkg/cfn/builder/vpc.go index 2129f54b8b..01e062eee8 100644 --- a/pkg/cfn/builder/vpc.go +++ b/pkg/cfn/builder/vpc.go @@ -1,655 +1,69 @@ package builder import ( - "fmt" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/pkg/errors" - gfncfn "github.com/weaveworks/goformation/v4/cloudformation/cloudformation" - gfnec2 "github.com/weaveworks/goformation/v4/cloudformation/ec2" gfnt "github.com/weaveworks/goformation/v4/cloudformation/types" - - api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" - "github.com/weaveworks/eksctl/pkg/cfn/outputs" - "github.com/weaveworks/eksctl/pkg/vpc" ) -var internetCIDR = gfnt.NewString("0.0.0.0/0") - const ( - cfnControlPlaneSGResource = "ControlPlaneSecurityGroup" - cfnSharedNodeSGResource = "ClusterSharedNodeSecurityGroup" - cfnIngressClusterToNodeSGResource = "IngressDefaultClusterToNodeSG" -) - -// A VPCResourceSet builds the resources required for the specified VPC -type VPCResourceSet struct { - rs *resourceSet - clusterConfig *api.ClusterConfig - ec2API ec2iface.EC2API - - vpcResource *VPCResource -} - -// VPCResource represents a VPC resource -type VPCResource struct { - VPC *gfnt.Value - SubnetDetails *subnetDetails -} - -type SubnetResource struct { - Subnet *gfnt.Value - RouteTable *gfnt.Value - AvailabilityZone string -} - -type subnetDetails struct { - Private []SubnetResource - Public []SubnetResource -} - -// NewVPCResourceSet creates and returns a new VPCResourceSet -func NewVPCResourceSet(rs *resourceSet, clusterConfig *api.ClusterConfig, ec2API ec2iface.EC2API) *VPCResourceSet { - var vpcRef *gfnt.Value - if clusterConfig.VPC.ID == "" { - vpcRef = rs.newResource("VPC", &gfnec2.VPC{ - CidrBlock: gfnt.NewString(clusterConfig.VPC.CIDR.String()), - EnableDnsSupport: gfnt.True(), - EnableDnsHostnames: gfnt.True(), - }) - } else { - vpcRef = gfnt.NewString(clusterConfig.VPC.ID) - } - - return &VPCResourceSet{ - rs: rs, - clusterConfig: clusterConfig, - ec2API: ec2API, - - vpcResource: &VPCResource{ - VPC: vpcRef, - SubnetDetails: &subnetDetails{}, - }, - } -} - -// AddResources adds all required resources -func (v *VPCResourceSet) AddResources() (*VPCResource, error) { - vpc := v.clusterConfig.VPC - if vpc.ID != "" { // custom VPC has been set - if err := v.importResources(); err != nil { - return nil, errors.Wrap(err, "error importing VPC resources") - } - return v.vpcResource, nil - } - - if api.IsEnabled(vpc.AutoAllocateIPv6) { - v.rs.newResource("AutoAllocatedCIDRv6", &gfnec2.VPCCidrBlock{ - VpcId: v.vpcResource.VPC, - AmazonProvidedIpv6CidrBlock: gfnt.True(), - }) - } - - if v.isFullyPrivate() { - v.noNAT() - v.vpcResource.SubnetDetails.Private = v.addSubnets(nil, api.SubnetTopologyPrivate, vpc.Subnets.Private) - return v.vpcResource, nil - } - - refIG := v.rs.newResource("InternetGateway", &gfnec2.InternetGateway{}) - vpcGA := "VPCGatewayAttachment" - v.rs.newResource(vpcGA, &gfnec2.VPCGatewayAttachment{ - InternetGatewayId: refIG, - VpcId: v.vpcResource.VPC, - }) - - refPublicRT := v.rs.newResource("PublicRouteTable", &gfnec2.RouteTable{ - VpcId: v.vpcResource.VPC, - }) - - v.rs.newResource("PublicSubnetRoute", &gfnec2.Route{ - RouteTableId: refPublicRT, - DestinationCidrBlock: internetCIDR, - GatewayId: refIG, - AWSCloudFormationDependsOn: []string{vpcGA}, - }) - - v.vpcResource.SubnetDetails.Public = v.addSubnets(refPublicRT, api.SubnetTopologyPublic, vpc.Subnets.Public) - - if err := v.addNATGateways(); err != nil { - return nil, err - } - - v.vpcResource.SubnetDetails.Private = v.addSubnets(nil, api.SubnetTopologyPrivate, vpc.Subnets.Private) - return v.vpcResource, nil -} - -func (s *subnetDetails) PublicSubnetRefs() []*gfnt.Value { - var subnetRefs []*gfnt.Value - for _, subnetAZ := range s.Public { - subnetRefs = append(subnetRefs, subnetAZ.Subnet) - } - return subnetRefs -} - -func (s *subnetDetails) PrivateSubnetRefs() []*gfnt.Value { - var subnetRefs []*gfnt.Value - for _, subnetAZ := range s.Private { - subnetRefs = append(subnetRefs, subnetAZ.Subnet) - } - return subnetRefs -} - -// AddOutputs adds VPC resource outputs -func (v *VPCResourceSet) AddOutputs() { - v.rs.defineOutput(outputs.ClusterVPC, v.vpcResource.VPC, true, func(val string) error { - v.clusterConfig.VPC.ID = val - return nil - }) - if v.clusterConfig.VPC.NAT != nil { - v.rs.defineOutputWithoutCollector(outputs.ClusterFeatureNATMode, v.clusterConfig.VPC.NAT.Gateway, false) - } - - addSubnetOutput := func(subnetRefs []*gfnt.Value, topology api.SubnetTopology, outputName string) { - v.rs.defineJoinedOutput(outputName, subnetRefs, true, func(value string) error { - return vpc.ImportSubnetsFromIDList(v.ec2API, v.clusterConfig, topology, strings.Split(value, ",")) - }) - } - - if subnetAZs := v.vpcResource.SubnetDetails.PrivateSubnetRefs(); len(subnetAZs) > 0 { - addSubnetOutput(subnetAZs, api.SubnetTopologyPrivate, outputs.ClusterSubnetsPrivate) - } - - if subnetAZs := v.vpcResource.SubnetDetails.PublicSubnetRefs(); len(subnetAZs) > 0 { - addSubnetOutput(subnetAZs, api.SubnetTopologyPublic, outputs.ClusterSubnetsPublic) - } - - if v.isFullyPrivate() { - v.rs.defineOutputWithoutCollector(outputs.ClusterFullyPrivate, true, true) - } -} - -// RenderJSON returns the rendered JSON -func (v *VPCResourceSet) RenderJSON() ([]byte, error) { - return v.rs.renderJSON() -} - -func (v *VPCResourceSet) addSubnets(refRT *gfnt.Value, topology api.SubnetTopology, subnets map[string]api.AZSubnetSpec) []SubnetResource { - var subnetIndexForIPv6 int - if api.IsEnabled(v.clusterConfig.VPC.AutoAllocateIPv6) { - // this is same kind of indexing we have in vpc.SetSubnets - switch topology { - case api.SubnetTopologyPrivate: - subnetIndexForIPv6 = len(v.clusterConfig.AvailabilityZones) - case api.SubnetTopologyPublic: - subnetIndexForIPv6 = 0 - } - } - - var subnetResources []SubnetResource - - for name, subnet := range subnets { - az := subnet.AZ - nameAlias := strings.ToUpper(strings.Join(strings.Split(name, "-"), "")) - subnet := &gfnec2.Subnet{ - AvailabilityZone: gfnt.NewString(az), - CidrBlock: gfnt.NewString(subnet.CIDR.String()), - VpcId: v.vpcResource.VPC, - } - - switch topology { - case api.SubnetTopologyPrivate: - // Choose the appropriate route table for private subnets - refRT = gfnt.MakeRef("PrivateRouteTable" + nameAlias) - subnet.Tags = []gfncfn.Tag{{ - Key: gfnt.NewString("kubernetes.io/role/internal-elb"), - Value: gfnt.NewString("1"), - }} - case api.SubnetTopologyPublic: - subnet.Tags = []gfncfn.Tag{{ - Key: gfnt.NewString("kubernetes.io/role/elb"), - Value: gfnt.NewString("1"), - }} - subnet.MapPublicIpOnLaunch = gfnt.True() - } - subnetAlias := string(topology) + nameAlias - refSubnet := v.rs.newResource("Subnet"+subnetAlias, subnet) - v.rs.newResource("RouteTableAssociation"+subnetAlias, &gfnec2.SubnetRouteTableAssociation{ - SubnetId: refSubnet, - RouteTableId: refRT, - }) - - if api.IsEnabled(v.clusterConfig.VPC.AutoAllocateIPv6) { - // get 8 of /64 subnets from the auto-allocated IPv6 block, - // and pick one block based on subnetIndexForIPv6 counter; - // NOTE: this is done inside of CloudFormation using Fn::Cidr, - // we don't slice it here, just construct the JSON expression - // that does slicing at runtime. - refAutoAllocateCIDRv6 := gfnt.MakeFnSelect( - gfnt.NewInteger(0), gfnt.MakeFnGetAttString("VPC", "Ipv6CidrBlocks"), - ) - refSubnetSlices := gfnt.MakeFnCIDR( - refAutoAllocateCIDRv6, gfnt.NewInteger(8), gfnt.NewInteger(64), - ) - v.rs.newResource(subnetAlias+"CIDRv6", &gfnec2.SubnetCidrBlock{ - SubnetId: refSubnet, - Ipv6CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(subnetIndexForIPv6), refSubnetSlices), - }) - subnetIndexForIPv6++ - } - - subnetResources = append(subnetResources, SubnetResource{ - AvailabilityZone: az, - RouteTable: refRT, - Subnet: refSubnet, - }) - } - return subnetResources -} - -func (v *VPCResourceSet) addNATGateways() error { - switch *v.clusterConfig.VPC.NAT.Gateway { - case api.ClusterHighlyAvailableNAT: - v.haNAT() - case api.ClusterSingleNAT: - v.singleNAT() - case api.ClusterDisableNAT: - v.noNAT() - default: - // TODO validate this before starting to add resources - return fmt.Errorf("%s is not a valid NAT gateway mode", *v.clusterConfig.VPC.NAT.Gateway) - } - return nil -} - -func (v *VPCResourceSet) importResources() error { - if subnets := v.clusterConfig.VPC.Subnets.Private; subnets != nil { - var ( - subnetRoutes map[string]string - err error - ) - if v.isFullyPrivate() { - subnetRoutes, err = importRouteTables(v.ec2API, v.clusterConfig.VPC.Subnets.Private) - if err != nil { - return err - } - } - - subnetResources, err := makeSubnetResources(subnets, subnetRoutes) - if err != nil { - return err - } - v.vpcResource.SubnetDetails.Private = subnetResources - } - - if subnets := v.clusterConfig.VPC.Subnets.Public; subnets != nil { - subnetResources, err := makeSubnetResources(subnets, nil) - if err != nil { - return err - } - v.vpcResource.SubnetDetails.Public = subnetResources - } - - return nil -} - -func makeSubnetResources(subnets map[string]api.AZSubnetSpec, subnetRoutes map[string]string) ([]SubnetResource, error) { - subnetResources := make([]SubnetResource, len(subnets)) - i := 0 - for _, network := range subnets { - az := network.AZ - sr := SubnetResource{ - AvailabilityZone: az, - Subnet: gfnt.NewString(network.ID), - } - - if subnetRoutes != nil { - rt, ok := subnetRoutes[network.ID] - if !ok { - return nil, errors.Errorf("failed to find an explicit route table associated with subnet %q; "+ - "eksctl does not modify the main route table if a subnet is not associated with an explicit route table", network.ID) - } - sr.RouteTable = gfnt.NewString(rt) - } - subnetResources[i] = sr - i++ - } - return subnetResources, nil -} - -func importRouteTables(ec2API ec2iface.EC2API, subnets map[string]api.AZSubnetSpec) (map[string]string, error) { - var subnetIDs []string - for _, subnet := range subnets { - subnetIDs = append(subnetIDs, subnet.ID) - } - - var routeTables []*ec2.RouteTable - var nextToken *string - - for { - output, err := ec2API.DescribeRouteTables(&ec2.DescribeRouteTablesInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("association.subnet-id"), - Values: aws.StringSlice(subnetIDs), - }, - }, - NextToken: nextToken, - }) - - if err != nil { - return nil, errors.Wrap(err, "error describing route tables") - } - - routeTables = append(routeTables, output.RouteTables...) - - if nextToken = output.NextToken; nextToken == nil { - break - } - } - - subnetRoutes := make(map[string]string) - for _, rt := range routeTables { - for _, rta := range rt.Associations { - if rta.Main != nil && *rta.Main { - return nil, errors.New("subnets must be associated with a non-main route table; eksctl does not modify the main route table") - } - subnetRoutes[*rta.SubnetId] = *rt.RouteTableId - } - } - return subnetRoutes, nil -} - -func (v *VPCResourceSet) isFullyPrivate() bool { - return v.clusterConfig.PrivateCluster.Enabled -} - -var ( - sgProtoTCP = gfnt.NewString("tcp") - sgSourceAnywhereIPv4 = gfnt.NewString("0.0.0.0/0") - sgSourceAnywhereIPv6 = gfnt.NewString("::/0") - - sgPortZero = gfnt.NewInteger(0) - sgMinNodePort = gfnt.NewInteger(1025) - sgMaxNodePort = gfnt.NewInteger(65535) - - sgPortHTTPS = gfnt.NewInteger(443) - sgPortSSH = gfnt.NewInteger(22) + VPCResourceKey = "VPC" + + // Gateways + IGWKey = "InternetGateway" + GAKey = "VPCGatewayAttachment" + EgressOnlyInternetGatewayKey = "EgressOnlyInternetGateway" + NATGatewayKey = "NATGateway" + ElasticIPKey = "EIP" + + // CIDRs + IPv6CIDRBlockKey = "IPv6CidrBlock" + InternetCIDR = "0.0.0.0/0" + InternetIPv6CIDR = "::/0" + + // Routing + PubRouteTableKey = "PublicRouteTable" + PrivateRouteTableKey = "PrivateRouteTable" + PubRouteTableAssociation = "RouteTableAssociationPublic" + PrivateRouteTableAssociation = "RouteTableAssociationPrivate" + PubSubRouteKey = "PublicSubnetDefaultRoute" + PubSubIPv6RouteKey = "PublicSubnetIPv6DefaultRoute" + PrivateSubnetRouteKey = "PrivateSubnetDefaultRoute" + PrivateSubnetIpv6RouteKey = "PrivateSubnetDefaultIpv6Route" + + // Subnets + PublicSubnetKey = "PublicSubnet" + PrivateSubnetKey = "PrivateSubnet" + PublicSubnetsOutputKey = "SubnetsPublic" + PrivateSubnetsOutputKey = "SubnetsPrivate" ) -type clusterSecurityGroup struct { - ControlPlane *gfnt.Value - ClusterSharedNode *gfnt.Value +//VPCResourceSet interface for creating cloudformation resource sets for generating VPC resources +type VPCResourceSet interface { + //CreateTemplate generates all of the resources & outputs required for the VPC. Returns the + CreateTemplate() (vpcID *gfnt.Value, subnetDetails *SubnetDetails, err error) } -// TODO move this -func (c *ClusterResourceSet) addResourcesForSecurityGroups(vpcResource *VPCResource) *clusterSecurityGroup { - var refControlPlaneSG, refClusterSharedNodeSG *gfnt.Value - - if c.spec.VPC.SecurityGroup == "" { - refControlPlaneSG = c.newResource(cfnControlPlaneSGResource, &gfnec2.SecurityGroup{ - GroupDescription: gfnt.NewString("Communication between the control plane and worker nodegroups"), - VpcId: vpcResource.VPC, - }) - - if len(c.spec.VPC.ExtraCIDRs) > 0 { - for i, cidr := range c.spec.VPC.ExtraCIDRs { - c.newResource(fmt.Sprintf("IngressControlPlaneExtraCIDR%d", i), &gfnec2.SecurityGroupIngress{ - GroupId: refControlPlaneSG, - CidrIp: gfnt.NewString(cidr), - Description: gfnt.NewString(fmt.Sprintf("Allow Extra CIDR %d (%s) to communicate to controlplane", i, cidr)), - IpProtocol: gfnt.NewString("tcp"), - FromPort: sgPortHTTPS, - ToPort: sgPortHTTPS, - }) - } - } - } else { - refControlPlaneSG = gfnt.NewString(c.spec.VPC.SecurityGroup) - } - c.securityGroups = []*gfnt.Value{refControlPlaneSG} // only this one SG is passed to EKS API, nodes are isolated - - if c.spec.VPC.SharedNodeSecurityGroup == "" { - refClusterSharedNodeSG = c.newResource(cfnSharedNodeSGResource, &gfnec2.SecurityGroup{ - GroupDescription: gfnt.NewString("Communication between all nodes in the cluster"), - VpcId: vpcResource.VPC, - }) - c.newResource("IngressInterNodeGroupSG", &gfnec2.SecurityGroupIngress{ - GroupId: refClusterSharedNodeSG, - SourceSecurityGroupId: refClusterSharedNodeSG, - Description: gfnt.NewString("Allow nodes to communicate with each other (all ports)"), - IpProtocol: gfnt.NewString("-1"), - FromPort: sgPortZero, - ToPort: sgMaxNodePort, - }) - } else { - refClusterSharedNodeSG = gfnt.NewString(c.spec.VPC.SharedNodeSecurityGroup) - } - - if c.supportsManagedNodes && api.IsEnabled(c.spec.VPC.ManageSharedNodeSecurityGroupRules) { - // To enable communication between both managed and unmanaged nodegroups, this allows ingress traffic from - // the default cluster security group ID that EKS creates by default - // EKS attaches this to Managed Nodegroups by default, but we need to handle this for unmanaged nodegroups - c.newResource(cfnIngressClusterToNodeSGResource, &gfnec2.SecurityGroupIngress{ - GroupId: refClusterSharedNodeSG, - SourceSecurityGroupId: gfnt.MakeFnGetAttString("ControlPlane", outputs.ClusterDefaultSecurityGroup), - Description: gfnt.NewString("Allow managed and unmanaged nodes to communicate with each other (all ports)"), - IpProtocol: gfnt.NewString("-1"), - FromPort: sgPortZero, - ToPort: sgMaxNodePort, - }) - c.newResource("IngressNodeToDefaultClusterSG", &gfnec2.SecurityGroupIngress{ - GroupId: gfnt.MakeFnGetAttString("ControlPlane", outputs.ClusterDefaultSecurityGroup), - SourceSecurityGroupId: refClusterSharedNodeSG, - Description: gfnt.NewString("Allow unmanaged nodes to communicate with control plane (all ports)"), - IpProtocol: gfnt.NewString("-1"), - FromPort: sgPortZero, - ToPort: sgMaxNodePort, - }) - } - - if c.spec.VPC == nil { - c.spec.VPC = &api.ClusterVPC{} - } - c.rs.defineOutput(outputs.ClusterSecurityGroup, refControlPlaneSG, true, func(v string) error { - c.spec.VPC.SecurityGroup = v - return nil - }) - c.rs.defineOutput(outputs.ClusterSharedNodeSecurityGroup, refClusterSharedNodeSG, true, func(v string) error { - c.spec.VPC.SharedNodeSecurityGroup = v - return nil - }) - - return &clusterSecurityGroup{ - ControlPlane: refControlPlaneSG, - ClusterSharedNode: refClusterSharedNodeSG, - } +func formatAZ(az string) string { + return strings.ToUpper(strings.ReplaceAll(az, "-", "")) } -// TODO move this -func (rs *resourceSet) addEFASecurityGroup(vpcID *gfnt.Value, clusterName, desc string) *gfnt.Value { - efaSG := rs.newResource("EFASG", &gfnec2.SecurityGroup{ - VpcId: vpcID, - GroupDescription: gfnt.NewString("EFA-enabled security group"), - Tags: []gfncfn.Tag{{ - Key: gfnt.NewString("kubernetes.io/cluster/" + clusterName), - Value: gfnt.NewString("owned"), - }}, - }) - rs.newResource("EFAIngressSelf", &gfnec2.SecurityGroupIngress{ - GroupId: efaSG, - SourceSecurityGroupId: efaSG, - Description: gfnt.NewString("Allow " + desc + " to communicate to itself (EFA-enabled)"), - IpProtocol: gfnt.NewString("-1"), - }) - rs.newResource("EFAEgressSelf", &gfnec2.SecurityGroupEgress{ - GroupId: efaSG, - DestinationSecurityGroupId: efaSG, - Description: gfnt.NewString("Allow " + desc + " to communicate to itself (EFA-enabled)"), - IpProtocol: gfnt.NewString("-1"), - }) - - return efaSG -} - -// TODO move this -func (n *NodeGroupResourceSet) addResourcesForSecurityGroups() { - for _, id := range n.spec.SecurityGroups.AttachIDs { - n.securityGroups = append(n.securityGroups, gfnt.NewString(id)) - } - - if api.IsEnabled(n.spec.SecurityGroups.WithShared) { - n.securityGroups = append(n.securityGroups, n.vpcImporter.SharedNodeSecurityGroup()) - } - - if api.IsDisabled(n.spec.SecurityGroups.WithLocal) { - return - } - - desc := "worker nodes in group " + n.spec.Name - vpcID := n.vpcImporter.VPC() - refControlPlaneSG := n.vpcImporter.ControlPlaneSecurityGroup() - - refNodeGroupLocalSG := n.newResource("SG", &gfnec2.SecurityGroup{ - VpcId: vpcID, - GroupDescription: gfnt.NewString("Communication between the control plane and " + desc), - Tags: []gfncfn.Tag{{ - Key: gfnt.NewString("kubernetes.io/cluster/" + n.clusterSpec.Metadata.Name), - Value: gfnt.NewString("owned"), - }}, - SecurityGroupIngress: makeNodeIngressRules(n.spec.NodeGroupBase, refControlPlaneSG, n.clusterSpec.VPC.CIDR.String(), desc), - }) - - n.securityGroups = append(n.securityGroups, refNodeGroupLocalSG) - - if api.IsEnabled(n.spec.EFAEnabled) { - efaSG := n.rs.addEFASecurityGroup(vpcID, n.clusterSpec.Metadata.Name, desc) - n.securityGroups = append(n.securityGroups, efaSG) - } - - n.newResource("EgressInterCluster", &gfnec2.SecurityGroupEgress{ - GroupId: refControlPlaneSG, - DestinationSecurityGroupId: refNodeGroupLocalSG, - Description: gfnt.NewString("Allow control plane to communicate with " + desc + " (kubelet and workload TCP ports)"), - IpProtocol: sgProtoTCP, - FromPort: sgMinNodePort, - ToPort: sgMaxNodePort, - }) - n.newResource("EgressInterClusterAPI", &gfnec2.SecurityGroupEgress{ - GroupId: refControlPlaneSG, - DestinationSecurityGroupId: refNodeGroupLocalSG, - Description: gfnt.NewString("Allow control plane to communicate with " + desc + " (workloads using HTTPS port, commonly used with extension API servers)"), - IpProtocol: sgProtoTCP, - FromPort: sgPortHTTPS, - ToPort: sgPortHTTPS, - }) - n.newResource("IngressInterClusterCP", &gfnec2.SecurityGroupIngress{ - GroupId: refControlPlaneSG, - SourceSecurityGroupId: refNodeGroupLocalSG, - Description: gfnt.NewString("Allow control plane to receive API requests from " + desc), - IpProtocol: sgProtoTCP, - FromPort: sgPortHTTPS, - ToPort: sgPortHTTPS, - }) -} - -func makeNodeIngressRules(ng *api.NodeGroupBase, controlPlaneSG *gfnt.Value, vpcCIDR, description string) []gfnec2.SecurityGroup_Ingress { - ingressRules := []gfnec2.SecurityGroup_Ingress{ - { - SourceSecurityGroupId: controlPlaneSG, - Description: gfnt.NewString(fmt.Sprintf("[IngressInterCluster] Allow %s to communicate with control plane (kubelet and workload TCP ports)", description)), - IpProtocol: sgProtoTCP, - FromPort: sgMinNodePort, - ToPort: sgMaxNodePort, - }, - { - SourceSecurityGroupId: controlPlaneSG, - Description: gfnt.NewString(fmt.Sprintf("[IngressInterClusterAPI] Allow %s to communicate with control plane (workloads using HTTPS port, commonly used with extension API servers)", description)), - IpProtocol: sgProtoTCP, - FromPort: sgPortHTTPS, - ToPort: sgPortHTTPS, - }, - } - - return append(ingressRules, makeSSHIngressRules(ng, vpcCIDR, description)...) -} - -func (v *VPCResourceSet) haNAT() { - for _, az := range v.clusterConfig.AvailabilityZones { - alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) - - // Allocate an EIP - v.rs.newResource("NATIP"+alphanumericUpperAZ, &gfnec2.EIP{ - Domain: gfnt.NewString("vpc"), - }) - // Allocate a NAT gateway in the public subnet - refNG := v.rs.newResource("NATGateway"+alphanumericUpperAZ, &gfnec2.NatGateway{ - AllocationId: gfnt.MakeFnGetAttString("NATIP"+alphanumericUpperAZ, "AllocationId"), - SubnetId: gfnt.MakeRef("SubnetPublic" + alphanumericUpperAZ), - }) - - // Allocate a routing table for the private subnet - refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ - VpcId: v.vpcResource.VPC, - }) - // Create a route that sends Internet traffic through the NAT gateway - v.rs.newResource("NATPrivateSubnetRoute"+alphanumericUpperAZ, &gfnec2.Route{ - RouteTableId: refRT, - DestinationCidrBlock: internetCIDR, - NatGatewayId: refNG, - }) - // Associate the routing table with the subnet - v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ - SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), - RouteTableId: refRT, - }) - } -} - -func (v *VPCResourceSet) singleNAT() { - sortedAZs := v.clusterConfig.AvailabilityZones - firstUpperAZ := strings.ToUpper(strings.Join(strings.Split(sortedAZs[0], "-"), "")) - - v.rs.newResource("NATIP", &gfnec2.EIP{ - Domain: gfnt.NewString("vpc"), - }) - refNG := v.rs.newResource("NATGateway", &gfnec2.NatGateway{ - AllocationId: gfnt.MakeFnGetAttString("NATIP", "AllocationId"), - SubnetId: gfnt.MakeRef("SubnetPublic" + firstUpperAZ), - }) - - for _, az := range v.clusterConfig.AvailabilityZones { - alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) - - refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ - VpcId: v.vpcResource.VPC, - }) - - v.rs.newResource("NATPrivateSubnetRoute"+alphanumericUpperAZ, &gfnec2.Route{ - RouteTableId: refRT, - DestinationCidrBlock: internetCIDR, - NatGatewayId: refNG, - }) - v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ - SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), - RouteTableId: refRT, - }) - } +func getSubnetIPv6CIDRBlock(cidrPartitions int) *gfnt.Value { + // get 8 of /64 subnets from the auto-allocated IPv6 block, + // and pick one block based on subnetIndexForIPv6 counter; + // NOTE: this is done inside of CloudFormation using Fn::Cidr, + // we don't slice it here, just construct the JSON expression + // that does slicing at runtime. + refIPv6CIDRv6 := gfnt.MakeFnSelect( + gfnt.NewInteger(0), gfnt.MakeFnGetAttString("VPC", "Ipv6CidrBlocks"), + ) + refSubnetSlices := gfnt.MakeFnCIDR(refIPv6CIDRv6, gfnt.NewInteger(cidrPartitions), gfnt.NewInteger(64)) + return refSubnetSlices } -func (v *VPCResourceSet) noNAT() { - for _, az := range v.clusterConfig.AvailabilityZones { - alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) - - refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ - VpcId: v.vpcResource.VPC, - }) - v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ - SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), - RouteTableId: refRT, - }) - } +func getSubnetIPv4CIDRBlock(cidrPartitions int) *gfnt.Value { + //TODO: should we be doing /19? Should we adjust for the partition size? + desiredMask := 19 + refSubnetSlices := gfnt.MakeFnCIDR(gfnt.MakeFnGetAttString("VPC", "CidrBlock"), gfnt.NewInteger(cidrPartitions), gfnt.NewInteger(32-desiredMask)) + return refSubnetSlices } diff --git a/pkg/cfn/builder/vpc_endpoint_test.go b/pkg/cfn/builder/vpc_endpoint_test.go index 6a2a4152c2..e7028abddd 100644 --- a/pkg/cfn/builder/vpc_endpoint_test.go +++ b/pkg/cfn/builder/vpc_endpoint_test.go @@ -29,7 +29,7 @@ type vpcResourceSetCase struct { var _ = Describe("VPC Endpoint Builder", func() { - DescribeTable("Add resources", func(vc vpcResourceSetCase) { + DescribeTable("Adds resources to template", func(vc vpcResourceSetCase) { api.SetClusterConfigDefaults(vc.clusterConfig) if len(vc.clusterConfig.AvailabilityZones) == 0 { @@ -54,8 +54,8 @@ var _ = Describe("VPC Endpoint Builder", func() { } rs := newResourceSet() - vpcResourceSet := NewVPCResourceSet(rs, vc.clusterConfig, provider.EC2()) - vpcResource, err := vpcResourceSet.AddResources() + vpcResourceSet := NewIPv4VPCResourceSet(rs, vc.clusterConfig, provider.EC2()) + vpcID, subnetDetails, err := vpcResourceSet.CreateTemplate() if vc.err != "" { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("subnets must be associated with a non-main route table")) @@ -64,7 +64,7 @@ var _ = Describe("VPC Endpoint Builder", func() { Expect(err).ToNot(HaveOccurred()) if vc.clusterConfig.PrivateCluster.Enabled { - vpcEndpointResourceSet := NewVPCEndpointResourceSet(provider.EC2(), provider.Region(), rs, vc.clusterConfig, vpcResource.VPC, vpcResource.SubnetDetails.Private, gfnt.NewString("sg-test")) + vpcEndpointResourceSet := NewVPCEndpointResourceSet(provider.EC2(), provider.Region(), rs, vc.clusterConfig, vpcID, subnetDetails.Private, gfnt.NewString("sg-test")) Expect(vpcEndpointResourceSet.AddResources()).To(Succeed()) s3Endpoint := rs.template.Resources["VPCEndpointS3"].(*gfnec2.VPCEndpoint) routeIdsSlice, ok := s3Endpoint.RouteTableIds.Raw().(gfnt.Slice) @@ -77,6 +77,7 @@ var _ = Describe("VPC Endpoint Builder", func() { return } + rs.template.Outputs = nil resourceJSON, err := rs.template.JSON() Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/cfn/builder/vpc_ipv4.go b/pkg/cfn/builder/vpc_ipv4.go new file mode 100644 index 0000000000..6a3dbb77d1 --- /dev/null +++ b/pkg/cfn/builder/vpc_ipv4.go @@ -0,0 +1,642 @@ +package builder + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/pkg/errors" + gfncfn "github.com/weaveworks/goformation/v4/cloudformation/cloudformation" + gfnec2 "github.com/weaveworks/goformation/v4/cloudformation/ec2" + gfnt "github.com/weaveworks/goformation/v4/cloudformation/types" + + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/outputs" + "github.com/weaveworks/eksctl/pkg/vpc" +) + +const ( + cfnControlPlaneSGResource = "ControlPlaneSecurityGroup" + cfnSharedNodeSGResource = "ClusterSharedNodeSecurityGroup" + cfnIngressClusterToNodeSGResource = "IngressDefaultClusterToNodeSG" +) + +// A IPv4VPCResourceSet builds the resources required for the specified VPC +type IPv4VPCResourceSet struct { + rs *resourceSet + clusterConfig *api.ClusterConfig + ec2API ec2iface.EC2API + vpcID *gfnt.Value + subnetDetails *SubnetDetails +} + +type SubnetResource struct { + Subnet *gfnt.Value + RouteTable *gfnt.Value + AvailabilityZone string +} + +type SubnetDetails struct { + Private []SubnetResource + Public []SubnetResource +} + +// NewIPv4VPCResourceSet creates and returns a new VPCResourceSet +func NewIPv4VPCResourceSet(rs *resourceSet, clusterConfig *api.ClusterConfig, ec2API ec2iface.EC2API) *IPv4VPCResourceSet { + var vpcRef *gfnt.Value + if clusterConfig.VPC.ID == "" { + vpcRef = rs.newResource("VPC", &gfnec2.VPC{ + CidrBlock: gfnt.NewString(clusterConfig.VPC.CIDR.String()), + EnableDnsSupport: gfnt.True(), + EnableDnsHostnames: gfnt.True(), + }) + } else { + vpcRef = gfnt.NewString(clusterConfig.VPC.ID) + } + + return &IPv4VPCResourceSet{ + rs: rs, + clusterConfig: clusterConfig, + ec2API: ec2API, + vpcID: vpcRef, + subnetDetails: &SubnetDetails{}, + } +} + +func (v *IPv4VPCResourceSet) CreateTemplate() (*gfnt.Value, *SubnetDetails, error) { + err := v.addResources() + if err != nil { + return nil, nil, err + } + v.addOutputs() + return v.vpcID, v.subnetDetails, nil +} + +// AddResources adds all required resources +func (v *IPv4VPCResourceSet) addResources() error { + vpc := v.clusterConfig.VPC + if vpc.ID != "" { // custom VPC has been set + if err := v.importResources(); err != nil { + return errors.Wrap(err, "error importing VPC resources") + } + return nil + } + + if api.IsEnabled(vpc.AutoAllocateIPv6) { + v.rs.newResource("AutoAllocatedCIDRv6", &gfnec2.VPCCidrBlock{ + VpcId: v.vpcID, + AmazonProvidedIpv6CidrBlock: gfnt.True(), + }) + } + + if v.isFullyPrivate() { + v.noNAT() + v.subnetDetails.Private = v.addSubnets(nil, api.SubnetTopologyPrivate, vpc.Subnets.Private) + return nil + } + + refIG := v.rs.newResource("InternetGateway", &gfnec2.InternetGateway{}) + vpcGA := "VPCGatewayAttachment" + v.rs.newResource(vpcGA, &gfnec2.VPCGatewayAttachment{ + InternetGatewayId: refIG, + VpcId: v.vpcID, + }) + + refPublicRT := v.rs.newResource("PublicRouteTable", &gfnec2.RouteTable{ + VpcId: v.vpcID, + }) + + v.rs.newResource("PublicSubnetRoute", &gfnec2.Route{ + RouteTableId: refPublicRT, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + GatewayId: refIG, + AWSCloudFormationDependsOn: []string{vpcGA}, + }) + + v.subnetDetails.Public = v.addSubnets(refPublicRT, api.SubnetTopologyPublic, vpc.Subnets.Public) + + if err := v.addNATGateways(); err != nil { + return err + } + + v.subnetDetails.Private = v.addSubnets(nil, api.SubnetTopologyPrivate, vpc.Subnets.Private) + return nil +} + +func (s *SubnetDetails) PublicSubnetRefs() []*gfnt.Value { + var subnetRefs []*gfnt.Value + for _, subnetAZ := range s.Public { + subnetRefs = append(subnetRefs, subnetAZ.Subnet) + } + return subnetRefs +} + +func (s *SubnetDetails) PrivateSubnetRefs() []*gfnt.Value { + var subnetRefs []*gfnt.Value + for _, subnetAZ := range s.Private { + subnetRefs = append(subnetRefs, subnetAZ.Subnet) + } + return subnetRefs +} + +// addOutputs adds VPC resource outputs +func (v *IPv4VPCResourceSet) addOutputs() { + v.rs.defineOutput(outputs.ClusterVPC, v.vpcID, true, func(val string) error { + v.clusterConfig.VPC.ID = val + return nil + }) + if v.clusterConfig.VPC.NAT != nil { + v.rs.defineOutputWithoutCollector(outputs.ClusterFeatureNATMode, v.clusterConfig.VPC.NAT.Gateway, false) + } + + addSubnetOutput := func(subnetRefs []*gfnt.Value, topology api.SubnetTopology, outputName string) { + v.rs.defineJoinedOutput(outputName, subnetRefs, true, func(value string) error { + return vpc.ImportSubnetsFromIDList(v.ec2API, v.clusterConfig, topology, strings.Split(value, ",")) + }) + } + + if subnetAZs := v.subnetDetails.PrivateSubnetRefs(); len(subnetAZs) > 0 { + addSubnetOutput(subnetAZs, api.SubnetTopologyPrivate, outputs.ClusterSubnetsPrivate) + } + + if subnetAZs := v.subnetDetails.PublicSubnetRefs(); len(subnetAZs) > 0 { + addSubnetOutput(subnetAZs, api.SubnetTopologyPublic, outputs.ClusterSubnetsPublic) + } + + if v.isFullyPrivate() { + v.rs.defineOutputWithoutCollector(outputs.ClusterFullyPrivate, true, true) + } +} + +// RenderJSON returns the rendered JSON +func (v *IPv4VPCResourceSet) RenderJSON() ([]byte, error) { + return v.rs.renderJSON() +} + +func (v *IPv4VPCResourceSet) addSubnets(refRT *gfnt.Value, topology api.SubnetTopology, subnets map[string]api.AZSubnetSpec) []SubnetResource { + var subnetIndexForIPv6 int + if api.IsEnabled(v.clusterConfig.VPC.AutoAllocateIPv6) { + // this is same kind of indexing we have in vpc.SetSubnets + switch topology { + case api.SubnetTopologyPrivate: + subnetIndexForIPv6 = len(v.clusterConfig.AvailabilityZones) + case api.SubnetTopologyPublic: + subnetIndexForIPv6 = 0 + } + } + + var subnetResources []SubnetResource + + for name, subnet := range subnets { + az := subnet.AZ + nameAlias := strings.ToUpper(strings.Join(strings.Split(name, "-"), "")) + subnet := &gfnec2.Subnet{ + AvailabilityZone: gfnt.NewString(az), + CidrBlock: gfnt.NewString(subnet.CIDR.String()), + VpcId: v.vpcID, + } + + switch topology { + case api.SubnetTopologyPrivate: + // Choose the appropriate route table for private subnets + refRT = gfnt.MakeRef("PrivateRouteTable" + nameAlias) + subnet.Tags = []gfncfn.Tag{{ + Key: gfnt.NewString("kubernetes.io/role/internal-elb"), + Value: gfnt.NewString("1"), + }} + case api.SubnetTopologyPublic: + subnet.Tags = []gfncfn.Tag{{ + Key: gfnt.NewString("kubernetes.io/role/elb"), + Value: gfnt.NewString("1"), + }} + subnet.MapPublicIpOnLaunch = gfnt.True() + } + subnetAlias := string(topology) + nameAlias + refSubnet := v.rs.newResource("Subnet"+subnetAlias, subnet) + v.rs.newResource("RouteTableAssociation"+subnetAlias, &gfnec2.SubnetRouteTableAssociation{ + SubnetId: refSubnet, + RouteTableId: refRT, + }) + + if api.IsEnabled(v.clusterConfig.VPC.AutoAllocateIPv6) { + refSubnetSlices := getSubnetIPv6CIDRBlock((len(v.clusterConfig.AvailabilityZones) * 2) + 2) + v.rs.newResource(subnetAlias+"CIDRv6", &gfnec2.SubnetCidrBlock{ + SubnetId: refSubnet, + Ipv6CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(subnetIndexForIPv6), refSubnetSlices), + }) + subnetIndexForIPv6++ + } + + subnetResources = append(subnetResources, SubnetResource{ + AvailabilityZone: az, + RouteTable: refRT, + Subnet: refSubnet, + }) + } + return subnetResources +} + +func (v *IPv4VPCResourceSet) addNATGateways() error { + switch *v.clusterConfig.VPC.NAT.Gateway { + case api.ClusterHighlyAvailableNAT: + v.haNAT() + case api.ClusterSingleNAT: + v.singleNAT() + case api.ClusterDisableNAT: + v.noNAT() + default: + // TODO validate this before starting to add resources + return fmt.Errorf("%s is not a valid NAT gateway mode", *v.clusterConfig.VPC.NAT.Gateway) + } + return nil +} + +func (v *IPv4VPCResourceSet) importResources() error { + if subnets := v.clusterConfig.VPC.Subnets.Private; subnets != nil { + var ( + subnetRoutes map[string]string + err error + ) + if v.isFullyPrivate() { + subnetRoutes, err = importRouteTables(v.ec2API, v.clusterConfig.VPC.Subnets.Private) + if err != nil { + return err + } + } + + subnetResources, err := makeSubnetResources(subnets, subnetRoutes) + if err != nil { + return err + } + v.subnetDetails.Private = subnetResources + } + + if subnets := v.clusterConfig.VPC.Subnets.Public; subnets != nil { + subnetResources, err := makeSubnetResources(subnets, nil) + if err != nil { + return err + } + v.subnetDetails.Public = subnetResources + } + + return nil +} + +func makeSubnetResources(subnets map[string]api.AZSubnetSpec, subnetRoutes map[string]string) ([]SubnetResource, error) { + subnetResources := make([]SubnetResource, len(subnets)) + i := 0 + for _, network := range subnets { + az := network.AZ + sr := SubnetResource{ + AvailabilityZone: az, + Subnet: gfnt.NewString(network.ID), + } + + if subnetRoutes != nil { + rt, ok := subnetRoutes[network.ID] + if !ok { + return nil, errors.Errorf("failed to find an explicit route table associated with subnet %q; "+ + "eksctl does not modify the main route table if a subnet is not associated with an explicit route table", network.ID) + } + sr.RouteTable = gfnt.NewString(rt) + } + subnetResources[i] = sr + i++ + } + return subnetResources, nil +} + +func importRouteTables(ec2API ec2iface.EC2API, subnets map[string]api.AZSubnetSpec) (map[string]string, error) { + var subnetIDs []string + for _, subnet := range subnets { + subnetIDs = append(subnetIDs, subnet.ID) + } + + var routeTables []*ec2.RouteTable + var nextToken *string + + for { + output, err := ec2API.DescribeRouteTables(&ec2.DescribeRouteTablesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("association.subnet-id"), + Values: aws.StringSlice(subnetIDs), + }, + }, + NextToken: nextToken, + }) + + if err != nil { + return nil, errors.Wrap(err, "error describing route tables") + } + + routeTables = append(routeTables, output.RouteTables...) + + if nextToken = output.NextToken; nextToken == nil { + break + } + } + + subnetRoutes := make(map[string]string) + for _, rt := range routeTables { + for _, rta := range rt.Associations { + if rta.Main != nil && *rta.Main { + return nil, errors.New("subnets must be associated with a non-main route table; eksctl does not modify the main route table") + } + subnetRoutes[*rta.SubnetId] = *rt.RouteTableId + } + } + return subnetRoutes, nil +} + +func (v *IPv4VPCResourceSet) isFullyPrivate() bool { + return v.clusterConfig.PrivateCluster.Enabled +} + +var ( + sgProtoTCP = gfnt.NewString("tcp") + sgSourceAnywhereIPv4 = gfnt.NewString("0.0.0.0/0") + sgSourceAnywhereIPv6 = gfnt.NewString("::/0") + + sgPortZero = gfnt.NewInteger(0) + sgMinNodePort = gfnt.NewInteger(1025) + sgMaxNodePort = gfnt.NewInteger(65535) + + sgPortHTTPS = gfnt.NewInteger(443) + sgPortSSH = gfnt.NewInteger(22) +) + +type clusterSecurityGroup struct { + ControlPlane *gfnt.Value + ClusterSharedNode *gfnt.Value +} + +func (c *ClusterResourceSet) addResourcesForSecurityGroups(vpcID *gfnt.Value) *clusterSecurityGroup { + var refControlPlaneSG, refClusterSharedNodeSG *gfnt.Value + + if c.spec.VPC.SecurityGroup == "" { + refControlPlaneSG = c.newResource(cfnControlPlaneSGResource, &gfnec2.SecurityGroup{ + GroupDescription: gfnt.NewString("Communication between the control plane and worker nodegroups"), + VpcId: vpcID, + }) + + if len(c.spec.VPC.ExtraCIDRs) > 0 { + for i, cidr := range c.spec.VPC.ExtraCIDRs { + c.newResource(fmt.Sprintf("IngressControlPlaneExtraCIDR%d", i), &gfnec2.SecurityGroupIngress{ + GroupId: refControlPlaneSG, + CidrIp: gfnt.NewString(cidr), + Description: gfnt.NewString(fmt.Sprintf("Allow Extra CIDR %d (%s) to communicate to controlplane", i, cidr)), + IpProtocol: gfnt.NewString("tcp"), + FromPort: sgPortHTTPS, + ToPort: sgPortHTTPS, + }) + } + } + } else { + refControlPlaneSG = gfnt.NewString(c.spec.VPC.SecurityGroup) + } + c.securityGroups = []*gfnt.Value{refControlPlaneSG} // only this one SG is passed to EKS API, nodes are isolated + + if c.spec.VPC.SharedNodeSecurityGroup == "" { + refClusterSharedNodeSG = c.newResource(cfnSharedNodeSGResource, &gfnec2.SecurityGroup{ + GroupDescription: gfnt.NewString("Communication between all nodes in the cluster"), + VpcId: vpcID, + }) + c.newResource("IngressInterNodeGroupSG", &gfnec2.SecurityGroupIngress{ + GroupId: refClusterSharedNodeSG, + SourceSecurityGroupId: refClusterSharedNodeSG, + Description: gfnt.NewString("Allow nodes to communicate with each other (all ports)"), + IpProtocol: gfnt.NewString("-1"), + FromPort: sgPortZero, + ToPort: sgMaxNodePort, + }) + } else { + refClusterSharedNodeSG = gfnt.NewString(c.spec.VPC.SharedNodeSecurityGroup) + } + + if c.supportsManagedNodes && api.IsEnabled(c.spec.VPC.ManageSharedNodeSecurityGroupRules) { + // To enable communication between both managed and unmanaged nodegroups, this allows ingress traffic from + // the default cluster security group ID that EKS creates by default + // EKS attaches this to Managed Nodegroups by default, but we need to handle this for unmanaged nodegroups + c.newResource(cfnIngressClusterToNodeSGResource, &gfnec2.SecurityGroupIngress{ + GroupId: refClusterSharedNodeSG, + SourceSecurityGroupId: gfnt.MakeFnGetAttString("ControlPlane", outputs.ClusterDefaultSecurityGroup), + Description: gfnt.NewString("Allow managed and unmanaged nodes to communicate with each other (all ports)"), + IpProtocol: gfnt.NewString("-1"), + FromPort: sgPortZero, + ToPort: sgMaxNodePort, + }) + c.newResource("IngressNodeToDefaultClusterSG", &gfnec2.SecurityGroupIngress{ + GroupId: gfnt.MakeFnGetAttString("ControlPlane", outputs.ClusterDefaultSecurityGroup), + SourceSecurityGroupId: refClusterSharedNodeSG, + Description: gfnt.NewString("Allow unmanaged nodes to communicate with control plane (all ports)"), + IpProtocol: gfnt.NewString("-1"), + FromPort: sgPortZero, + ToPort: sgMaxNodePort, + }) + } + + if c.spec.VPC == nil { + c.spec.VPC = &api.ClusterVPC{} + } + c.rs.defineOutput(outputs.ClusterSecurityGroup, refControlPlaneSG, true, func(v string) error { + c.spec.VPC.SecurityGroup = v + return nil + }) + c.rs.defineOutput(outputs.ClusterSharedNodeSecurityGroup, refClusterSharedNodeSG, true, func(v string) error { + c.spec.VPC.SharedNodeSecurityGroup = v + return nil + }) + + return &clusterSecurityGroup{ + ControlPlane: refControlPlaneSG, + ClusterSharedNode: refClusterSharedNodeSG, + } +} + +// TODO move this +func (rs *resourceSet) addEFASecurityGroup(vpcID *gfnt.Value, clusterName, desc string) *gfnt.Value { + efaSG := rs.newResource("EFASG", &gfnec2.SecurityGroup{ + VpcId: vpcID, + GroupDescription: gfnt.NewString("EFA-enabled security group"), + Tags: []gfncfn.Tag{{ + Key: gfnt.NewString("kubernetes.io/cluster/" + clusterName), + Value: gfnt.NewString("owned"), + }}, + }) + rs.newResource("EFAIngressSelf", &gfnec2.SecurityGroupIngress{ + GroupId: efaSG, + SourceSecurityGroupId: efaSG, + Description: gfnt.NewString("Allow " + desc + " to communicate to itself (EFA-enabled)"), + IpProtocol: gfnt.NewString("-1"), + }) + rs.newResource("EFAEgressSelf", &gfnec2.SecurityGroupEgress{ + GroupId: efaSG, + DestinationSecurityGroupId: efaSG, + Description: gfnt.NewString("Allow " + desc + " to communicate to itself (EFA-enabled)"), + IpProtocol: gfnt.NewString("-1"), + }) + + return efaSG +} + +// TODO move this +func (n *NodeGroupResourceSet) addResourcesForSecurityGroups() { + for _, id := range n.spec.SecurityGroups.AttachIDs { + n.securityGroups = append(n.securityGroups, gfnt.NewString(id)) + } + + if api.IsEnabled(n.spec.SecurityGroups.WithShared) { + n.securityGroups = append(n.securityGroups, n.vpcImporter.SharedNodeSecurityGroup()) + } + + if api.IsDisabled(n.spec.SecurityGroups.WithLocal) { + return + } + + desc := "worker nodes in group " + n.spec.Name + vpcID := n.vpcImporter.VPC() + refControlPlaneSG := n.vpcImporter.ControlPlaneSecurityGroup() + + refNodeGroupLocalSG := n.newResource("SG", &gfnec2.SecurityGroup{ + VpcId: vpcID, + GroupDescription: gfnt.NewString("Communication between the control plane and " + desc), + Tags: []gfncfn.Tag{{ + Key: gfnt.NewString("kubernetes.io/cluster/" + n.clusterSpec.Metadata.Name), + Value: gfnt.NewString("owned"), + }}, + SecurityGroupIngress: makeNodeIngressRules(n.spec.NodeGroupBase, refControlPlaneSG, n.clusterSpec.VPC.CIDR.String(), desc), + }) + + n.securityGroups = append(n.securityGroups, refNodeGroupLocalSG) + + if api.IsEnabled(n.spec.EFAEnabled) { + efaSG := n.rs.addEFASecurityGroup(vpcID, n.clusterSpec.Metadata.Name, desc) + n.securityGroups = append(n.securityGroups, efaSG) + } + + n.newResource("EgressInterCluster", &gfnec2.SecurityGroupEgress{ + GroupId: refControlPlaneSG, + DestinationSecurityGroupId: refNodeGroupLocalSG, + Description: gfnt.NewString("Allow control plane to communicate with " + desc + " (kubelet and workload TCP ports)"), + IpProtocol: sgProtoTCP, + FromPort: sgMinNodePort, + ToPort: sgMaxNodePort, + }) + n.newResource("EgressInterClusterAPI", &gfnec2.SecurityGroupEgress{ + GroupId: refControlPlaneSG, + DestinationSecurityGroupId: refNodeGroupLocalSG, + Description: gfnt.NewString("Allow control plane to communicate with " + desc + " (workloads using HTTPS port, commonly used with extension API servers)"), + IpProtocol: sgProtoTCP, + FromPort: sgPortHTTPS, + ToPort: sgPortHTTPS, + }) + n.newResource("IngressInterClusterCP", &gfnec2.SecurityGroupIngress{ + GroupId: refControlPlaneSG, + SourceSecurityGroupId: refNodeGroupLocalSG, + Description: gfnt.NewString("Allow control plane to receive API requests from " + desc), + IpProtocol: sgProtoTCP, + FromPort: sgPortHTTPS, + ToPort: sgPortHTTPS, + }) +} + +func makeNodeIngressRules(ng *api.NodeGroupBase, controlPlaneSG *gfnt.Value, vpcCIDR, description string) []gfnec2.SecurityGroup_Ingress { + ingressRules := []gfnec2.SecurityGroup_Ingress{ + { + SourceSecurityGroupId: controlPlaneSG, + Description: gfnt.NewString(fmt.Sprintf("[IngressInterCluster] Allow %s to communicate with control plane (kubelet and workload TCP ports)", description)), + IpProtocol: sgProtoTCP, + FromPort: sgMinNodePort, + ToPort: sgMaxNodePort, + }, + { + SourceSecurityGroupId: controlPlaneSG, + Description: gfnt.NewString(fmt.Sprintf("[IngressInterClusterAPI] Allow %s to communicate with control plane (workloads using HTTPS port, commonly used with extension API servers)", description)), + IpProtocol: sgProtoTCP, + FromPort: sgPortHTTPS, + ToPort: sgPortHTTPS, + }, + } + + return append(ingressRules, makeSSHIngressRules(ng, vpcCIDR, description)...) +} + +func (v *IPv4VPCResourceSet) haNAT() { + for _, az := range v.clusterConfig.AvailabilityZones { + alphanumericUpperAZ := formatAZ(az) + + // Allocate an EIP + v.rs.newResource("NATIP"+alphanumericUpperAZ, &gfnec2.EIP{ + Domain: gfnt.NewString("vpc"), + }) + // Allocate a NAT gateway in the public subnet + refNG := v.rs.newResource("NATGateway"+alphanumericUpperAZ, &gfnec2.NatGateway{ + AllocationId: gfnt.MakeFnGetAttString("NATIP"+alphanumericUpperAZ, "AllocationId"), + SubnetId: gfnt.MakeRef("SubnetPublic" + alphanumericUpperAZ), + }) + + // Allocate a routing table for the private subnet + refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ + VpcId: v.vpcID, + }) + // Create a route that sends Internet traffic through the NAT gateway + v.rs.newResource("NATPrivateSubnetRoute"+alphanumericUpperAZ, &gfnec2.Route{ + RouteTableId: refRT, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + NatGatewayId: refNG, + }) + // Associate the routing table with the subnet + v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ + SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), + RouteTableId: refRT, + }) + } +} + +func (v *IPv4VPCResourceSet) singleNAT() { + sortedAZs := v.clusterConfig.AvailabilityZones + firstUpperAZ := strings.ToUpper(strings.Join(strings.Split(sortedAZs[0], "-"), "")) + + v.rs.newResource("NATIP", &gfnec2.EIP{ + Domain: gfnt.NewString("vpc"), + }) + refNG := v.rs.newResource("NATGateway", &gfnec2.NatGateway{ + AllocationId: gfnt.MakeFnGetAttString("NATIP", "AllocationId"), + SubnetId: gfnt.MakeRef("SubnetPublic" + firstUpperAZ), + }) + + for _, az := range v.clusterConfig.AvailabilityZones { + alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) + + refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ + VpcId: v.vpcID, + }) + + v.rs.newResource("NATPrivateSubnetRoute"+alphanumericUpperAZ, &gfnec2.Route{ + RouteTableId: refRT, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + NatGatewayId: refNG, + }) + v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ + SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), + RouteTableId: refRT, + }) + } +} + +func (v *IPv4VPCResourceSet) noNAT() { + for _, az := range v.clusterConfig.AvailabilityZones { + alphanumericUpperAZ := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) + + refRT := v.rs.newResource("PrivateRouteTable"+alphanumericUpperAZ, &gfnec2.RouteTable{ + VpcId: v.vpcID, + }) + v.rs.newResource("RouteTableAssociationPrivate"+alphanumericUpperAZ, &gfnec2.SubnetRouteTableAssociation{ + SubnetId: gfnt.MakeRef("SubnetPrivate" + alphanumericUpperAZ), + RouteTableId: refRT, + }) + } +} diff --git a/pkg/cfn/builder/vpc_test.go b/pkg/cfn/builder/vpc_ipv4_test.go similarity index 93% rename from pkg/cfn/builder/vpc_test.go rename to pkg/cfn/builder/vpc_ipv4_test.go index 62835d8fae..8e2b698523 100644 --- a/pkg/cfn/builder/vpc_test.go +++ b/pkg/cfn/builder/vpc_ipv4_test.go @@ -17,7 +17,7 @@ import ( var _ = Describe("VPC Template Builder", func() { var ( - vpcRs *builder.VPCResourceSet + vpcRs *builder.IPv4VPCResourceSet cfg *api.ClusterConfig mockEC2 = &mocks.EC2API{} ) @@ -29,18 +29,19 @@ var _ = Describe("VPC Template Builder", func() { }) JustBeforeEach(func() { - vpcRs = builder.NewVPCResourceSet(builder.NewRS(), cfg, mockEC2) + vpcRs = builder.NewIPv4VPCResourceSet(builder.NewRS(), cfg, mockEC2) }) Describe("AddResources", func() { var ( - addErr error - result *builder.VPCResource - vpcTemplate *fakes.FakeTemplate + addErr error + vpcID *gfnt.Value + subnetDetails *builder.SubnetDetails + vpcTemplate *fakes.FakeTemplate ) JustBeforeEach(func() { - result, addErr = vpcRs.AddResources() + vpcID, subnetDetails, addErr = vpcRs.CreateTemplate() vpcTemplate = &fakes.FakeTemplate{} templateBody, err := vpcRs.RenderJSON() Expect(err).ShouldNot(HaveOccurred()) @@ -52,7 +53,7 @@ var _ = Describe("VPC Template Builder", func() { }) It("returns the VPC resource", func() { - Expect(result.VPC).To(Equal(gfnt.MakeRef(vpcResourceKey))) + Expect(vpcID).To(Equal(gfnt.MakeRef(vpcResourceKey))) }) It("adds the correct gateway resources to the resource set", func() { @@ -73,12 +74,12 @@ var _ = Describe("VPC Template Builder", func() { }) It("returns public subnet settings", func() { - Expect(result.SubnetDetails.Public).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Public).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.MakeRef(publicSubnetRef2), RouteTable: gfnt.MakeRef(pubRouteTable), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Public).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Public).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.MakeRef(publicSubnetRef1), RouteTable: gfnt.MakeRef(pubRouteTable), AvailabilityZone: azA, @@ -119,13 +120,13 @@ var _ = Describe("VPC Template Builder", func() { }) It("returns private subnet settings", func() { - Expect(result.SubnetDetails.Private).To(HaveLen(2)) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(HaveLen(2)) + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.MakeRef(privateSubnetRef2), RouteTable: gfnt.MakeRef(privRouteTableB), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.MakeRef(privateSubnetRef1), RouteTable: gfnt.MakeRef(privRouteTableA), AvailabilityZone: azA, @@ -270,7 +271,7 @@ var _ = Describe("VPC Template Builder", func() { }) It("the correct VPC resource values are loaded into the VPCResource", func() { - Expect(result.VPC).To(Equal(gfnt.NewString("custom-vpc"))) + Expect(vpcID).To(Equal(gfnt.NewString("custom-vpc"))) }) It("no resources are added to the set", func() { @@ -279,12 +280,12 @@ var _ = Describe("VPC Template Builder", func() { Context("private subnets are configured", func() { It("the private subnet resource values are loaded into the VPCResource", func() { - Expect(result.SubnetDetails.Private).To(HaveLen(2)) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(HaveLen(2)) + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(privateSubnet2), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(privateSubnet1), AvailabilityZone: azA, })) @@ -310,13 +311,13 @@ var _ = Describe("VPC Template Builder", func() { }) It("the private subnet resource values are loaded into the VPCResource with route table association", func() { - Expect(result.SubnetDetails.Private).To(HaveLen(2)) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(HaveLen(2)) + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(privateSubnet2), RouteTable: gfnt.NewString("this-is-a-route-table"), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Private).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Private).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(privateSubnet1), RouteTable: gfnt.NewString("this-is-a-route-table"), AvailabilityZone: azA, @@ -354,12 +355,12 @@ var _ = Describe("VPC Template Builder", func() { Context("public subnets are configured", func() { It("the public subnet resource values are loaded into the VPCResource", func() { - Expect(result.SubnetDetails.Public).To(HaveLen(2)) - Expect(result.SubnetDetails.Public).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Public).To(HaveLen(2)) + Expect(subnetDetails.Public).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(publicSubnet2), AvailabilityZone: azB, })) - Expect(result.SubnetDetails.Public).To(ContainElement(builder.SubnetResource{ + Expect(subnetDetails.Public).To(ContainElement(builder.SubnetResource{ Subnet: gfnt.NewString(publicSubnet1), AvailabilityZone: azA, })) @@ -372,7 +373,7 @@ var _ = Describe("VPC Template Builder", func() { BeforeEach(func() { autoAllocated := true cfg.VPC.AutoAllocateIPv6 = &autoAllocated - expectedFnCIDR = `{ "Fn::Cidr": [{ "Fn::Select": [ 0, { "Fn::GetAtt": ["VPC", "Ipv6CidrBlocks"] }]}, 8, 64 ]}` + expectedFnCIDR = `{ "Fn::Cidr": [{ "Fn::Select": [ 0, { "Fn::GetAtt": ["VPC", "Ipv6CidrBlocks"] }]}, 6, 64 ]}` }) It("adds the AutoAllocatedCIDRv6 vpc resource to the resource set", func() { @@ -429,7 +430,7 @@ var _ = Describe("VPC Template Builder", func() { }) It("does not set public subnet resources", func() { - Expect(result.SubnetDetails.Public).To(HaveLen(0)) + Expect(subnetDetails.Public).To(HaveLen(0)) Expect(vpcTemplate.Resources).ToNot(HaveKey(pubSubnetRoute)) Expect(vpcTemplate.Resources).ToNot(HaveKey(pubSubnetRoute)) Expect(vpcTemplate.Resources).ToNot(HaveKey(publicSubnetRef1)) @@ -437,7 +438,7 @@ var _ = Describe("VPC Template Builder", func() { Expect(vpcTemplate.Resources).ToNot(HaveKey(rtaPublicA)) Expect(vpcTemplate.Resources).ToNot(HaveKey(rtaPublicB)) - Expect(result.SubnetDetails.Private).To(HaveLen(2)) + Expect(subnetDetails.Private).To(HaveLen(2)) Expect(vpcTemplate.Resources).To(HaveKey(privRouteTableA)) Expect(vpcTemplate.Resources).To(HaveKey(privRouteTableB)) Expect(vpcTemplate.Resources).To(HaveKey(privateSubnetRef1)) @@ -461,9 +462,8 @@ var _ = Describe("VPC Template Builder", func() { ) JustBeforeEach(func() { - _, err := vpcRs.AddResources() + _, _, err := vpcRs.CreateTemplate() Expect(err).NotTo(HaveOccurred()) - vpcRs.AddOutputs() vpcTemplate = &fakes.FakeTemplate{} templateBody, err := vpcRs.RenderJSON() Expect(err).ShouldNot(HaveOccurred()) @@ -508,9 +508,9 @@ var _ = Describe("VPC Template Builder", func() { Describe("PublicSubnetRefs", func() { It("returns the references of public subnets", func() { - result, err := vpcRs.AddResources() + _, subnetDetails, err := vpcRs.CreateTemplate() Expect(err).NotTo(HaveOccurred()) - refs := result.SubnetDetails.PublicSubnetRefs() + refs := subnetDetails.PublicSubnetRefs() Expect(refs).To(HaveLen(2)) Expect(refs).To(ContainElement(makePrimitive(publicSubnetRef1))) Expect(refs).To(ContainElement(makePrimitive(publicSubnetRef2))) @@ -519,9 +519,9 @@ var _ = Describe("VPC Template Builder", func() { Describe("PrivateSubnetRefs", func() { It("returns the references of private subnets", func() { - result, err := vpcRs.AddResources() + _, subnetDetails, err := vpcRs.CreateTemplate() Expect(err).NotTo(HaveOccurred()) - refs := result.SubnetDetails.PrivateSubnetRefs() + refs := subnetDetails.PrivateSubnetRefs() Expect(refs).To(HaveLen(2)) Expect(refs).To(ContainElement(makePrimitive(privateSubnetRef1))) Expect(refs).To(ContainElement(makePrimitive(privateSubnetRef2))) diff --git a/pkg/cfn/builder/vpc_ipv6.go b/pkg/cfn/builder/vpc_ipv6.go new file mode 100644 index 0000000000..3d5245cd31 --- /dev/null +++ b/pkg/cfn/builder/vpc_ipv6.go @@ -0,0 +1,180 @@ +package builder + +import ( + "strings" + + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/outputs" + "github.com/weaveworks/eksctl/pkg/vpc" + "github.com/weaveworks/goformation/v4/cloudformation/cloudformation" + gfnec2 "github.com/weaveworks/goformation/v4/cloudformation/ec2" + gfnt "github.com/weaveworks/goformation/v4/cloudformation/types" +) + +// A IPv6VPCResourceSet builds the resources required for the specified VPC +type IPv6VPCResourceSet struct { + rs *resourceSet + clusterConfig *api.ClusterConfig + ec2API ec2iface.EC2API +} + +// NewIPv6VPCResourceSet creates and returns a new VPCResourceSet +func NewIPv6VPCResourceSet(rs *resourceSet, clusterConfig *api.ClusterConfig, ec2API ec2iface.EC2API) *IPv6VPCResourceSet { + return &IPv6VPCResourceSet{ + rs: rs, + clusterConfig: clusterConfig, + ec2API: ec2API, + } +} + +func (v *IPv6VPCResourceSet) CreateTemplate() (*gfnt.Value, *SubnetDetails, error) { + var publicSubnetResourceRefs, privateSubnetResourceRefs []*gfnt.Value + vpcResourceRef := v.rs.newResource(VPCResourceKey, &gfnec2.VPC{ + CidrBlock: gfnt.NewString(v.clusterConfig.VPC.CIDR.String()), + EnableDnsSupport: gfnt.True(), + EnableDnsHostnames: gfnt.True(), + }) + + v.rs.newResource(IPv6CIDRBlockKey, &gfnec2.VPCCidrBlock{ + AmazonProvidedIpv6CidrBlock: gfnt.True(), + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + refIGW := v.rs.newResource(IGWKey, &gfnec2.InternetGateway{}) + + v.rs.newResource(GAKey, &gfnec2.VPCGatewayAttachment{ + InternetGatewayId: gfnt.MakeRef(IGWKey), + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + v.rs.newResource(EgressOnlyInternetGatewayKey, &gfnec2.EgressOnlyInternetGateway{ + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + firstPublicSubnet := PublicSubnetKey + formatAZ(v.clusterConfig.AvailabilityZones[0]) + v.rs.newResource(NATGatewayKey, &gfnec2.NatGateway{ + AWSCloudFormationDependsOn: []string{ + ElasticIPKey, + firstPublicSubnet, + GAKey, + }, + AllocationId: gfnt.MakeFnGetAtt(ElasticIPKey, gfnt.NewString("AllocationId")), + SubnetId: gfnt.MakeRef(firstPublicSubnet), + }) + + v.rs.newResource(ElasticIPKey, &gfnec2.EIP{ + Domain: gfnt.NewString("vpc"), + AWSCloudFormationDependsOn: []string{GAKey}, + }) + + v.rs.newResource(PubRouteTableKey, &gfnec2.RouteTable{ + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + v.rs.newResource(PubSubRouteKey, &gfnec2.Route{ + AWSCloudFormationDependsOn: []string{GAKey}, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + GatewayId: refIGW, + RouteTableId: gfnt.MakeRef(PubRouteTableKey), + }) + + v.rs.newResource(PubSubIPv6RouteKey, &gfnec2.Route{ + AWSCloudFormationDependsOn: []string{GAKey}, + DestinationIpv6CidrBlock: gfnt.NewString(InternetIPv6CIDR), + GatewayId: refIGW, + RouteTableId: gfnt.MakeRef(PubRouteTableKey), + }) + + cidrPartitions := (len(v.clusterConfig.AvailabilityZones) * 2) + 2 + for i, az := range v.clusterConfig.AvailabilityZones { + azFormatted := formatAZ(az) + v.rs.newResource(PrivateRouteTableKey+azFormatted, &gfnec2.RouteTable{ + VpcId: gfnt.MakeRef(VPCResourceKey), + }) + + v.rs.newResource(PrivateSubnetIpv6RouteKey+azFormatted, &gfnec2.Route{ + DestinationIpv6CidrBlock: gfnt.NewString(InternetIPv6CIDR), + EgressOnlyInternetGatewayId: gfnt.MakeRef(EgressOnlyInternetGatewayKey), + RouteTableId: gfnt.MakeRef(PrivateRouteTableKey + azFormatted), + }) + + v.rs.newResource(PrivateSubnetRouteKey+azFormatted, &gfnec2.Route{ + AWSCloudFormationDependsOn: []string{NATGatewayKey, GAKey}, + DestinationCidrBlock: gfnt.NewString(InternetCIDR), + NatGatewayId: gfnt.MakeRef(NATGatewayKey), + RouteTableId: gfnt.MakeRef(PrivateRouteTableKey + azFormatted), + }) + + publicSubnetResourceRefs = append(publicSubnetResourceRefs, v.createSubnet(az, azFormatted, i, cidrPartitions, false)) + privateSubnetResourceRefs = append(privateSubnetResourceRefs, v.createSubnet(az, azFormatted, i+len(v.clusterConfig.AvailabilityZones), cidrPartitions, true)) + + v.rs.newResource(PubRouteTableAssociation+azFormatted, &gfnec2.SubnetRouteTableAssociation{ + RouteTableId: gfnt.MakeRef(PubRouteTableKey), + SubnetId: gfnt.MakeRef(PublicSubnetKey + azFormatted), + }) + + v.rs.newResource(PrivateRouteTableAssociation+azFormatted, &gfnec2.SubnetRouteTableAssociation{ + RouteTableId: gfnt.MakeRef(PrivateRouteTableKey + azFormatted), + SubnetId: gfnt.MakeRef(PrivateSubnetKey + azFormatted), + }) + } + + v.rs.defineOutput(outputs.ClusterVPC, vpcResourceRef, true, func(val string) error { + v.clusterConfig.VPC.ID = val + return nil + }) + + addSubnetOutput := func(subnetRefs []*gfnt.Value, topology api.SubnetTopology, outputName string) { + v.rs.defineJoinedOutput(outputName, subnetRefs, true, func(value string) error { + return vpc.ImportSubnetsFromIDList(v.ec2API, v.clusterConfig, topology, strings.Split(value, ",")) + }) + } + + addSubnetOutput(publicSubnetResourceRefs, api.SubnetTopologyPublic, outputs.ClusterSubnetsPublic) + addSubnetOutput(privateSubnetResourceRefs, api.SubnetTopologyPrivate, outputs.ClusterSubnetsPrivate) + + var publicSubnets, privateSubnets []SubnetResource + for _, s := range publicSubnetResourceRefs { + publicSubnets = append(publicSubnets, SubnetResource{Subnet: s}) + } + for _, s := range privateSubnetResourceRefs { + privateSubnets = append(privateSubnets, SubnetResource{Subnet: s}) + } + return vpcResourceRef, &SubnetDetails{ + Private: privateSubnets, + Public: publicSubnets, + }, nil +} + +func (v *IPv6VPCResourceSet) RenderJSON() ([]byte, error) { + return v.rs.renderJSON() +} + +func (v *IPv6VPCResourceSet) createSubnet(az, azFormatted string, i, cidrPartitions int, private bool) *gfnt.Value { + var assignIpv6AddressOnCreation *gfnt.Value + subnetKey := PublicSubnetKey + azFormatted + mapPublicIPOnLaunch := gfnt.True() + elbTagKey := "kubernetes.io/role/elb" + + if private { + subnetKey = PrivateSubnetKey + azFormatted + mapPublicIPOnLaunch = nil + assignIpv6AddressOnCreation = gfnt.True() + elbTagKey = "kubernetes.io/role/internal-elb" + } + + return v.rs.newResource(subnetKey, &gfnec2.Subnet{ + AWSCloudFormationDependsOn: []string{IPv6CIDRBlockKey}, + AvailabilityZone: gfnt.NewString(az), + CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(i), getSubnetIPv4CIDRBlock(cidrPartitions)), + Ipv6CidrBlock: gfnt.MakeFnSelect(gfnt.NewInteger(i), getSubnetIPv6CIDRBlock(cidrPartitions)), + MapPublicIpOnLaunch: mapPublicIPOnLaunch, + AssignIpv6AddressOnCreation: assignIpv6AddressOnCreation, + VpcId: gfnt.MakeRef(VPCResourceKey), + Tags: []cloudformation.Tag{{ + Key: gfnt.NewString(elbTagKey), + Value: gfnt.NewString("1"), + }}, + }) +} diff --git a/pkg/cfn/builder/vpc_ipv6_test.go b/pkg/cfn/builder/vpc_ipv6_test.go new file mode 100644 index 0000000000..be26bfa518 --- /dev/null +++ b/pkg/cfn/builder/vpc_ipv6_test.go @@ -0,0 +1,385 @@ +package builder_test + +import ( + "encoding/json" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/builder" + "github.com/weaveworks/eksctl/pkg/cfn/builder/fakes" + "github.com/weaveworks/eksctl/pkg/cfn/outputs" +) + +var _ = Describe("IPv6 VPC builder", func() { + var ( + vpcTemplate *fakes.FakeTemplate + vpcRs *builder.IPv6VPCResourceSet + cfg *api.ClusterConfig + ) + + BeforeEach(func() { + cfg = api.NewClusterConfig() + cfg.VPC.IPFamily = aws.String("ipv6") + }) + + It("creates the ipv6 VPC and its resources", func() { + cfg.AvailabilityZones = []string{azA, azB} + vpcRs = builder.NewIPv6VPCResourceSet(builder.NewRS(), cfg, nil) + + _, subnetDetails, err := vpcRs.CreateTemplate() + Expect(err).NotTo(HaveOccurred()) + + By("returning the references of public subnets") + pubRefs := subnetDetails.PublicSubnetRefs() + Expect(pubRefs).To(HaveLen(2)) + Expect(pubRefs).To(ContainElement(makePrimitive(builder.PublicSubnetKey + azAFormatted))) + Expect(pubRefs).To(ContainElement(makePrimitive(builder.PublicSubnetKey + azBFormatted))) + + By("returning the references of private subnets") + privRef := subnetDetails.PrivateSubnetRefs() + Expect(privRef).To(HaveLen(2)) + Expect(privRef).To(ContainElement(makePrimitive(builder.PrivateSubnetKey + azBFormatted))) + Expect(privRef).To(ContainElement(makePrimitive(builder.PrivateSubnetKey + azBFormatted))) + + vpcTemplate = &fakes.FakeTemplate{} + templateBody, err := vpcRs.RenderJSON() + Expect(err).ShouldNot(HaveOccurred()) + Expect(json.Unmarshal(templateBody, vpcTemplate)).To(Succeed()) + + By("creating the VPC resource") + Expect(vpcTemplate.Resources).To(HaveKey(builder.VPCResourceKey)) + Expect(vpcTemplate.Resources[builder.VPCResourceKey].Type).To(Equal("AWS::EC2::VPC")) + defaultCidr := api.DefaultCIDR() + cidr := &defaultCidr + Expect(vpcTemplate.Resources[builder.VPCResourceKey].Properties).To(Equal(fakes.Properties{ + CidrBlock: cidr.String(), + EnableDNSHostnames: true, + EnableDNSSupport: true, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": "${AWS::StackName}/VPC"}, + }, + }, + })) + + By("creating the IPv6 CIDR") + Expect(vpcTemplate.Resources).To(HaveKey(builder.IPv6CIDRBlockKey)) + Expect(vpcTemplate.Resources[builder.IPv6CIDRBlockKey].Type).To(Equal("AWS::EC2::VPCCidrBlock")) + Expect(vpcTemplate.Resources[builder.IPv6CIDRBlockKey].Properties).To(Equal(fakes.Properties{ + AmazonProvidedIpv6CidrBlock: true, + VpcID: map[string]interface{}{"Ref": "VPC"}, + })) + + By("creating the internet gateway") + Expect(vpcTemplate.Resources).To(HaveKey(builder.IGWKey)) + Expect(vpcTemplate.Resources[builder.IGWKey].Type).To(Equal("AWS::EC2::InternetGateway")) + Expect(vpcTemplate.Resources[builder.IGWKey].Properties).To(Equal(fakes.Properties{ + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": "${AWS::StackName}/InternetGateway"}, + }, + }, + })) + + By("creating a VPC gateway attachment to associate the IGW with the VPC") + Expect(vpcTemplate.Resources).To(HaveKey(builder.GAKey)) + Expect(vpcTemplate.Resources[builder.GAKey].Type).To(Equal("AWS::EC2::VPCGatewayAttachment")) + Expect(vpcTemplate.Resources[builder.GAKey].Properties).To(Equal(fakes.Properties{ + InternetGatewayID: map[string]interface{}{"Ref": "InternetGateway"}, + VpcID: map[string]interface{}{"Ref": "VPC"}, + })) + + By("creating a VPC gateway attachment to associate the IGW with the VPC") + Expect(vpcTemplate.Resources).To(HaveKey(builder.EgressOnlyInternetGatewayKey)) + Expect(vpcTemplate.Resources[builder.EgressOnlyInternetGatewayKey].Type).To(Equal("AWS::EC2::EgressOnlyInternetGateway")) + Expect(vpcTemplate.Resources[builder.EgressOnlyInternetGatewayKey].Properties).To(Equal(fakes.Properties{ + VpcID: map[string]interface{}{"Ref": "VPC"}, + })) + + By("creating the NAT gateway") + Expect(vpcTemplate.Resources).To(HaveKey(builder.NATGatewayKey)) + Expect(vpcTemplate.Resources[builder.NATGatewayKey].Type).To(Equal("AWS::EC2::NatGateway")) + Expect(vpcTemplate.Resources[builder.NATGatewayKey].DependsOn).To(ConsistOf(builder.ElasticIPKey, builder.PublicSubnetKey+azAFormatted, builder.GAKey)) + Expect(vpcTemplate.Resources[builder.NATGatewayKey].Properties).To(Equal(fakes.Properties{ + AllocationID: map[string]interface{}{ + "Fn::GetAtt": []interface{}{ + builder.ElasticIPKey, + "AllocationId", + }, + }, + SubnetID: map[string]interface{}{"Ref": builder.PublicSubnetKey + azAFormatted}, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", builder.NATGatewayKey)}, + }, + }, + })) + + By("creating an Elastic IP for the Nat Gateway") + Expect(vpcTemplate.Resources).To(HaveKey(builder.ElasticIPKey)) + Expect(vpcTemplate.Resources[builder.ElasticIPKey].Type).To(Equal("AWS::EC2::EIP")) + Expect(vpcTemplate.Resources[builder.ElasticIPKey].DependsOn).To(ConsistOf(gaKey)) + Expect(vpcTemplate.Resources[builder.ElasticIPKey].Properties).To(Equal(fakes.Properties{ + Domain: "vpc", + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", builder.ElasticIPKey)}, + }, + }, + })) + + By("creating a public Route Table") + Expect(vpcTemplate.Resources).To(HaveKey(builder.PubRouteTableKey)) + Expect(vpcTemplate.Resources[builder.PubRouteTableKey].Type).To(Equal("AWS::EC2::RouteTable")) + Expect(vpcTemplate.Resources[builder.PubRouteTableKey].Properties).To(Equal(fakes.Properties{ + VpcID: map[string]interface{}{"Ref": "VPC"}, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", builder.PubRouteTableKey)}, + }, + }, + })) + + By("creating public subnet route for IPv4 traffic to IPv4 CIDR") + Expect(vpcTemplate.Resources).To(HaveKey(builder.PubSubRouteKey)) + Expect(vpcTemplate.Resources[builder.PubSubRouteKey].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[builder.PubSubRouteKey].DependsOn).To(ConsistOf(builder.GAKey)) + Expect(vpcTemplate.Resources[builder.PubSubRouteKey].Properties).To(Equal(fakes.Properties{ + DestinationCidrBlock: builder.InternetCIDR, + GatewayID: map[string]interface{}{"Ref": builder.IGWKey}, + RouteTableID: map[string]interface{}{"Ref": builder.PubRouteTableKey}, + })) + + By("creating public subnet route for IPv6 traffic to IPv6 CIDR") + Expect(vpcTemplate.Resources).To(HaveKey(builder.PubSubIPv6RouteKey)) + Expect(vpcTemplate.Resources[builder.PubSubIPv6RouteKey].Type).To(Equal("AWS::EC2::Route")) + //TODO: we added this, wasn't in the example template. We think its correct? + Expect(vpcTemplate.Resources[builder.PubSubIPv6RouteKey].DependsOn).To(ConsistOf(builder.GAKey)) + Expect(vpcTemplate.Resources[builder.PubSubIPv6RouteKey].Properties).To(Equal(fakes.Properties{ + DestinationIpv6CidrBlock: builder.InternetIPv6CIDR, + GatewayID: map[string]interface{}{"Ref": builder.IGWKey}, + RouteTableID: map[string]interface{}{"Ref": builder.PubRouteTableKey}, + })) + + By("creating a private route table for each AZ") + privateRouteTableA := builder.PrivateRouteTableKey + azAFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteTableA)) + Expect(vpcTemplate.Resources[privateRouteTableA].Type).To(Equal("AWS::EC2::RouteTable")) + Expect(vpcTemplate.Resources[privateRouteTableA].Properties).To(Equal(fakes.Properties{ + VpcID: map[string]interface{}{"Ref": "VPC"}, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", privateRouteTableA)}, + }, + }, + })) + privateRouteTableB := builder.PrivateRouteTableKey + azBFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteTableB)) + Expect(vpcTemplate.Resources[privateRouteTableB].Type).To(Equal("AWS::EC2::RouteTable")) + Expect(vpcTemplate.Resources[privateRouteTableB].Properties).To(Equal(fakes.Properties{ + VpcID: map[string]interface{}{"Ref": "VPC"}, + Tags: []fakes.Tag{ + { + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", privateRouteTableB)}, + }, + }, + })) + + By("creating a route to the NAT gateway for each private subnet in the AZs") + privateRouteA := builder.PrivateSubnetRouteKey + azAFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteA)) + Expect(vpcTemplate.Resources[privateRouteA].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[privateRouteA].DependsOn).To(ConsistOf(builder.NATGatewayKey, builder.GAKey)) + Expect(vpcTemplate.Resources[privateRouteA].Properties).To(Equal(fakes.Properties{ + DestinationCidrBlock: builder.InternetCIDR, + NatGatewayID: map[string]interface{}{"Ref": builder.NATGatewayKey}, + RouteTableID: map[string]interface{}{"Ref": privateRouteTableA}, + })) + + privateRouteB := builder.PrivateSubnetRouteKey + azBFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteB)) + Expect(vpcTemplate.Resources[privateRouteB].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[privateRouteB].DependsOn).To(ConsistOf(builder.NATGatewayKey, builder.GAKey)) + Expect(vpcTemplate.Resources[privateRouteB].Properties).To(Equal(fakes.Properties{ + DestinationCidrBlock: builder.InternetCIDR, + NatGatewayID: map[string]interface{}{"Ref": builder.NATGatewayKey}, + RouteTableID: map[string]interface{}{"Ref": privateRouteTableB}, + })) + + By("creating a ipv6 route to the ingress only internet gateway for each private subnet in the AZs") + privateRouteA = builder.PrivateSubnetIpv6RouteKey + azAFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteA)) + Expect(vpcTemplate.Resources[privateRouteA].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[privateRouteA].Properties).To(Equal(fakes.Properties{ + DestinationIpv6CidrBlock: builder.InternetIPv6CIDR, + EgressOnlyInternetGatewayID: map[string]interface{}{"Ref": builder.EgressOnlyInternetGatewayKey}, + RouteTableID: map[string]interface{}{"Ref": privateRouteTableA}, + })) + privateRouteB = builder.PrivateSubnetIpv6RouteKey + azBFormatted + Expect(vpcTemplate.Resources).To(HaveKey(privateRouteB)) + Expect(vpcTemplate.Resources[privateRouteB].Type).To(Equal("AWS::EC2::Route")) + Expect(vpcTemplate.Resources[privateRouteB].Properties).To(Equal(fakes.Properties{ + DestinationIpv6CidrBlock: builder.InternetIPv6CIDR, + EgressOnlyInternetGatewayID: map[string]interface{}{"Ref": builder.EgressOnlyInternetGatewayKey}, + RouteTableID: map[string]interface{}{"Ref": privateRouteTableB}, + })) + + By("creating a public and private subnet for each AZ") + assertSubnetSet := func(az, subnetKey, kubernetesTag string, cidrBlockIndex float64, mapPublicIpOnLaunch bool) { + Expect(vpcTemplate.Resources).To(HaveKey(subnetKey)) + Expect(vpcTemplate.Resources[subnetKey].Type).To(Equal("AWS::EC2::Subnet")) + Expect(vpcTemplate.Resources[subnetKey].DependsOn).To(ConsistOf(builder.IPv6CIDRBlockKey)) + Expect(vpcTemplate.Resources[subnetKey].Properties.AvailabilityZone).To(Equal(az)) + Expect(vpcTemplate.Resources[subnetKey].Properties.MapPublicIPOnLaunch).To(Equal(mapPublicIpOnLaunch)) + + Expect(vpcTemplate.Resources[subnetKey].Properties.VpcID).To(Equal(map[string]interface{}{"Ref": "VPC"})) + Expect(vpcTemplate.Resources[subnetKey].Properties.Tags).To(ConsistOf( + fakes.Tag{ + Key: kubernetesTag, + Value: "1", + }, + fakes.Tag{ + Key: "Name", + Value: map[string]interface{}{"Fn::Sub": fmt.Sprintf("${AWS::StackName}/%s", subnetKey)}, + }, + )) + + expectedFnIPv4CIDR := `{ "Fn::Cidr": [{ "Fn::GetAtt": ["VPC", "CidrBlock"]}, 6, 13 ]}` + Expect(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"]).To(HaveLen(2)) + Expect(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"].([]interface{})[0].(float64)).To(Equal(cidrBlockIndex)) + actualFnCIDR, err := json.Marshal(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"].([]interface{})[1]) + Expect(err).NotTo(HaveOccurred()) + Expect(actualFnCIDR).To(MatchJSON([]byte(expectedFnIPv4CIDR))) + + expectedFnIPv6CIDR := `{ "Fn::Cidr": [{ "Fn::Select": [ 0, { "Fn::GetAtt": ["VPC", "Ipv6CidrBlocks"] }]}, 6, 64 ]}` + Expect(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"]).To(HaveLen(2)) + Expect(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"][0].(float64)).To(Equal(cidrBlockIndex)) + actualFnIPv6CIDR, err := json.Marshal(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"][1]) + Expect(err).NotTo(HaveOccurred()) + Expect(actualFnIPv6CIDR).To(MatchJSON([]byte(expectedFnIPv6CIDR))) + } + assertSubnetSet(azA, builder.PublicSubnetKey+azAFormatted, "kubernetes.io/role/elb", float64(0), true) + Expect(vpcTemplate.Resources[builder.PublicSubnetKey+azAFormatted].Properties.AssignIpv6AddressOnCreation).To(BeNil()) + assertSubnetSet(azB, builder.PublicSubnetKey+azBFormatted, "kubernetes.io/role/elb", float64(1), true) + Expect(vpcTemplate.Resources[builder.PublicSubnetKey+azBFormatted].Properties.AssignIpv6AddressOnCreation).To(BeNil()) + + assertSubnetSet(azA, builder.PrivateSubnetKey+azAFormatted, "kubernetes.io/role/internal-elb", float64(2), false) + Expect(*vpcTemplate.Resources[builder.PrivateSubnetKey+azAFormatted].Properties.AssignIpv6AddressOnCreation).To(Equal(true)) + assertSubnetSet(azB, builder.PrivateSubnetKey+azBFormatted, "kubernetes.io/role/internal-elb", float64(3), false) + Expect(*vpcTemplate.Resources[builder.PrivateSubnetKey+azAFormatted].Properties.AssignIpv6AddressOnCreation).To(Equal(true)) + + By("creating route table associations", func() { + assertSubnetRouteTableAssociation := func(routeTableAssociationKey, subnetKey, routeTableKey string) { + Expect(vpcTemplate.Resources).To(HaveKey(routeTableAssociationKey)) + Expect(vpcTemplate.Resources[routeTableAssociationKey].Type).To(Equal("AWS::EC2::SubnetRouteTableAssociation")) + Expect(vpcTemplate.Resources[routeTableAssociationKey].Properties).To(Equal(fakes.Properties{ + RouteTableID: map[string]interface{}{"Ref": routeTableKey}, + SubnetID: map[string]interface{}{"Ref": subnetKey}, + })) + } + + By("associating all public subnets with the public route table", func() { + assertSubnetRouteTableAssociation(builder.PubRouteTableAssociation+azAFormatted, builder.PublicSubnetKey+azAFormatted, builder.PubRouteTableKey) + assertSubnetRouteTableAssociation(builder.PubRouteTableAssociation+azBFormatted, builder.PublicSubnetKey+azBFormatted, builder.PubRouteTableKey) + }) + + By("associating each private subnet with its private route table", func() { + assertSubnetRouteTableAssociation(builder.PrivateRouteTableAssociation+azAFormatted, builder.PrivateSubnetKey+azAFormatted, builder.PrivateRouteTableKey+azAFormatted) + assertSubnetRouteTableAssociation(builder.PrivateRouteTableAssociation+azBFormatted, builder.PrivateSubnetKey+azBFormatted, builder.PrivateRouteTableKey+azBFormatted) + }) + }) + + By("outputting the VPC on the stack") + Expect(vpcTemplate.Outputs).To(HaveKey(builder.VPCResourceKey)) + Expect(vpcTemplate.Outputs.(map[string]interface{})[builder.VPCResourceKey].(map[string]interface{})["Value"]).To(Equal(map[string]interface{}{"Ref": builder.VPCResourceKey})) + Expect(vpcTemplate.Outputs.(map[string]interface{})[builder.VPCResourceKey].(map[string]interface{})["Export"]).To(Equal(map[string]interface{}{ + "Name": map[string]interface{}{ + "Fn::Sub": fmt.Sprintf("${AWS::StackName}::%s", builder.VPCResourceKey), + }, + })) + + By("outputting the public subnets on the stack") + Expect(vpcTemplate.Outputs).To(HaveKey(outputs.ClusterSubnetsPublic)) + Expect(vpcTemplate.Outputs.(map[string]interface{})[outputs.ClusterSubnetsPublic].(map[string]interface{})["Value"]).To(Equal(map[string]interface{}{ + "Fn::Join": []interface{}{ + ",", + []interface{}{ + map[string]interface{}{"Ref": builder.PublicSubnetKey + azAFormatted}, + map[string]interface{}{"Ref": builder.PublicSubnetKey + azBFormatted}, + }, + }, + })) + Expect(vpcTemplate.Outputs.(map[string]interface{})[outputs.ClusterSubnetsPublic].(map[string]interface{})["Export"]).To(Equal(map[string]interface{}{ + "Name": map[string]interface{}{ + "Fn::Sub": fmt.Sprintf("${AWS::StackName}::%s", outputs.ClusterSubnetsPublic), + }, + })) + + By("outputting the private subnets on the stack") + Expect(vpcTemplate.Outputs).To(HaveKey(outputs.ClusterSubnetsPrivate)) + Expect(vpcTemplate.Outputs.(map[string]interface{})[outputs.ClusterSubnetsPrivate].(map[string]interface{})["Value"]).To(Equal(map[string]interface{}{ + "Fn::Join": []interface{}{ + ",", + []interface{}{ + map[string]interface{}{"Ref": builder.PrivateSubnetKey + azAFormatted}, + map[string]interface{}{"Ref": builder.PrivateSubnetKey + azBFormatted}, + }, + }, + })) + Expect(vpcTemplate.Outputs.(map[string]interface{})[outputs.ClusterSubnetsPrivate].(map[string]interface{})["Export"]).To(Equal(map[string]interface{}{ + "Name": map[string]interface{}{ + "Fn::Sub": fmt.Sprintf("${AWS::StackName}::%s", outputs.ClusterSubnetsPrivate), + }, + })) + }) + + Context("when there are 3 AZs", func() { + It("scales the CIDR blocks accordingly", func() { + cfg.AvailabilityZones = []string{azA, azB, azC} + vpcRs = builder.NewIPv6VPCResourceSet(builder.NewRS(), cfg, nil) + + _, _, err := vpcRs.CreateTemplate() + Expect(err).NotTo(HaveOccurred()) + + vpcTemplate = &fakes.FakeTemplate{} + templateBody, err := vpcRs.RenderJSON() + Expect(err).ShouldNot(HaveOccurred()) + Expect(json.Unmarshal(templateBody, vpcTemplate)).To(Succeed()) + + assertSubnetSet := func(az, subnetKey string, cidrBlockIndex float64) { + Expect(vpcTemplate.Resources).To(HaveKey(subnetKey)) + expectedFnIPv4CIDR := `{ "Fn::Cidr": [{ "Fn::GetAtt": ["VPC", "CidrBlock"]}, 8, 13 ]}` + Expect(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"]).To(HaveLen(2)) + Expect(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"].([]interface{})[0].(float64)).To(Equal(cidrBlockIndex)) + actualFnCIDR, err := json.Marshal(vpcTemplate.Resources[subnetKey].Properties.CidrBlock.(map[string]interface{})["Fn::Select"].([]interface{})[1]) + Expect(err).NotTo(HaveOccurred()) + Expect(actualFnCIDR).To(MatchJSON([]byte(expectedFnIPv4CIDR))) + + expectedFnIPv6CIDR := `{ "Fn::Cidr": [{ "Fn::Select": [ 0, { "Fn::GetAtt": ["VPC", "Ipv6CidrBlocks"] }]}, 8, 64 ]}` + Expect(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"]).To(HaveLen(2)) + Expect(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"][0].(float64)).To(Equal(cidrBlockIndex)) + actualFnIPv6CIDR, err := json.Marshal(vpcTemplate.Resources[subnetKey].Properties.Ipv6CidrBlock["Fn::Select"][1]) + Expect(err).NotTo(HaveOccurred()) + Expect(actualFnIPv6CIDR).To(MatchJSON([]byte(expectedFnIPv6CIDR))) + } + assertSubnetSet(azA, builder.PublicSubnetKey+azAFormatted, float64(0)) + assertSubnetSet(azB, builder.PublicSubnetKey+azBFormatted, float64(1)) + assertSubnetSet(azC, builder.PublicSubnetKey+azCFormatted, float64(2)) + + assertSubnetSet(azA, builder.PrivateSubnetKey+azAFormatted, float64(3)) + assertSubnetSet(azB, builder.PrivateSubnetKey+azBFormatted, float64(4)) + assertSubnetSet(azC, builder.PrivateSubnetKey+azCFormatted, float64(5)) + + }) + }) +}) diff --git a/pkg/cfn/manager/create_tasks.go b/pkg/cfn/manager/create_tasks.go index a038e31aea..36eb6fa411 100644 --- a/pkg/cfn/manager/create_tasks.go +++ b/pkg/cfn/manager/create_tasks.go @@ -8,6 +8,7 @@ import ( api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" iamoidc "github.com/weaveworks/eksctl/pkg/iam/oidc" "github.com/weaveworks/eksctl/pkg/kubernetes" + utilsstrings "github.com/weaveworks/eksctl/pkg/utils/strings" "github.com/weaveworks/eksctl/pkg/utils/tasks" "github.com/weaveworks/eksctl/pkg/vpc" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +34,15 @@ func (c *StackCollection) NewTasksToCreateClusterWithNodeGroups(nodeGroups []*ap }, ) + if utilsstrings.Value(c.spec.VPC.IPFamily) == string(api.IPV6Family) { + taskTree.Append( + &AssignIpv6AddressOnCreationTask{ + ClusterConfig: c.spec, + EC2API: c.ec2API, + }, + ) + } + appendNodeGroupTasksTo := func(taskTree *tasks.TaskTree) { vpcImporter := vpc.NewStackConfigImporter(c.MakeClusterStackName()) nodeGroupTasks := c.NewUnmanagedNodeGroupTask(nodeGroups, false, vpcImporter) diff --git a/pkg/cfn/manager/create_tasks_test.go b/pkg/cfn/manager/create_tasks_test.go new file mode 100644 index 0000000000..7a4886b30f --- /dev/null +++ b/pkg/cfn/manager/create_tasks_test.go @@ -0,0 +1,80 @@ +package manager_test + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" + "github.com/weaveworks/eksctl/pkg/cfn/manager" + "github.com/weaveworks/eksctl/pkg/testutils/mockprovider" +) + +var _ = Describe("CreateTasks", func() { + var subnetIDs = []string{"123", "456"} + var clusterConfig *api.ClusterConfig + Context("AssignIpv6AddressOnCreationTask", func() { + BeforeEach(func() { + clusterConfig = api.NewClusterConfig() + clusterConfig.VPC.Subnets = &api.ClusterSubnets{} + }) + + It("sets AssignIpv6AddressOnCreation to true for all public subnets", func() { + clusterConfig.VPC.Subnets.Public = map[string]api.AZSubnetSpec{ + "0": {ID: subnetIDs[0]}, + "1": {ID: subnetIDs[1]}, + } + modifySubnetAttributeCallCount := 0 + p := mockprovider.NewMockProvider() + p.MockEC2().On("ModifySubnetAttribute", mock.Anything).Run(func(args mock.Arguments) { + Expect(args).To(HaveLen(1)) + Expect(args[0]).To(BeAssignableToTypeOf(&ec2.ModifySubnetAttributeInput{})) + modifySubnetAttributeInput := args[0].(*ec2.ModifySubnetAttributeInput) + Expect(*modifySubnetAttributeInput.AssignIpv6AddressOnCreation.Value).To(BeTrue()) + Expect(subnetIDs).To(ContainElement(*modifySubnetAttributeInput.SubnetId)) + modifySubnetAttributeCallCount++ + }).Return(&ec2.ModifySubnetAttributeOutput{}, nil) + + task := manager.AssignIpv6AddressOnCreationTask{ + EC2API: p.EC2(), + ClusterConfig: clusterConfig, + } + errorCh := make(chan error) + err := task.Do(errorCh) + Expect(err).NotTo(HaveOccurred()) + Expect(modifySubnetAttributeCallCount).To(Equal(2)) + + By("closing the error channel") + Eventually(errorCh).Should(BeClosed()) + }) + + When("the API call errors", func() { + It("errors", func() { + clusterConfig.VPC.Subnets.Public = map[string]api.AZSubnetSpec{ + "0": {ID: subnetIDs[0]}, + } + p := mockprovider.NewMockProvider() + p.MockEC2().On("ModifySubnetAttribute", mock.Anything).Run(func(args mock.Arguments) { + Expect(args).To(HaveLen(1)) + Expect(args[0]).To(BeAssignableToTypeOf(&ec2.ModifySubnetAttributeInput{})) + modifySubnetAttributeInput := args[0].(*ec2.ModifySubnetAttributeInput) + Expect(*modifySubnetAttributeInput.AssignIpv6AddressOnCreation.Value).To(BeTrue()) + Expect(subnetIDs).To(ContainElement(*modifySubnetAttributeInput.SubnetId)) + }).Return(&ec2.ModifySubnetAttributeOutput{}, fmt.Errorf("foo")) + + task := manager.AssignIpv6AddressOnCreationTask{ + EC2API: p.EC2(), + ClusterConfig: clusterConfig, + } + errorCh := make(chan error) + err := task.Do(errorCh) + Expect(err).To(MatchError("failed to update subnet \"123\": foo")) + + By("closing the error channel") + Eventually(errorCh).Should(BeClosed()) + }) + }) + }) +}) diff --git a/pkg/cfn/manager/tasks.go b/pkg/cfn/manager/tasks.go index a4f1a36c03..11da02282a 100644 --- a/pkg/cfn/manager/tasks.go +++ b/pkg/cfn/manager/tasks.go @@ -6,6 +6,9 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5" iamoidc "github.com/weaveworks/eksctl/pkg/iam/oidc" kubewrapper "github.com/weaveworks/eksctl/pkg/kubernetes" @@ -131,3 +134,30 @@ func (t *kubernetesTask) Do(errs chan error) error { close(errs) return err } + +type AssignIpv6AddressOnCreationTask struct { + EC2API ec2iface.EC2API + ClusterConfig *api.ClusterConfig +} + +func (t *AssignIpv6AddressOnCreationTask) Describe() string { + return "set AssignIpv6AddressOnCreation to true for public subnets" +} + +func (t *AssignIpv6AddressOnCreationTask) Do(errs chan error) error { + defer close(errs) + if t.ClusterConfig.VPC.Subnets.Public != nil { + for _, subnet := range t.ClusterConfig.VPC.Subnets.Public.WithIDs() { + _, err := t.EC2API.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String(subnet), + }) + if err != nil { + return fmt.Errorf("failed to update subnet %q: %v", subnet, err) + } + } + } + return nil +} diff --git a/pkg/cfn/manager/tasks_test.go b/pkg/cfn/manager/tasks_test.go index 37b6481398..496ff5afee 100644 --- a/pkg/cfn/manager/tasks_test.go +++ b/pkg/cfn/manager/tasks_test.go @@ -3,6 +3,7 @@ package manager import ( "fmt" + "github.com/aws/aws-sdk-go/aws" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" @@ -59,83 +60,67 @@ var _ = Describe("StackCollection Tasks", func() { } Describe("TaskTree", func() { - Context("With real tasks", func() { - - BeforeEach(func() { - - p = mockprovider.NewMockProvider() + BeforeEach(func() { + p = mockprovider.NewMockProvider() + cfg = newClusterConfig("test-cluster") + stackManager = NewStackCollection(p, cfg) + }) - cfg = newClusterConfig("test-cluster") + It("should have nice description", func() { + fakeVPCImporter := new(vpcfakes.FakeImporter) + // TODO use DescribeTable + + // The supportsManagedNodes argument has no effect on the Describe call, so the values are alternated + // in these tests + { + tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("bar", "foo"), false, fakeVPCImporter) + Expect(tasks.Describe()).To(Equal(`2 parallel tasks: { create nodegroup "bar", create nodegroup "foo" }`)) + } + { + tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("bar"), false, fakeVPCImporter) + Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "bar" }`)) + } + { + tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("foo"), false, fakeVPCImporter) + Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "foo" }`)) + } + { + tasks := stackManager.NewUnmanagedNodeGroupTask(nil, false, fakeVPCImporter) + Expect(tasks.Describe()).To(Equal(`no tasks`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), nil, true) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo" } }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar"), nil, false) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", create nodegroup "bar" }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(nil, nil, true) + Expect(tasks.Describe()).To(Equal(`1 task: { create cluster control plane "test-cluster" }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), makeManagedNodeGroups("m1", "m2"), false) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 4 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo", create managed nodegroup "m1", create managed nodegroup "m2" } }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("foo"), makeManagedNodeGroups("m1"), true) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 parallel sub-tasks: { create nodegroup "foo", create managed nodegroup "m1" } }`)) + } + { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar"), nil, false, &task{id: 1}) + Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 sequential sub-tasks: { task 1, create nodegroup "bar" } }`)) + } + }) - stackManager = NewStackCollection(p, cfg) + When("IPFamily is set to ipv6", func() { + BeforeEach(func() { + cfg.VPC.IPFamily = aws.String(string(api.IPV6Family)) }) - - It("should have nice description", func() { - makeNodeGroups := func(names ...string) []*api.NodeGroup { - var nodeGroups []*api.NodeGroup - for _, name := range names { - ng := api.NewNodeGroup() - ng.Name = name - nodeGroups = append(nodeGroups, ng) - } - return nodeGroups - } - - makeManagedNodeGroups := func(names ...string) []*api.ManagedNodeGroup { - var managedNodeGroups []*api.ManagedNodeGroup - for _, name := range names { - ng := api.NewManagedNodeGroup() - ng.Name = name - managedNodeGroups = append(managedNodeGroups, ng) - } - return managedNodeGroups - } - - fakeVPCImporter := new(vpcfakes.FakeImporter) - // TODO use DescribeTable - - // The supportsManagedNodes argument has no effect on the Describe call, so the values are alternated - // in these tests - { - tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("bar", "foo"), false, fakeVPCImporter) - Expect(tasks.Describe()).To(Equal(`2 parallel tasks: { create nodegroup "bar", create nodegroup "foo" }`)) - } - { - tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("bar"), false, fakeVPCImporter) - Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "bar" }`)) - } - { - tasks := stackManager.NewUnmanagedNodeGroupTask(makeNodeGroups("foo"), false, fakeVPCImporter) - Expect(tasks.Describe()).To(Equal(`1 task: { create nodegroup "foo" }`)) - } - { - tasks := stackManager.NewUnmanagedNodeGroupTask(nil, false, fakeVPCImporter) - Expect(tasks.Describe()).To(Equal(`no tasks`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), nil, true) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo" } }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar"), nil, false) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", create nodegroup "bar" }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(nil, nil, true) - Expect(tasks.Describe()).To(Equal(`1 task: { create cluster control plane "test-cluster" }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), makeManagedNodeGroups("m1", "m2"), false) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 4 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo", create managed nodegroup "m1", create managed nodegroup "m2" } }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("foo"), makeManagedNodeGroups("m1"), true) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 parallel sub-tasks: { create nodegroup "foo", create managed nodegroup "m1" } }`)) - } - { - tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar"), nil, false, &task{id: 1}) - Expect(tasks.Describe()).To(Equal(`2 sequential tasks: { create cluster control plane "test-cluster", 2 sequential sub-tasks: { task 1, create nodegroup "bar" } }`)) - } + It("appends the AssignIpv6AddressOnCreation task to occur after the cluster creation", func() { + tasks := stackManager.NewTasksToCreateClusterWithNodeGroups(makeNodeGroups("bar", "foo"), nil, true) + Expect(tasks.Describe()).To(Equal(`3 sequential tasks: { create cluster control plane "test-cluster", set AssignIpv6AddressOnCreation to true for public subnets, 2 parallel sub-tasks: { create nodegroup "bar", create nodegroup "foo" } }`)) }) }) }) @@ -175,3 +160,23 @@ var _ = Describe("StackCollection Tasks", func() { }) }) }) + +func makeNodeGroups(names ...string) []*api.NodeGroup { + var nodeGroups []*api.NodeGroup + for _, name := range names { + ng := api.NewNodeGroup() + ng.Name = name + nodeGroups = append(nodeGroups, ng) + } + return nodeGroups +} + +func makeManagedNodeGroups(names ...string) []*api.ManagedNodeGroup { + var managedNodeGroups []*api.ManagedNodeGroup + for _, name := range names { + ng := api.NewManagedNodeGroup() + ng.Name = name + managedNodeGroups = append(managedNodeGroups, ng) + } + return managedNodeGroups +} diff --git a/pkg/ctl/cmdutils/configfile.go b/pkg/ctl/cmdutils/configfile.go index 9b0dd42c71..afe3ae4e93 100644 --- a/pkg/ctl/cmdutils/configfile.go +++ b/pkg/ctl/cmdutils/configfile.go @@ -17,6 +17,7 @@ import ( "github.com/weaveworks/eksctl/pkg/ctl/cmdutils/filter" "github.com/weaveworks/eksctl/pkg/eks" "github.com/weaveworks/eksctl/pkg/utils/names" + utilsstrings "github.com/weaveworks/eksctl/pkg/utils/strings" ) // AddConfigFileFlag adds common --config-file flag @@ -238,10 +239,12 @@ func NewCreateClusterLoader(cmd *Cmd, ngFilter *filter.NodeGroupFilter, ng *api. } if clusterConfig.VPC.NAT == nil { - clusterConfig.VPC.NAT = api.DefaultClusterNAT() + if utilsstrings.Value(clusterConfig.VPC.IPFamily) != string(api.IPV6Family) { + clusterConfig.VPC.NAT = api.DefaultClusterNAT() + } } - if !api.IsSetAndNonEmptyString(clusterConfig.VPC.NAT.Gateway) { + if clusterConfig.VPC.NAT != nil && api.IsEmpty(clusterConfig.VPC.NAT.Gateway) { *clusterConfig.VPC.NAT.Gateway = api.ClusterSingleNAT } diff --git a/pkg/ctl/cmdutils/configfile_test.go b/pkg/ctl/cmdutils/configfile_test.go index 93281c3542..bd33dcafae 100644 --- a/pkg/ctl/cmdutils/configfile_test.go +++ b/pkg/ctl/cmdutils/configfile_test.go @@ -200,6 +200,22 @@ var _ = Describe("cmdutils configfile", func() { } }) + When("using ipv6", func() { + It("should default VPC.NAT to nil", func() { + cmd := &Cmd{ + CobraCommand: newCmd(), + ClusterConfigFile: filepath.Join(examplesDir, "29-vpc-with-ip-family.yaml"), + ClusterConfig: api.NewClusterConfig(), + ProviderConfig: api.ProviderConfig{}, + } + params := &CreateClusterCmdParams{WithoutNodeGroup: true, CreateManagedNGOptions: CreateManagedNGOptions{ + Managed: false, + }} + Expect(NewCreateClusterLoader(cmd, filter.NewNodeGroupFilter(), nil, params).Load()).To(Succeed()) + Expect(cmd.ClusterConfig.VPC.NAT).To(BeNil()) + }) + }) + It("loader should handle named and unnamed nodegroups without config file", func() { unnamedNG := api.NewNodeGroup() diff --git a/pkg/vpc/vpc.go b/pkg/vpc/vpc.go index bb4320f47a..f987474f38 100644 --- a/pkg/vpc/vpc.go +++ b/pkg/vpc/vpc.go @@ -369,7 +369,7 @@ func importSubnetsForTopology(ec2API ec2iface.EC2API, spec *api.ClusterConfig, t return ImportSubnets(ec2API, spec, topology, subnets) } -// ImportSubnetsFromIDList will update spec with subnets _only specified by ID_ +// ImportSubnetsFromIDList will update cluster config with subnets _only specified by ID_ // then pass resulting subnets to ImportSubnets // NOTE: it does respect all fields set in spec.VPC, and will error if // there is a mismatch of local vs remote states