diff --git a/cloudmock/aws/mockelbv2/api.go b/cloudmock/aws/mockelbv2/api.go index 9f6d0cb0294eb..46685ba72dad8 100644 --- a/cloudmock/aws/mockelbv2/api.go +++ b/cloudmock/aws/mockelbv2/api.go @@ -19,6 +19,8 @@ package mockelbv2 import ( "sync" + "k8s.io/kops/cloudmock/aws/mockec2" + "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" ) @@ -28,6 +30,7 @@ type MockELBV2 struct { mutex sync.Mutex + EC2 *mockec2.MockEC2 LoadBalancers map[string]*loadBalancer lbCount int TargetGroups map[string]*targetGroup diff --git a/cloudmock/aws/mockelbv2/loadbalancers.go b/cloudmock/aws/mockelbv2/loadbalancers.go index 4903d3c22bf87..5a72384cf95fa 100644 --- a/cloudmock/aws/mockelbv2/loadbalancers.go +++ b/cloudmock/aws/mockelbv2/loadbalancers.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elbv2" "k8s.io/klog/v2" ) @@ -88,10 +89,17 @@ func (m *MockELBV2) CreateLoadBalancer(request *elbv2.CreateLoadBalancerInput) ( CanonicalHostedZoneId: aws.String("HZ123456"), } zones := make([]*elbv2.AvailabilityZone, 0) + vpc := "vpc-1" for _, subnet := range request.Subnets { zones = append(zones, &elbv2.AvailabilityZone{ SubnetId: subnet, }) + subnetsOutput, err := m.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{ + SubnetIds: []*string{subnet}, + }) + if err == nil { + vpc = *subnetsOutput.Subnets[0].VpcId + } } for _, subnetMapping := range request.SubnetMappings { var lbAddrs []*elbv2.LoadBalancerAddress @@ -105,12 +113,16 @@ func (m *MockELBV2) CreateLoadBalancer(request *elbv2.CreateLoadBalancerInput) ( SubnetId: subnetMapping.SubnetId, LoadBalancerAddresses: lbAddrs, }) + subnetsOutput, err := m.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{ + SubnetIds: []*string{subnetMapping.SubnetId}, + }) + if err == nil { + vpc = *subnetsOutput.Subnets[0].VpcId + } } lb.AvailabilityZones = zones - // This is hardcoded because AWS derives it from the subnets above - // But we don'y rely on the NLB's VPC ID at all in awstasks - lb.VpcId = aws.String("vpc-1") + lb.VpcId = aws.String(vpc) m.lbCount++ arn := fmt.Sprintf("arn:aws-test:elasticloadbalancing:us-test-1:000000000000:loadbalancer/net/%v/%v", aws.StringValue(request.Name), m.lbCount) diff --git a/docs/releases/1.26-NOTES.md b/docs/releases/1.26-NOTES.md index 3fd0659c87996..c4d3f58acace2 100644 --- a/docs/releases/1.26-NOTES.md +++ b/docs/releases/1.26-NOTES.md @@ -6,7 +6,11 @@ This is a document to gather the release notes prior to the release. # Significant changes -* Instance group images can now be dynamically fetched through an AWS SSM Parameter (AWS only). +## AWS only + +* Bastions are now fronted by a Network Load Balancer. + +* Instance group images can now be dynamically fetched through an AWS SSM Parameter. # Breaking changes diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index fffe3fe459bce..3bba77ca6a5bd 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -5604,13 +5604,13 @@ spec: bastionPublicName: type: string idleTimeoutSeconds: - description: IdleTimeoutSeconds is the bastion's Loadbalancer - idle timeout + description: IdleTimeoutSeconds is unused format: int64 type: integer loadBalancer: properties: additionalSecurityGroups: + description: AdditionalSecurityGroups is unused items: type: string type: array diff --git a/pkg/apis/kops/bastion.go b/pkg/apis/kops/bastion.go index a8af627eda2b8..ff59c92dd15de 100644 --- a/pkg/apis/kops/bastion.go +++ b/pkg/apis/kops/bastion.go @@ -19,14 +19,11 @@ package kops type BastionSpec struct { // PublicName is the domain name for the bastion load balancer. PublicName string `json:"publicName,omitempty"` - // IdleTimeoutSeconds is the bastion's load balancer idle timeout. - IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` // LoadBalancer contains settings for the load balancer fronting bastion instances. LoadBalancer *BastionLoadBalancerSpec `json:"loadBalancer,omitempty"` } type BastionLoadBalancerSpec struct { - AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` // Type of load balancer to create, it can be Public or Internal. Type LoadBalancerType `json:"type,omitempty"` } diff --git a/pkg/apis/kops/v1alpha2/bastion.go b/pkg/apis/kops/v1alpha2/bastion.go index d1dd653477221..74552fd9edd3e 100644 --- a/pkg/apis/kops/v1alpha2/bastion.go +++ b/pkg/apis/kops/v1alpha2/bastion.go @@ -18,12 +18,15 @@ package v1alpha2 type BastionSpec struct { PublicName string `json:"bastionPublicName,omitempty"` - // IdleTimeoutSeconds is the bastion's Loadbalancer idle timeout + // IdleTimeoutSeconds is unused + // +k8s:conversion-gen=false IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` LoadBalancer *BastionLoadBalancerSpec `json:"loadBalancer,omitempty"` } type BastionLoadBalancerSpec struct { + // AdditionalSecurityGroups is unused + // +k8s:conversion-gen=false AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` // Type of load balancer to create, it can be Public or Internal. Type LoadBalancerType `json:"type,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 662f898eb6730..b49815a953438 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1746,7 +1746,7 @@ func Convert_kops_AzureSpec_To_v1alpha2_AzureSpec(in *kops.AzureSpec, out *Azure } func autoConvert_v1alpha2_BastionLoadBalancerSpec_To_kops_BastionLoadBalancerSpec(in *BastionLoadBalancerSpec, out *kops.BastionLoadBalancerSpec, s conversion.Scope) error { - out.AdditionalSecurityGroups = in.AdditionalSecurityGroups + // INFO: in.AdditionalSecurityGroups opted out of conversion generation out.Type = kops.LoadBalancerType(in.Type) return nil } @@ -1757,7 +1757,6 @@ func Convert_v1alpha2_BastionLoadBalancerSpec_To_kops_BastionLoadBalancerSpec(in } func autoConvert_kops_BastionLoadBalancerSpec_To_v1alpha2_BastionLoadBalancerSpec(in *kops.BastionLoadBalancerSpec, out *BastionLoadBalancerSpec, s conversion.Scope) error { - out.AdditionalSecurityGroups = in.AdditionalSecurityGroups out.Type = LoadBalancerType(in.Type) return nil } @@ -1769,7 +1768,7 @@ func Convert_kops_BastionLoadBalancerSpec_To_v1alpha2_BastionLoadBalancerSpec(in func autoConvert_v1alpha2_BastionSpec_To_kops_BastionSpec(in *BastionSpec, out *kops.BastionSpec, s conversion.Scope) error { out.PublicName = in.PublicName - out.IdleTimeoutSeconds = in.IdleTimeoutSeconds + // INFO: in.IdleTimeoutSeconds opted out of conversion generation if in.LoadBalancer != nil { in, out := &in.LoadBalancer, &out.LoadBalancer *out = new(kops.BastionLoadBalancerSpec) @@ -1789,7 +1788,6 @@ func Convert_v1alpha2_BastionSpec_To_kops_BastionSpec(in *BastionSpec, out *kops func autoConvert_kops_BastionSpec_To_v1alpha2_BastionSpec(in *kops.BastionSpec, out *BastionSpec, s conversion.Scope) error { out.PublicName = in.PublicName - out.IdleTimeoutSeconds = in.IdleTimeoutSeconds if in.LoadBalancer != nil { in, out := &in.LoadBalancer, &out.LoadBalancer *out = new(BastionLoadBalancerSpec) diff --git a/pkg/apis/kops/v1alpha3/bastion.go b/pkg/apis/kops/v1alpha3/bastion.go index 4c1ded6795f34..5b159bcd71487 100644 --- a/pkg/apis/kops/v1alpha3/bastion.go +++ b/pkg/apis/kops/v1alpha3/bastion.go @@ -19,14 +19,11 @@ package v1alpha3 type BastionSpec struct { // PublicName is the domain name for the bastion load balancer. PublicName string `json:"publicName,omitempty"` - // IdleTimeoutSeconds is the bastion's load balancer idle timeout. - IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` // LoadBalancer contains settings for the load balancer fronting bastion instances. LoadBalancer *BastionLoadBalancerSpec `json:"loadBalancer,omitempty"` } type BastionLoadBalancerSpec struct { - AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` // Type of load balancer to create, it can be Public or Internal. Type LoadBalancerType `json:"type,omitempty"` } diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index fc9c9767d27e4..1fbcacc944220 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -1774,7 +1774,6 @@ func Convert_kops_AzureSpec_To_v1alpha3_AzureSpec(in *kops.AzureSpec, out *Azure } func autoConvert_v1alpha3_BastionLoadBalancerSpec_To_kops_BastionLoadBalancerSpec(in *BastionLoadBalancerSpec, out *kops.BastionLoadBalancerSpec, s conversion.Scope) error { - out.AdditionalSecurityGroups = in.AdditionalSecurityGroups out.Type = kops.LoadBalancerType(in.Type) return nil } @@ -1785,7 +1784,6 @@ func Convert_v1alpha3_BastionLoadBalancerSpec_To_kops_BastionLoadBalancerSpec(in } func autoConvert_kops_BastionLoadBalancerSpec_To_v1alpha3_BastionLoadBalancerSpec(in *kops.BastionLoadBalancerSpec, out *BastionLoadBalancerSpec, s conversion.Scope) error { - out.AdditionalSecurityGroups = in.AdditionalSecurityGroups out.Type = LoadBalancerType(in.Type) return nil } @@ -1797,7 +1795,6 @@ func Convert_kops_BastionLoadBalancerSpec_To_v1alpha3_BastionLoadBalancerSpec(in func autoConvert_v1alpha3_BastionSpec_To_kops_BastionSpec(in *BastionSpec, out *kops.BastionSpec, s conversion.Scope) error { out.PublicName = in.PublicName - out.IdleTimeoutSeconds = in.IdleTimeoutSeconds if in.LoadBalancer != nil { in, out := &in.LoadBalancer, &out.LoadBalancer *out = new(kops.BastionLoadBalancerSpec) @@ -1817,7 +1814,6 @@ func Convert_v1alpha3_BastionSpec_To_kops_BastionSpec(in *BastionSpec, out *kops func autoConvert_kops_BastionSpec_To_v1alpha3_BastionSpec(in *kops.BastionSpec, out *BastionSpec, s conversion.Scope) error { out.PublicName = in.PublicName - out.IdleTimeoutSeconds = in.IdleTimeoutSeconds if in.LoadBalancer != nil { in, out := &in.LoadBalancer, &out.LoadBalancer *out = new(BastionLoadBalancerSpec) diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index 9dd5d3e63cd7b..6fb780ec481a7 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -422,11 +422,6 @@ func (in *AzureSpec) DeepCopy() *AzureSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BastionLoadBalancerSpec) DeepCopyInto(out *BastionLoadBalancerSpec) { *out = *in - if in.AdditionalSecurityGroups != nil { - in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups - *out = make([]string, len(*in)) - copy(*out, *in) - } return } @@ -443,15 +438,10 @@ func (in *BastionLoadBalancerSpec) DeepCopy() *BastionLoadBalancerSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BastionSpec) DeepCopyInto(out *BastionSpec) { *out = *in - if in.IdleTimeoutSeconds != nil { - in, out := &in.IdleTimeoutSeconds, &out.IdleTimeoutSeconds - *out = new(int64) - **out = **in - } if in.LoadBalancer != nil { in, out := &in.LoadBalancer, &out.LoadBalancer *out = new(BastionLoadBalancerSpec) - (*in).DeepCopyInto(*out) + **out = **in } return } diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index efee4c05c9646..8405dfb79a7e3 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -414,16 +414,9 @@ func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *f } if topology.Bastion != nil { - bastion := topology.Bastion if topology.Masters == kops.TopologyPublic || topology.Nodes == kops.TopologyPublic { allErrs = append(allErrs, field.Forbidden(fieldPath.Child("bastion"), "bastion requires masters and nodes to have private topology")) } - if bastion.IdleTimeoutSeconds != nil && *bastion.IdleTimeoutSeconds <= 0 { - allErrs = append(allErrs, field.Invalid(fieldPath.Child("bastion", "idleTimeoutSeconds"), *bastion.IdleTimeoutSeconds, "bastion idleTimeoutSeconds should be greater than zero")) - } - if bastion.IdleTimeoutSeconds != nil && *bastion.IdleTimeoutSeconds > 3600 { - allErrs = append(allErrs, field.Invalid(fieldPath.Child("bastion", "idleTimeoutSeconds"), *bastion.IdleTimeoutSeconds, "bastion idleTimeoutSeconds cannot be greater than one hour")) - } } if topology.DNS != nil { diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 4274e49f55156..4ea0776d74c6b 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -421,11 +421,6 @@ func (in *AzureSpec) DeepCopy() *AzureSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BastionLoadBalancerSpec) DeepCopyInto(out *BastionLoadBalancerSpec) { *out = *in - if in.AdditionalSecurityGroups != nil { - in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups - *out = make([]string, len(*in)) - copy(*out, *in) - } return } @@ -442,15 +437,10 @@ func (in *BastionLoadBalancerSpec) DeepCopy() *BastionLoadBalancerSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BastionSpec) DeepCopyInto(out *BastionSpec) { *out = *in - if in.IdleTimeoutSeconds != nil { - in, out := &in.IdleTimeoutSeconds, &out.IdleTimeoutSeconds - *out = new(int64) - **out = **in - } if in.LoadBalancer != nil { in, out := &in.LoadBalancer, &out.LoadBalancer *out = new(BastionLoadBalancerSpec) - (*in).DeepCopyInto(*out) + **out = **in } return } diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 3f7978d4c28aa..77f310a85e60f 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -460,7 +460,7 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil } if ig.Spec.Role == kops.InstanceGroupRoleBastion { - t.LoadBalancers = append(t.LoadBalancers, b.LinkToCLB("bastion")) + t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("bastion")) } } diff --git a/pkg/model/awsmodel/bastion.go b/pkg/model/awsmodel/bastion.go index 7643d1db3f011..c6a28a358f5ba 100644 --- a/pkg/model/awsmodel/bastion.go +++ b/pkg/model/awsmodel/bastion.go @@ -19,18 +19,14 @@ package awsmodel import ( "fmt" "sort" - "time" + "strings" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" -) - -const ( - BastionELBSecurityGroupPrefix = "bastion" - BastionELBDefaultIdleTimeout = 5 * time.Minute + "k8s.io/kops/upup/pkg/fi/utils" ) // BastionModelBuilder adds model objects to support bastions @@ -104,7 +100,7 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error { var bastionLoadBalancerType kops.LoadBalancerType { - // Check if we requested a public or internal ELB + // Check if we requested a public or internal NLB if b.Cluster.Spec.Topology != nil && b.Cluster.Spec.Topology.Bastion != nil && b.Cluster.Spec.Topology.Bastion.LoadBalancer != nil { if b.Cluster.Spec.Topology.Bastion.LoadBalancer.Type != "" { switch b.Cluster.Spec.Topology.Bastion.LoadBalancer.Type { @@ -126,21 +122,6 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error { } } - // Allow incoming SSH traffic to bastions, through the ELB - // TODO: Could we get away without an ELB here? Tricky to fix if dns-controller breaks though... - for _, dest := range bastionGroups { - t := &awstasks.SecurityGroupRule{ - Name: fi.String("ssh-elb-to-bastion" + dest.Suffix), - Lifecycle: b.SecurityLifecycle, - SecurityGroup: dest.Task, - SourceGroup: b.LinkToELBSecurityGroup(BastionELBSecurityGroupPrefix), - Protocol: fi.String("tcp"), - FromPort: fi.Int64(22), - ToPort: fi.Int64(22), - } - AddDirectionalGroupRule(c, t) - } - // Allow bastion nodes to SSH to masters for _, src := range bastionGroups { for _, dest := range masterGroups { @@ -173,58 +154,10 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error { } } - // Create security group for bastion ELB - { - t := &awstasks.SecurityGroup{ - Name: fi.String(b.ELBSecurityGroupName(BastionELBSecurityGroupPrefix)), - Lifecycle: b.SecurityLifecycle, - VPC: b.LinkToVPC(), - Description: fi.String("Security group for bastion ELB"), - RemoveExtraRules: []string{"port=22"}, - } - t.Tags = b.CloudTags(*t.Name, false) - c.AddTask(t) - } - - // Allow traffic from ELB to egress freely - { - t := &awstasks.SecurityGroupRule{ - Name: fi.String("ipv4-bastion-elb-egress"), - Lifecycle: b.SecurityLifecycle, - SecurityGroup: b.LinkToELBSecurityGroup(BastionELBSecurityGroupPrefix), - Egress: fi.Bool(true), - CIDR: fi.String("0.0.0.0/0"), - } - AddDirectionalGroupRule(c, t) - } + var sshAllowedCIDRs []string + var nlbSubnetMappings []*awstasks.SubnetMapping { - t := &awstasks.SecurityGroupRule{ - Name: fi.String("ipv6-bastion-elb-egress"), - Lifecycle: b.SecurityLifecycle, - SecurityGroup: b.LinkToELBSecurityGroup(BastionELBSecurityGroupPrefix), - Egress: fi.Bool(true), - IPv6CIDR: fi.String("::/0"), - } - AddDirectionalGroupRule(c, t) - } - - // Allow external access to ELB - for _, sshAccess := range b.Cluster.Spec.SSHAccess { - t := &awstasks.SecurityGroupRule{ - Name: fi.String("ssh-external-to-bastion-elb-" + sshAccess), - Lifecycle: b.SecurityLifecycle, - SecurityGroup: b.LinkToELBSecurityGroup(BastionELBSecurityGroupPrefix), - Protocol: fi.String("tcp"), - FromPort: fi.Int64(22), - ToPort: fi.Int64(22), - } - t.SetCidrOrPrefix(sshAccess) - AddDirectionalGroupRule(c, t) - } - - var elbSubnets []*awstasks.Subnet - { - // Compute the subnets - only one per zone, and then break ties based on chooseBestSubnetForELB + // Compute the subnets - only one per zone, and then break ties based on chooseBestSubnetForNLB subnetsByZone := make(map[string][]*kops.ClusterSubnetSpec) for i := range b.Cluster.Spec.Subnets { subnet := &b.Cluster.Spec.Subnets[i] @@ -248,23 +181,67 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error { } for zone, subnets := range subnetsByZone { - subnet := b.chooseBestSubnetForELB(zone, subnets) + for _, subnet := range subnets { + sshAllowedCIDRs = append(sshAllowedCIDRs, subnet.CIDR) + } + subnet := b.chooseBestSubnetForNLB(zone, subnets) + nlbSubnetMappings = append(nlbSubnetMappings, &awstasks.SubnetMapping{Subnet: b.LinkToSubnet(subnet)}) + } + } - elbSubnet := b.LinkToSubnet(subnet) - elbSubnets = append(elbSubnets, elbSubnet) + sshAllowedCIDRs = append(sshAllowedCIDRs, b.Cluster.Spec.SSHAccess...) + for _, cidr := range sshAllowedCIDRs { + // Allow incoming SSH traffic to bastions, through the NLB + // TODO: Could we get away without an NLB here? Tricky to fix if dns-controller breaks though... + for _, bastionGroup := range bastionGroups { + { + t := &awstasks.SecurityGroupRule{ + Name: fi.String(fmt.Sprintf("ssh-nlb-%s", cidr)), + Lifecycle: b.SecurityLifecycle, + SecurityGroup: bastionGroup.Task, + Protocol: fi.String("tcp"), + FromPort: fi.Int64(22), + ToPort: fi.Int64(22), + } + t.SetCidrOrPrefix(cidr) + AddDirectionalGroupRule(c, t) + } + + if strings.HasPrefix(cidr, "pl-") { + // In case of a prefix list we do not add a rule for ICMP traffic for PMTU discovery. + // This would require calling out to AWS to check whether the prefix list is IPv4 or IPv6. + } else if utils.IsIPv6CIDR(cidr) { + // Allow ICMP traffic required for PMTU discovery + t := &awstasks.SecurityGroupRule{ + Name: fi.String("icmpv6-pmtu-ssh-nlb-" + cidr), + Lifecycle: b.SecurityLifecycle, + FromPort: fi.Int64(-1), + Protocol: fi.String("icmpv6"), + SecurityGroup: bastionGroup.Task, + ToPort: fi.Int64(-1), + } + t.SetCidrOrPrefix(cidr) + c.AddTask(t) + } else { + t := &awstasks.SecurityGroupRule{ + Name: fi.String("icmp-pmtu-ssh-nlb-" + cidr), + Lifecycle: b.SecurityLifecycle, + FromPort: fi.Int64(3), + Protocol: fi.String("icmp"), + SecurityGroup: bastionGroup.Task, + ToPort: fi.Int64(4), + } + t.SetCidrOrPrefix(cidr) + c.AddTask(t) + } } } - // Create ELB itself - var elb *awstasks.ClassicLoadBalancer + // Create NLB itself + var nlb *awstasks.NetworkLoadBalancer { loadBalancerName := b.LBName32("bastion") - idleTimeout := BastionELBDefaultIdleTimeout - if b.Cluster.Spec.Topology != nil && b.Cluster.Spec.Topology.Bastion != nil && b.Cluster.Spec.Topology.Bastion.IdleTimeoutSeconds != nil { - idleTimeout = time.Second * time.Duration(*b.Cluster.Spec.Topology.Bastion.IdleTimeoutSeconds) - } - tags := b.CloudTags(loadBalancerName, false) for k, v := range b.Cluster.Spec.CloudLabels { tags[k] = v @@ -272,59 +249,64 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error { // Override the returned name to be the expected ELB name tags["Name"] = "bastion." + b.ClusterName() - elb = &awstasks.ClassicLoadBalancer{ - Name: fi.String("bastion." + b.ClusterName()), + nlbListeners := []*awstasks.NetworkLoadBalancerListener{ + { + Port: 22, + TargetGroupName: b.NLBTargetGroupName("bastion"), + }, + } + nlb = &awstasks.NetworkLoadBalancer{ + Name: fi.String(b.NLBName("bastion")), Lifecycle: b.Lifecycle, LoadBalancerName: fi.String(loadBalancerName), - SecurityGroups: []*awstasks.SecurityGroup{ - b.LinkToELBSecurityGroup(BastionELBSecurityGroupPrefix), - }, - Subnets: elbSubnets, - Listeners: map[string]*awstasks.ClassicLoadBalancerListener{ - "22": {InstancePort: 22}, - }, - - HealthCheck: &awstasks.ClassicLoadBalancerHealthCheck{ - Target: fi.String("TCP:22"), - Timeout: fi.Int64(5), - Interval: fi.Int64(10), - HealthyThreshold: fi.Int64(2), - UnhealthyThreshold: fi.Int64(2), - }, - - ConnectionSettings: &awstasks.ClassicLoadBalancerConnectionSettings{ - IdleTimeout: fi.Int64(int64(idleTimeout.Seconds())), - }, - - Tags: tags, + SubnetMappings: nlbSubnetMappings, + Listeners: nlbListeners, + TargetGroups: make([]*awstasks.TargetGroup, 0), + + Tags: tags, + VPC: b.LinkToVPC(), + Type: fi.String("network"), + IpAddressType: fi.String("ipv4"), } - // Add additional security groups to the ELB - if b.Cluster.Spec.Topology != nil && b.Cluster.Spec.Topology.Bastion != nil && b.Cluster.Spec.Topology.Bastion.LoadBalancer != nil && b.Cluster.Spec.Topology.Bastion.LoadBalancer.AdditionalSecurityGroups != nil { - for _, id := range b.Cluster.Spec.Topology.Bastion.LoadBalancer.AdditionalSecurityGroups { - t := &awstasks.SecurityGroup{ - Name: fi.String(id), - Lifecycle: b.SecurityLifecycle, - ID: fi.String(id), - Shared: fi.Bool(true), - } - if err := c.EnsureTask(t); err != nil { - return err - } - elb.SecurityGroups = append(elb.SecurityGroups, t) - } + if useIPv6ForBastion(b) { + nlb.IpAddressType = fi.String("dualstack") } - // Set the elb Scheme according to load balancer Type + // Set the NLB Scheme according to load balancer Type switch bastionLoadBalancerType { case kops.LoadBalancerTypeInternal: - elb.Scheme = fi.String("internal") + nlb.Scheme = fi.String("internal") case kops.LoadBalancerTypePublic: - elb.Scheme = nil + nlb.Scheme = nil default: return fmt.Errorf("unhandled bastion LoadBalancer type %q", bastionLoadBalancerType) } - c.AddTask(elb) + sshGroupName := b.NLBTargetGroupName("bastion") + sshGroupTags := b.CloudTags(sshGroupName, false) + + // Override the returned name to be the expected NLB TG name + sshGroupTags["Name"] = sshGroupName + + tg := &awstasks.TargetGroup{ + Name: fi.String(sshGroupName), + Lifecycle: b.Lifecycle, + VPC: b.LinkToVPC(), + Tags: sshGroupTags, + Protocol: fi.String("TCP"), + Port: fi.Int64(22), + Interval: fi.Int64(10), + HealthyThreshold: fi.Int64(2), + UnhealthyThreshold: fi.Int64(2), + Shared: fi.Bool(false), + } + + c.AddTask(tg) + + nlb.TargetGroups = append(nlb.TargetGroups, tg) + + sort.Stable(awstasks.OrderTargetGroupsByName(nlb.TargetGroups)) + c.AddTask(nlb) } publicName := "" @@ -341,19 +323,47 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error { Zone: b.LinkToDNSZone(), ResourceName: fi.String(publicName), ResourceType: fi.String("A"), - TargetLoadBalancer: elb, + TargetLoadBalancer: b.LinkToNLB("bastion"), } c.AddTask(t) + if *nlb.IpAddressType == "dualstack" { + t := &awstasks.DNSName{ + Name: fi.String(publicName + "-AAAA"), + Lifecycle: b.Lifecycle, + + Zone: b.LinkToDNSZone(), + ResourceName: fi.String(publicName), + ResourceType: fi.String("AAAA"), + TargetLoadBalancer: b.LinkToNLB("bastion"), + } + c.AddTask(t) + } } return nil } +func useIPv6ForBastion(b *BastionModelBuilder) bool { + for _, ig := range b.InstanceGroups { + for _, igSubnetName := range ig.Spec.Subnets { + for _, clusterSubnet := range b.Cluster.Spec.Subnets { + if igSubnetName != clusterSubnet.Name { + continue + } + if clusterSubnet.IPv6CIDR != "" { + return true + } + } + } + } + return false +} + // Choose between subnets in a zone. -// We have already applied the rules to match internal subnets to internal ELBs and vice-versa for public-facing ELBs. -// For internal ELBs: we prefer the master subnets -// For public facing ELBs: we prefer the utility subnets -func (b *BastionModelBuilder) chooseBestSubnetForELB(zone string, subnets []*kops.ClusterSubnetSpec) *kops.ClusterSubnetSpec { +// We have already applied the rules to match internal subnets to internal NLBs and vice-versa for public-facing NLBs. +// For internal NLBs: we prefer the master subnets +// For public facing NLBs: we prefer the utility subnets +func (b *BastionModelBuilder) chooseBestSubnetForNLB(zone string, subnets []*kops.ClusterSubnetSpec) *kops.ClusterSubnetSpec { if len(subnets) == 0 { return nil } @@ -376,8 +386,12 @@ func (b *BastionModelBuilder) chooseBestSubnetForELB(zone string, subnets []*kop score += 1 } + if subnet.Type == kops.SubnetTypeDualStack { + score += 2 + } + if subnet.Type == kops.SubnetTypeUtility { - score += 1 + score += 3 } scoredSubnets = append(scoredSubnets, &scoredSubnet{ @@ -389,7 +403,7 @@ func (b *BastionModelBuilder) chooseBestSubnetForELB(zone string, subnets []*kop sort.Sort(ByScoreDescending(scoredSubnets)) if scoredSubnets[0].score == scoredSubnets[1].score { - klog.V(2).Infof("Making arbitrary choice between subnets in zone %q to attach to ELB (%q vs %q)", zone, scoredSubnets[0].subnet.Name, scoredSubnets[1].subnet.Name) + klog.V(2).Infof("Making arbitrary choice between subnets in zone %q to attach to NLB (%q vs %q)", zone, scoredSubnets[0].subnet.Name, scoredSubnets[1].subnet.Name) } return scoredSubnets[0].subnet diff --git a/pkg/testutils/integrationtestharness.go b/pkg/testutils/integrationtestharness.go index 78686c66e5409..3b6eaad8e7422 100644 --- a/pkg/testutils/integrationtestharness.go +++ b/pkg/testutils/integrationtestharness.go @@ -139,7 +139,7 @@ func (h *IntegrationTestHarness) SetupMockAWS() *awsup.MockAWSCloud { cloud.MockRoute53 = mockRoute53 mockELB := &mockelb.MockELB{} cloud.MockELB = mockELB - mockELBV2 := &mockelbv2.MockELBV2{} + mockELBV2 := &mockelbv2.MockELBV2{EC2: mockEC2} cloud.MockELBV2 = mockELBV2 mockIAM := &mockiam.MockIAM{} cloud.MockIAM = mockIAM diff --git a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_object_cluster-completed.spec_content index da4d67e5ee66b..e813e84cbf710 100644 --- a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_object_cluster-completed.spec_content @@ -183,9 +183,7 @@ spec: zone: us-test-1a topology: bastion: - loadBalancer: - additionalSecurityGroups: - - sg-exampleid + loadBalancer: {} dns: type: Public masters: private diff --git a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf index 6d529ec5a1c68..62121ff8493c9 100644 --- a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf +++ b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf @@ -121,7 +121,6 @@ resource "aws_autoscaling_group" "bastion-bastionuserdata-example-com" { id = aws_launch_template.bastion-bastionuserdata-example-com.id version = aws_launch_template.bastion-bastionuserdata-example-com.latest_version } - load_balancers = [aws_elb.bastion-bastionuserdata-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -163,6 +162,7 @@ resource "aws_autoscaling_group" "bastion-bastionuserdata-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-bastionuserdata-e-4grhsv.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-bastionuserdata-example-com.id] } @@ -351,31 +351,6 @@ resource "aws_elb" "api-bastionuserdata-example-com" { } } -resource "aws_elb" "bastion-bastionuserdata-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-bastionuserdata-e-4grhsv" - security_groups = [aws_security_group.bastion-elb-bastionuserdata-example-com.id, "sg-exampleid"] - subnets = [aws_subnet.utility-us-test-1a-bastionuserdata-example-com.id] - tags = { - "KubernetesCluster" = "bastionuserdata.example.com" - "Name" = "bastion.bastionuserdata.example.com" - "kubernetes.io/cluster/bastionuserdata.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-bastionuserdata-example-com" { name = "bastions.bastionuserdata.example.com" role = aws_iam_role.bastions-bastionuserdata-example-com.name @@ -705,6 +680,49 @@ resource "aws_launch_template" "nodes-bastionuserdata-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data") } +resource "aws_lb" "bastion-bastionuserdata-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-bastionuserdata-e-4grhsv" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-bastionuserdata-example-com.id + } + tags = { + "KubernetesCluster" = "bastionuserdata.example.com" + "Name" = "bastion.bastionuserdata.example.com" + "kubernetes.io/cluster/bastionuserdata.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-bastionuserdata-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-bastionuserdata-e-4grhsv.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-bastionuserdata-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-bastionuserdata-e-4grhsv" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-bastionuserdata-e-4grhsv" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "bastionuserdata.example.com" + "Name" = "bastion-bastionuserdata-e-4grhsv" + "kubernetes.io/cluster/bastionuserdata.example.com" = "owned" + } + vpc_id = aws_vpc.bastionuserdata-example-com.id +} + resource "aws_nat_gateway" "us-test-1a-bastionuserdata-example-com" { allocation_id = aws_eip.us-test-1a-bastionuserdata-example-com.id subnet_id = aws_subnet.utility-us-test-1a-bastionuserdata-example-com.id @@ -932,17 +950,6 @@ resource "aws_security_group" "bastion-bastionuserdata-example-com" { vpc_id = aws_vpc.bastionuserdata-example-com.id } -resource "aws_security_group" "bastion-elb-bastionuserdata-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.bastionuserdata.example.com" - tags = { - "KubernetesCluster" = "bastionuserdata.example.com" - "Name" = "bastion-elb.bastionuserdata.example.com" - "kubernetes.io/cluster/bastionuserdata.example.com" = "owned" - } - vpc_id = aws_vpc.bastionuserdata-example-com.id -} - resource "aws_security_group" "masters-bastionuserdata-example-com" { description = "Security group for masters" name = "masters.bastionuserdata.example.com" @@ -965,11 +972,11 @@ resource "aws_security_group" "nodes-bastionuserdata-example-com" { vpc_id = aws_vpc.bastionuserdata-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-bastionuserdata-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-bastionuserdata-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-bastionuserdata-example-com.id + security_group_id = aws_security_group.bastion-bastionuserdata-example-com.id to_port = 22 type = "ingress" } @@ -983,6 +990,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-bastionuserdata-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-bastionuserdata-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-bastionuserdata-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1037,33 +1053,6 @@ resource "aws_security_group_rule" "from-bastion-bastionuserdata-example-com-ing type = "ingress" } -resource "aws_security_group_rule" "from-bastion-elb-bastionuserdata-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-bastionuserdata-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-bastionuserdata-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-bastionuserdata-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-bastionuserdata-example-com-ingress-tcp-22to22-bastion-bastionuserdata-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-bastionuserdata-example-com.id - source_security_group_id = aws_security_group.bastion-elb-bastionuserdata-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-masters-bastionuserdata-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1181,6 +1170,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-bastionuserdata-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-bastionuserdata-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-bastionuserdata-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/private-shared-ip/cloudformation.json b/tests/integration/update_cluster/private-shared-ip/cloudformation.json index fb9adf7fc2808..a594186700125 100644 --- a/tests/integration/update_cluster/private-shared-ip/cloudformation.json +++ b/tests/integration/update_cluster/private-shared-ip/cloudformation.json @@ -74,9 +74,9 @@ ] } ], - "LoadBalancerNames": [ + "TargetGroupARNs": [ { - "Ref": "AWSElasticLoadBalancingLoadBalancerbastionprivatesharedipexamplecom" + "Ref": "AWSElasticLoadBalancingV2TargetGroupbastionprivatesharedipeepmph" } ] } @@ -761,30 +761,6 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupEgressfrombastionelbprivatesharedipexamplecomegressall0to00": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivatesharedipexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIpv6": "::/0" - } - }, - "AWSEC2SecurityGroupEgressfrombastionelbprivatesharedipexamplecomegressall0to000000": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivatesharedipexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, "AWSEC2SecurityGroupEgressfrombastionprivatesharedipexamplecomegressall0to00": { "Type": "AWS::EC2::SecurityGroupEgress", "Properties": { @@ -857,11 +833,11 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22bastionelbprivatesharedipexamplecom": { + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22bastionprivatesharedipexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivatesharedipexamplecom" + "Ref": "AWSEC2SecurityGroupbastionprivatesharedipexamplecom" }, "FromPort": 22, "ToPort": 22, @@ -881,18 +857,16 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrombastionelbprivatesharedipexamplecomingresstcp22to22bastionprivatesharedipexamplecom": { + "AWSEC2SecurityGroupIngressfrom172204022ingresstcp22to22bastionprivatesharedipexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { "Ref": "AWSEC2SecurityGroupbastionprivatesharedipexamplecom" }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivatesharedipexamplecom" - }, "FromPort": 22, "ToPort": 22, - "IpProtocol": "tcp" + "IpProtocol": "tcp", + "CidrIp": "172.20.4.0/22" } }, "AWSEC2SecurityGroupIngressfrombastionprivatesharedipexamplecomingresstcp22to22mastersprivatesharedipexamplecom": { @@ -1047,6 +1021,30 @@ "CidrIp": "0.0.0.0/0" } }, + "AWSEC2SecurityGroupIngressicmppmtusshnlb00000": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupbastionprivatesharedipexamplecom" + }, + "FromPort": 3, + "ToPort": 4, + "IpProtocol": "icmp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressicmppmtusshnlb172204022": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupbastionprivatesharedipexamplecom" + }, + "FromPort": 3, + "ToPort": 4, + "IpProtocol": "icmp", + "CidrIp": "172.20.4.0/22" + } + }, "AWSEC2SecurityGroupapielbprivatesharedipexamplecom": { "Type": "AWS::EC2::SecurityGroup", "Properties": { @@ -1069,28 +1067,6 @@ ] } }, - "AWSEC2SecurityGroupbastionelbprivatesharedipexamplecom": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupName": "bastion-elb.private-shared-ip.example.com", - "VpcId": "vpc-12345678", - "GroupDescription": "Security group for bastion ELB", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "private-shared-ip.example.com" - }, - { - "Key": "Name", - "Value": "bastion-elb.private-shared-ip.example.com" - }, - { - "Key": "kubernetes.io/cluster/private-shared-ip.example.com", - "Value": "owned" - } - ] - } - }, "AWSEC2SecurityGroupbastionprivatesharedipexamplecom": { "Type": "AWS::EC2::SecurityGroup", "Properties": { @@ -1370,38 +1346,37 @@ ] } }, - "AWSElasticLoadBalancingLoadBalancerbastionprivatesharedipexamplecom": { - "Type": "AWS::ElasticLoadBalancing::LoadBalancer", + "AWSElasticLoadBalancingV2Listenerbastionprivatesharedipexamplecom22": { + "Type": "AWS::ElasticLoadBalancingV2::Listener", "Properties": { - "LoadBalancerName": "bastion-private-shared-ip-eepmph", - "Listeners": [ + "DefaultActions": [ { - "InstancePort": "22", - "InstanceProtocol": "TCP", - "LoadBalancerPort": "22", - "Protocol": "TCP" - } - ], - "SecurityGroups": [ - { - "Ref": "AWSEC2SecurityGroupbastionelbprivatesharedipexamplecom" + "Type": "forward", + "TargetGroupArn": { + "Ref": "AWSElasticLoadBalancingV2TargetGroupbastionprivatesharedipeepmph" + } } ], - "Subnets": [ + "LoadBalancerArn": { + "Ref": "AWSElasticLoadBalancingV2LoadBalancerbastionprivatesharedipexamplecom" + }, + "Port": 22, + "Protocol": "TCP" + } + }, + "AWSElasticLoadBalancingV2LoadBalancerbastionprivatesharedipexamplecom": { + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + "Properties": { + "Name": "bastion-private-shared-ip-eepmph", + "Scheme": "internet-facing", + "SubnetMappings": [ { - "Ref": "AWSEC2Subnetutilityustest1aprivatesharedipexamplecom" + "SubnetId": { + "Ref": "AWSEC2Subnetutilityustest1aprivatesharedipexamplecom" + } } ], - "HealthCheck": { - "Target": "TCP:22", - "HealthyThreshold": "2", - "UnhealthyThreshold": "2", - "Interval": "10", - "Timeout": "5" - }, - "ConnectionSettings": { - "IdleTimeout": 300 - }, + "Type": "network", "Tags": [ { "Key": "KubernetesCluster", @@ -1418,6 +1393,32 @@ ] } }, + "AWSElasticLoadBalancingV2TargetGroupbastionprivatesharedipeepmph": { + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", + "Properties": { + "Name": "bastion-private-shared-ip-eepmph", + "Port": 22, + "Protocol": "TCP", + "VpcId": "vpc-12345678", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "private-shared-ip.example.com" + }, + { + "Key": "Name", + "Value": "bastion-private-shared-ip-eepmph" + }, + { + "Key": "kubernetes.io/cluster/private-shared-ip.example.com", + "Value": "owned" + } + ], + "HealthCheckProtocol": "TCP", + "HealthyThresholdCount": 2, + "UnhealthyThresholdCount": 2 + } + }, "AWSIAMInstanceProfilebastionsprivatesharedipexamplecom": { "Type": "AWS::IAM::InstanceProfile", "Properties": { diff --git a/tests/integration/update_cluster/private-shared-ip/kubernetes.tf b/tests/integration/update_cluster/private-shared-ip/kubernetes.tf index 2cd3238a5cf50..c463dc29b8de3 100644 --- a/tests/integration/update_cluster/private-shared-ip/kubernetes.tf +++ b/tests/integration/update_cluster/private-shared-ip/kubernetes.tf @@ -116,7 +116,6 @@ resource "aws_autoscaling_group" "bastion-private-shared-ip-example-com" { id = aws_launch_template.bastion-private-shared-ip-example-com.id version = aws_launch_template.bastion-private-shared-ip-example-com.latest_version } - load_balancers = [aws_elb.bastion-private-shared-ip-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -158,6 +157,7 @@ resource "aws_autoscaling_group" "bastion-private-shared-ip-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-private-shared-ip-eepmph.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-private-shared-ip-example-com.id] } @@ -337,31 +337,6 @@ resource "aws_elb" "api-private-shared-ip-example-com" { } } -resource "aws_elb" "bastion-private-shared-ip-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-private-shared-ip-eepmph" - security_groups = [aws_security_group.bastion-elb-private-shared-ip-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-private-shared-ip-example-com.id] - tags = { - "KubernetesCluster" = "private-shared-ip.example.com" - "Name" = "bastion.private-shared-ip.example.com" - "kubernetes.io/cluster/private-shared-ip.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-private-shared-ip-example-com" { name = "bastions.private-shared-ip.example.com" role = aws_iam_role.bastions-private-shared-ip-example-com.name @@ -681,6 +656,49 @@ resource "aws_launch_template" "nodes-private-shared-ip-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.private-shared-ip.example.com_user_data") } +resource "aws_lb" "bastion-private-shared-ip-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-private-shared-ip-eepmph" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-private-shared-ip-example-com.id + } + tags = { + "KubernetesCluster" = "private-shared-ip.example.com" + "Name" = "bastion.private-shared-ip.example.com" + "kubernetes.io/cluster/private-shared-ip.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-private-shared-ip-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-private-shared-ip-eepmph.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-private-shared-ip-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-private-shared-ip-eepmph" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-private-shared-ip-eepmph" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "private-shared-ip.example.com" + "Name" = "bastion-private-shared-ip-eepmph" + "kubernetes.io/cluster/private-shared-ip.example.com" = "owned" + } + vpc_id = "vpc-12345678" +} + resource "aws_nat_gateway" "us-test-1a-private-shared-ip-example-com" { allocation_id = "eipalloc-12345678" subnet_id = aws_subnet.utility-us-test-1a-private-shared-ip-example-com.id @@ -889,17 +907,6 @@ resource "aws_security_group" "api-elb-private-shared-ip-example-com" { vpc_id = "vpc-12345678" } -resource "aws_security_group" "bastion-elb-private-shared-ip-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.private-shared-ip.example.com" - tags = { - "KubernetesCluster" = "private-shared-ip.example.com" - "Name" = "bastion-elb.private-shared-ip.example.com" - "kubernetes.io/cluster/private-shared-ip.example.com" = "owned" - } - vpc_id = "vpc-12345678" -} - resource "aws_security_group" "bastion-private-shared-ip-example-com" { description = "Security group for bastion" name = "bastion.private-shared-ip.example.com" @@ -933,11 +940,11 @@ resource "aws_security_group" "nodes-private-shared-ip-example-com" { vpc_id = "vpc-12345678" } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-private-shared-ip-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-private-shared-ip-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-private-shared-ip-example-com.id + security_group_id = aws_security_group.bastion-private-shared-ip-example-com.id to_port = 22 type = "ingress" } @@ -951,6 +958,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-private-shared-ip-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-private-shared-ip-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-private-shared-ip-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -969,33 +985,6 @@ resource "aws_security_group_rule" "from-api-elb-private-shared-ip-example-com-e type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-private-shared-ip-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-private-shared-ip-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-private-shared-ip-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-private-shared-ip-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-private-shared-ip-example-com-ingress-tcp-22to22-bastion-private-shared-ip-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-private-shared-ip-example-com.id - source_security_group_id = aws_security_group.bastion-elb-private-shared-ip-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-private-shared-ip-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1149,6 +1138,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-private-shared-ip-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-private-shared-ip-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-private-shared-ip-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/private-shared-subnet/kubernetes.tf b/tests/integration/update_cluster/private-shared-subnet/kubernetes.tf index fcc45ef4c395a..c5354be3c2cc2 100644 --- a/tests/integration/update_cluster/private-shared-subnet/kubernetes.tf +++ b/tests/integration/update_cluster/private-shared-subnet/kubernetes.tf @@ -111,7 +111,6 @@ resource "aws_autoscaling_group" "bastion-private-shared-subnet-example-com" { id = aws_launch_template.bastion-private-shared-subnet-example-com.id version = aws_launch_template.bastion-private-shared-subnet-example-com.latest_version } - load_balancers = [aws_elb.bastion-private-shared-subnet-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -153,6 +152,7 @@ resource "aws_autoscaling_group" "bastion-private-shared-subnet-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-private-shared-su-5ol32q.id] vpc_zone_identifier = ["subnet-abcdef"] } @@ -332,31 +332,6 @@ resource "aws_elb" "api-private-shared-subnet-example-com" { } } -resource "aws_elb" "bastion-private-shared-subnet-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-private-shared-su-5ol32q" - security_groups = [aws_security_group.bastion-elb-private-shared-subnet-example-com.id] - subnets = ["subnet-abcdef"] - tags = { - "KubernetesCluster" = "private-shared-subnet.example.com" - "Name" = "bastion.private-shared-subnet.example.com" - "kubernetes.io/cluster/private-shared-subnet.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-private-shared-subnet-example-com" { name = "bastions.private-shared-subnet.example.com" role = aws_iam_role.bastions-private-shared-subnet-example-com.name @@ -676,6 +651,49 @@ resource "aws_launch_template" "nodes-private-shared-subnet-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.private-shared-subnet.example.com_user_data") } +resource "aws_lb" "bastion-private-shared-subnet-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-private-shared-su-5ol32q" + subnet_mapping { + subnet_id = "subnet-abcdef" + } + tags = { + "KubernetesCluster" = "private-shared-subnet.example.com" + "Name" = "bastion.private-shared-subnet.example.com" + "kubernetes.io/cluster/private-shared-subnet.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-private-shared-subnet-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-private-shared-su-5ol32q.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-private-shared-subnet-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-private-shared-su-5ol32q" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-private-shared-su-5ol32q" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "private-shared-subnet.example.com" + "Name" = "bastion-private-shared-su-5ol32q" + "kubernetes.io/cluster/private-shared-subnet.example.com" = "owned" + } + vpc_id = "vpc-12345678" +} + resource "aws_route53_record" "api-private-shared-subnet-example-com" { alias { evaluate_target_health = false @@ -826,17 +844,6 @@ resource "aws_security_group" "api-elb-private-shared-subnet-example-com" { vpc_id = "vpc-12345678" } -resource "aws_security_group" "bastion-elb-private-shared-subnet-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.private-shared-subnet.example.com" - tags = { - "KubernetesCluster" = "private-shared-subnet.example.com" - "Name" = "bastion-elb.private-shared-subnet.example.com" - "kubernetes.io/cluster/private-shared-subnet.example.com" = "owned" - } - vpc_id = "vpc-12345678" -} - resource "aws_security_group" "bastion-private-shared-subnet-example-com" { description = "Security group for bastion" name = "bastion.private-shared-subnet.example.com" @@ -870,11 +877,11 @@ resource "aws_security_group" "nodes-private-shared-subnet-example-com" { vpc_id = "vpc-12345678" } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-private-shared-subnet-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-private-shared-subnet-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-private-shared-subnet-example-com.id + security_group_id = aws_security_group.bastion-private-shared-subnet-example-com.id to_port = 22 type = "ingress" } @@ -888,6 +895,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-private-shared-subnet-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-private-shared-subnet-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-private-shared-subnet-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -906,33 +922,6 @@ resource "aws_security_group_rule" "from-api-elb-private-shared-subnet-example-c type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-private-shared-subnet-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-private-shared-subnet-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-private-shared-subnet-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-private-shared-subnet-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-private-shared-subnet-example-com-ingress-tcp-22to22-bastion-private-shared-subnet-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-private-shared-subnet-example-com.id - source_security_group_id = aws_security_group.bastion-elb-private-shared-subnet-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-private-shared-subnet-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1086,6 +1075,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-private-shared-subnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-private-shared-subnet-example-com.id + to_port = 4 + type = "ingress" +} + terraform { required_version = ">= 0.15.0" required_providers { diff --git a/tests/integration/update_cluster/privatecalico/kubernetes.tf b/tests/integration/update_cluster/privatecalico/kubernetes.tf index 967dfa7987872..4db1eaa660ef0 100644 --- a/tests/integration/update_cluster/privatecalico/kubernetes.tf +++ b/tests/integration/update_cluster/privatecalico/kubernetes.tf @@ -121,7 +121,6 @@ resource "aws_autoscaling_group" "bastion-privatecalico-example-com" { id = aws_launch_template.bastion-privatecalico-example-com.id version = aws_launch_template.bastion-privatecalico-example-com.latest_version } - load_balancers = [aws_elb.bastion-privatecalico-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -158,6 +157,7 @@ resource "aws_autoscaling_group" "bastion-privatecalico-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privatecalico-exa-hocohm.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privatecalico-example-com.id] } @@ -331,31 +331,6 @@ resource "aws_elb" "api-privatecalico-example-com" { } } -resource "aws_elb" "bastion-privatecalico-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privatecalico-exa-hocohm" - security_groups = [aws_security_group.bastion-elb-privatecalico-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privatecalico-example-com.id] - tags = { - "KubernetesCluster" = "privatecalico.example.com" - "Name" = "bastion.privatecalico.example.com" - "kubernetes.io/cluster/privatecalico.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privatecalico-example-com" { name = "bastions.privatecalico.example.com" role = aws_iam_role.bastions-privatecalico-example-com.name @@ -668,6 +643,49 @@ resource "aws_launch_template" "nodes-privatecalico-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privatecalico.example.com_user_data") } +resource "aws_lb" "bastion-privatecalico-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privatecalico-exa-hocohm" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privatecalico-example-com.id + } + tags = { + "KubernetesCluster" = "privatecalico.example.com" + "Name" = "bastion.privatecalico.example.com" + "kubernetes.io/cluster/privatecalico.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privatecalico-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privatecalico-exa-hocohm.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privatecalico-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privatecalico-exa-hocohm" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privatecalico-exa-hocohm" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privatecalico.example.com" + "Name" = "bastion-privatecalico-exa-hocohm" + "kubernetes.io/cluster/privatecalico.example.com" = "owned" + } + vpc_id = aws_vpc.privatecalico-example-com.id +} + resource "aws_nat_gateway" "us-test-1a-privatecalico-example-com" { allocation_id = aws_eip.us-test-1a-privatecalico-example-com.id subnet_id = aws_subnet.utility-us-test-1a-privatecalico-example-com.id @@ -908,17 +926,6 @@ resource "aws_security_group" "api-elb-privatecalico-example-com" { vpc_id = aws_vpc.privatecalico-example-com.id } -resource "aws_security_group" "bastion-elb-privatecalico-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privatecalico.example.com" - tags = { - "KubernetesCluster" = "privatecalico.example.com" - "Name" = "bastion-elb.privatecalico.example.com" - "kubernetes.io/cluster/privatecalico.example.com" = "owned" - } - vpc_id = aws_vpc.privatecalico-example-com.id -} - resource "aws_security_group" "bastion-privatecalico-example-com" { description = "Security group for bastion" name = "bastion.privatecalico.example.com" @@ -952,11 +959,11 @@ resource "aws_security_group" "nodes-privatecalico-example-com" { vpc_id = aws_vpc.privatecalico-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privatecalico-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privatecalico-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privatecalico-example-com.id + security_group_id = aws_security_group.bastion-privatecalico-example-com.id to_port = 22 type = "ingress" } @@ -970,11 +977,20 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } -resource "aws_security_group_rule" "from-__--0-ingress-tcp-22to22-bastion-elb-privatecalico-example-com" { +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privatecalico-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatecalico-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-__--0-ingress-tcp-22to22-bastion-privatecalico-example-com" { from_port = 22 ipv6_cidr_blocks = ["::/0"] protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privatecalico-example-com.id + security_group_id = aws_security_group.bastion-privatecalico-example-com.id to_port = 22 type = "ingress" } @@ -1006,33 +1022,6 @@ resource "aws_security_group_rule" "from-api-elb-privatecalico-example-com-egres type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privatecalico-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatecalico-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatecalico-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatecalico-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatecalico-example-com-ingress-tcp-22to22-bastion-privatecalico-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privatecalico-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privatecalico-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privatecalico-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1195,6 +1184,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatecalico-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatecalico-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_security_group_rule" "icmpv6-pmtu-api-elb-__--0" { from_port = -1 ipv6_cidr_blocks = ["::/0"] @@ -1204,6 +1211,15 @@ resource "aws_security_group_rule" "icmpv6-pmtu-api-elb-__--0" { type = "ingress" } +resource "aws_security_group_rule" "icmpv6-pmtu-ssh-nlb-__--0" { + from_port = -1 + ipv6_cidr_blocks = ["::/0"] + protocol = "icmpv6" + security_group_id = aws_security_group.bastion-privatecalico-example-com.id + to_port = -1 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privatecalico-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/privatecanal/kubernetes.tf b/tests/integration/update_cluster/privatecanal/kubernetes.tf index 15b1d13fb5cd0..3d1d749569e64 100644 --- a/tests/integration/update_cluster/privatecanal/kubernetes.tf +++ b/tests/integration/update_cluster/privatecanal/kubernetes.tf @@ -121,7 +121,6 @@ resource "aws_autoscaling_group" "bastion-privatecanal-example-com" { id = aws_launch_template.bastion-privatecanal-example-com.id version = aws_launch_template.bastion-privatecanal-example-com.latest_version } - load_balancers = [aws_elb.bastion-privatecanal-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -158,6 +157,7 @@ resource "aws_autoscaling_group" "bastion-privatecanal-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privatecanal-exam-hmhsp5.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privatecanal-example-com.id] } @@ -331,31 +331,6 @@ resource "aws_elb" "api-privatecanal-example-com" { } } -resource "aws_elb" "bastion-privatecanal-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privatecanal-exam-hmhsp5" - security_groups = [aws_security_group.bastion-elb-privatecanal-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privatecanal-example-com.id] - tags = { - "KubernetesCluster" = "privatecanal.example.com" - "Name" = "bastion.privatecanal.example.com" - "kubernetes.io/cluster/privatecanal.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privatecanal-example-com" { name = "bastions.privatecanal.example.com" role = aws_iam_role.bastions-privatecanal-example-com.name @@ -672,6 +647,49 @@ resource "aws_launch_template" "nodes-privatecanal-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privatecanal.example.com_user_data") } +resource "aws_lb" "bastion-privatecanal-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privatecanal-exam-hmhsp5" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privatecanal-example-com.id + } + tags = { + "KubernetesCluster" = "privatecanal.example.com" + "Name" = "bastion.privatecanal.example.com" + "kubernetes.io/cluster/privatecanal.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privatecanal-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privatecanal-exam-hmhsp5.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privatecanal-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privatecanal-exam-hmhsp5" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privatecanal-exam-hmhsp5" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privatecanal.example.com" + "Name" = "bastion-privatecanal-exam-hmhsp5" + "kubernetes.io/cluster/privatecanal.example.com" = "owned" + } + vpc_id = aws_vpc.privatecanal-example-com.id +} + resource "aws_nat_gateway" "us-test-1a-privatecanal-example-com" { allocation_id = aws_eip.us-test-1a-privatecanal-example-com.id subnet_id = aws_subnet.utility-us-test-1a-privatecanal-example-com.id @@ -912,17 +930,6 @@ resource "aws_security_group" "api-elb-privatecanal-example-com" { vpc_id = aws_vpc.privatecanal-example-com.id } -resource "aws_security_group" "bastion-elb-privatecanal-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privatecanal.example.com" - tags = { - "KubernetesCluster" = "privatecanal.example.com" - "Name" = "bastion-elb.privatecanal.example.com" - "kubernetes.io/cluster/privatecanal.example.com" = "owned" - } - vpc_id = aws_vpc.privatecanal-example-com.id -} - resource "aws_security_group" "bastion-privatecanal-example-com" { description = "Security group for bastion" name = "bastion.privatecanal.example.com" @@ -956,11 +963,11 @@ resource "aws_security_group" "nodes-privatecanal-example-com" { vpc_id = aws_vpc.privatecanal-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privatecanal-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privatecanal-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privatecanal-example-com.id + security_group_id = aws_security_group.bastion-privatecanal-example-com.id to_port = 22 type = "ingress" } @@ -974,6 +981,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privatecanal-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatecanal-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-privatecanal-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -992,33 +1008,6 @@ resource "aws_security_group_rule" "from-api-elb-privatecanal-example-com-egress type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privatecanal-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatecanal-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatecanal-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatecanal-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatecanal-example-com-ingress-tcp-22to22-bastion-privatecanal-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privatecanal-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privatecanal-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privatecanal-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1172,6 +1161,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatecanal-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatecanal-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privatecanal-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/privatecilium/cloudformation.json b/tests/integration/update_cluster/privatecilium/cloudformation.json index eb481b149a235..70eeb034353d9 100644 --- a/tests/integration/update_cluster/privatecilium/cloudformation.json +++ b/tests/integration/update_cluster/privatecilium/cloudformation.json @@ -74,9 +74,9 @@ ] } ], - "LoadBalancerNames": [ + "TargetGroupARNs": [ { - "Ref": "AWSElasticLoadBalancingLoadBalancerbastionprivateciliumexamplecom" + "Ref": "AWSElasticLoadBalancingV2TargetGroupbastionprivateciliumexal2ms01" } ] } @@ -836,30 +836,6 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupEgressfrombastionelbprivateciliumexamplecomegressall0to00": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIpv6": "::/0" - } - }, - "AWSEC2SecurityGroupEgressfrombastionelbprivateciliumexamplecomegressall0to000000": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, "AWSEC2SecurityGroupEgressfrombastionprivateciliumexamplecomegressall0to00": { "Type": "AWS::EC2::SecurityGroupEgress", "Properties": { @@ -932,11 +908,11 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22bastionelbprivateciliumexamplecom": { + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22bastionprivateciliumexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" + "Ref": "AWSEC2SecurityGroupbastionprivateciliumexamplecom" }, "FromPort": 22, "ToPort": 22, @@ -956,18 +932,16 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrombastionelbprivateciliumexamplecomingresstcp22to22bastionprivateciliumexamplecom": { + "AWSEC2SecurityGroupIngressfrom172204022ingresstcp22to22bastionprivateciliumexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { "Ref": "AWSEC2SecurityGroupbastionprivateciliumexamplecom" }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" - }, "FromPort": 22, "ToPort": 22, - "IpProtocol": "tcp" + "IpProtocol": "tcp", + "CidrIp": "172.20.4.0/22" } }, "AWSEC2SecurityGroupIngressfrombastionprivateciliumexamplecomingresstcp22to22mastersprivateciliumexamplecom": { @@ -1122,38 +1096,38 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupapielbprivateciliumexamplecom": { - "Type": "AWS::EC2::SecurityGroup", + "AWSEC2SecurityGroupIngressicmppmtusshnlb00000": { + "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { - "GroupName": "api-elb.privatecilium.example.com", - "VpcId": { - "Ref": "AWSEC2VPCprivateciliumexamplecom" + "GroupId": { + "Ref": "AWSEC2SecurityGroupbastionprivateciliumexamplecom" }, - "GroupDescription": "Security group for api ELB", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "privatecilium.example.com" - }, - { - "Key": "Name", - "Value": "api-elb.privatecilium.example.com" - }, - { - "Key": "kubernetes.io/cluster/privatecilium.example.com", - "Value": "owned" - } - ] + "FromPort": 3, + "ToPort": 4, + "IpProtocol": "icmp", + "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom": { + "AWSEC2SecurityGroupIngressicmppmtusshnlb172204022": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupbastionprivateciliumexamplecom" + }, + "FromPort": 3, + "ToPort": 4, + "IpProtocol": "icmp", + "CidrIp": "172.20.4.0/22" + } + }, + "AWSEC2SecurityGroupapielbprivateciliumexamplecom": { "Type": "AWS::EC2::SecurityGroup", "Properties": { - "GroupName": "bastion-elb.privatecilium.example.com", + "GroupName": "api-elb.privatecilium.example.com", "VpcId": { "Ref": "AWSEC2VPCprivateciliumexamplecom" }, - "GroupDescription": "Security group for bastion ELB", + "GroupDescription": "Security group for api ELB", "Tags": [ { "Key": "KubernetesCluster", @@ -1161,7 +1135,7 @@ }, { "Key": "Name", - "Value": "bastion-elb.privatecilium.example.com" + "Value": "api-elb.privatecilium.example.com" }, { "Key": "kubernetes.io/cluster/privatecilium.example.com", @@ -1512,37 +1486,61 @@ ] } }, - "AWSElasticLoadBalancingLoadBalancerbastionprivateciliumexamplecom": { - "Type": "AWS::ElasticLoadBalancing::LoadBalancer", + "AWSElasticLoadBalancingV2Listenerbastionprivateciliumexamplecom22": { + "Type": "AWS::ElasticLoadBalancingV2::Listener", "Properties": { - "LoadBalancerName": "bastion-privatecilium-exa-l2ms01", - "Listeners": [ + "DefaultActions": [ { - "InstancePort": "22", - "InstanceProtocol": "TCP", - "LoadBalancerPort": "22", - "Protocol": "TCP" + "Type": "forward", + "TargetGroupArn": { + "Ref": "AWSElasticLoadBalancingV2TargetGroupbastionprivateciliumexal2ms01" + } } ], - "SecurityGroups": [ + "LoadBalancerArn": { + "Ref": "AWSElasticLoadBalancingV2LoadBalancerbastionprivateciliumexamplecom" + }, + "Port": 22, + "Protocol": "TCP" + } + }, + "AWSElasticLoadBalancingV2LoadBalancerbastionprivateciliumexamplecom": { + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + "Properties": { + "Name": "bastion-privatecilium-exa-l2ms01", + "Scheme": "internet-facing", + "SubnetMappings": [ { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" + "SubnetId": { + "Ref": "AWSEC2Subnetutilityustest1aprivateciliumexamplecom" + } } ], - "Subnets": [ + "Type": "network", + "Tags": [ { - "Ref": "AWSEC2Subnetutilityustest1aprivateciliumexamplecom" + "Key": "KubernetesCluster", + "Value": "privatecilium.example.com" + }, + { + "Key": "Name", + "Value": "bastion.privatecilium.example.com" + }, + { + "Key": "kubernetes.io/cluster/privatecilium.example.com", + "Value": "owned" } - ], - "HealthCheck": { - "Target": "TCP:22", - "HealthyThreshold": "2", - "UnhealthyThreshold": "2", - "Interval": "10", - "Timeout": "5" - }, - "ConnectionSettings": { - "IdleTimeout": 300 + ] + } + }, + "AWSElasticLoadBalancingV2TargetGroupbastionprivateciliumexal2ms01": { + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", + "Properties": { + "Name": "bastion-privatecilium-exa-l2ms01", + "Port": 22, + "Protocol": "TCP", + "VpcId": { + "Ref": "AWSEC2VPCprivateciliumexamplecom" }, "Tags": [ { @@ -1551,13 +1549,16 @@ }, { "Key": "Name", - "Value": "bastion.privatecilium.example.com" + "Value": "bastion-privatecilium-exa-l2ms01" }, { "Key": "kubernetes.io/cluster/privatecilium.example.com", "Value": "owned" } - ] + ], + "HealthCheckProtocol": "TCP", + "HealthyThresholdCount": 2, + "UnhealthyThresholdCount": 2 } }, "AWSIAMInstanceProfilebastionsprivateciliumexamplecom": { diff --git a/tests/integration/update_cluster/privatecilium/kubernetes.tf b/tests/integration/update_cluster/privatecilium/kubernetes.tf index 4751a009e53ec..3edaa4532f51f 100644 --- a/tests/integration/update_cluster/privatecilium/kubernetes.tf +++ b/tests/integration/update_cluster/privatecilium/kubernetes.tf @@ -121,7 +121,6 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" { id = aws_launch_template.bastion-privatecilium-example-com.id version = aws_launch_template.bastion-privatecilium-example-com.latest_version } - load_balancers = [aws_elb.bastion-privatecilium-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -163,6 +162,7 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privatecilium-exa-l2ms01.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privatecilium-example-com.id] } @@ -351,31 +351,6 @@ resource "aws_elb" "api-privatecilium-example-com" { } } -resource "aws_elb" "bastion-privatecilium-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privatecilium-exa-l2ms01" - security_groups = [aws_security_group.bastion-elb-privatecilium-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privatecilium-example-com.id] - tags = { - "KubernetesCluster" = "privatecilium.example.com" - "Name" = "bastion.privatecilium.example.com" - "kubernetes.io/cluster/privatecilium.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privatecilium-example-com" { name = "bastions.privatecilium.example.com" role = aws_iam_role.bastions-privatecilium-example-com.name @@ -704,6 +679,49 @@ resource "aws_launch_template" "nodes-privatecilium-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privatecilium.example.com_user_data") } +resource "aws_lb" "bastion-privatecilium-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privatecilium-exa-l2ms01" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privatecilium-example-com.id + } + tags = { + "KubernetesCluster" = "privatecilium.example.com" + "Name" = "bastion.privatecilium.example.com" + "kubernetes.io/cluster/privatecilium.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privatecilium-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privatecilium-exa-l2ms01.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privatecilium-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privatecilium-exa-l2ms01" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privatecilium-exa-l2ms01" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privatecilium.example.com" + "Name" = "bastion-privatecilium-exa-l2ms01" + "kubernetes.io/cluster/privatecilium.example.com" = "owned" + } + vpc_id = aws_vpc.privatecilium-example-com.id +} + resource "aws_nat_gateway" "us-test-1a-privatecilium-example-com" { allocation_id = aws_eip.us-test-1a-privatecilium-example-com.id subnet_id = aws_subnet.utility-us-test-1a-privatecilium-example-com.id @@ -920,17 +938,6 @@ resource "aws_security_group" "api-elb-privatecilium-example-com" { vpc_id = aws_vpc.privatecilium-example-com.id } -resource "aws_security_group" "bastion-elb-privatecilium-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privatecilium.example.com" - tags = { - "KubernetesCluster" = "privatecilium.example.com" - "Name" = "bastion-elb.privatecilium.example.com" - "kubernetes.io/cluster/privatecilium.example.com" = "owned" - } - vpc_id = aws_vpc.privatecilium-example-com.id -} - resource "aws_security_group" "bastion-privatecilium-example-com" { description = "Security group for bastion" name = "bastion.privatecilium.example.com" @@ -964,11 +971,11 @@ resource "aws_security_group" "nodes-privatecilium-example-com" { vpc_id = aws_vpc.privatecilium-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privatecilium-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privatecilium-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privatecilium-example-com.id + security_group_id = aws_security_group.bastion-privatecilium-example-com.id to_port = 22 type = "ingress" } @@ -982,6 +989,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privatecilium-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatecilium-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-privatecilium-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1000,33 +1016,6 @@ resource "aws_security_group_rule" "from-api-elb-privatecilium-example-com-egres type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privatecilium-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatecilium-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatecilium-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatecilium-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatecilium-example-com-ingress-tcp-22to22-bastion-privatecilium-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privatecilium-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privatecilium-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privatecilium-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1180,6 +1169,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatecilium-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatecilium-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privatecilium-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/privatecilium2/cloudformation.json b/tests/integration/update_cluster/privatecilium2/cloudformation.json index a3747771b9b8e..1b5cc5438a483 100644 --- a/tests/integration/update_cluster/privatecilium2/cloudformation.json +++ b/tests/integration/update_cluster/privatecilium2/cloudformation.json @@ -69,9 +69,9 @@ ] } ], - "LoadBalancerNames": [ + "TargetGroupARNs": [ { - "Ref": "AWSElasticLoadBalancingLoadBalancerbastionprivateciliumexamplecom" + "Ref": "AWSElasticLoadBalancingV2TargetGroupbastionprivateciliumexal2ms01" } ] } @@ -784,30 +784,6 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupEgressfrombastionelbprivateciliumexamplecomegressall0to00": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIpv6": "::/0" - } - }, - "AWSEC2SecurityGroupEgressfrombastionelbprivateciliumexamplecomegressall0to000000": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, "AWSEC2SecurityGroupEgressfrombastionprivateciliumexamplecomegressall0to00": { "Type": "AWS::EC2::SecurityGroupEgress", "Properties": { @@ -880,11 +856,11 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22bastionelbprivateciliumexamplecom": { + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22bastionprivateciliumexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" + "Ref": "AWSEC2SecurityGroupbastionprivateciliumexamplecom" }, "FromPort": 22, "ToPort": 22, @@ -904,18 +880,16 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrombastionelbprivateciliumexamplecomingresstcp22to22bastionprivateciliumexamplecom": { + "AWSEC2SecurityGroupIngressfrom172204022ingresstcp22to22bastionprivateciliumexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { "Ref": "AWSEC2SecurityGroupbastionprivateciliumexamplecom" }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" - }, "FromPort": 22, "ToPort": 22, - "IpProtocol": "tcp" + "IpProtocol": "tcp", + "CidrIp": "172.20.4.0/22" } }, "AWSEC2SecurityGroupIngressfrombastionprivateciliumexamplecomingresstcp22to22mastersprivateciliumexamplecom": { @@ -1070,38 +1044,38 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupapielbprivateciliumexamplecom": { - "Type": "AWS::EC2::SecurityGroup", + "AWSEC2SecurityGroupIngressicmppmtusshnlb00000": { + "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { - "GroupName": "api-elb.privatecilium.example.com", - "VpcId": { - "Ref": "AWSEC2VPCprivateciliumexamplecom" + "GroupId": { + "Ref": "AWSEC2SecurityGroupbastionprivateciliumexamplecom" }, - "GroupDescription": "Security group for api ELB", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "privatecilium.example.com" - }, - { - "Key": "Name", - "Value": "api-elb.privatecilium.example.com" - }, - { - "Key": "kubernetes.io/cluster/privatecilium.example.com", - "Value": "owned" - } - ] + "FromPort": 3, + "ToPort": 4, + "IpProtocol": "icmp", + "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom": { + "AWSEC2SecurityGroupIngressicmppmtusshnlb172204022": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupbastionprivateciliumexamplecom" + }, + "FromPort": 3, + "ToPort": 4, + "IpProtocol": "icmp", + "CidrIp": "172.20.4.0/22" + } + }, + "AWSEC2SecurityGroupapielbprivateciliumexamplecom": { "Type": "AWS::EC2::SecurityGroup", "Properties": { - "GroupName": "bastion-elb.privatecilium.example.com", + "GroupName": "api-elb.privatecilium.example.com", "VpcId": { "Ref": "AWSEC2VPCprivateciliumexamplecom" }, - "GroupDescription": "Security group for bastion ELB", + "GroupDescription": "Security group for api ELB", "Tags": [ { "Key": "KubernetesCluster", @@ -1109,7 +1083,7 @@ }, { "Key": "Name", - "Value": "bastion-elb.privatecilium.example.com" + "Value": "api-elb.privatecilium.example.com" }, { "Key": "kubernetes.io/cluster/privatecilium.example.com", @@ -1460,37 +1434,61 @@ ] } }, - "AWSElasticLoadBalancingLoadBalancerbastionprivateciliumexamplecom": { - "Type": "AWS::ElasticLoadBalancing::LoadBalancer", + "AWSElasticLoadBalancingV2Listenerbastionprivateciliumexamplecom22": { + "Type": "AWS::ElasticLoadBalancingV2::Listener", "Properties": { - "LoadBalancerName": "bastion-privatecilium-exa-l2ms01", - "Listeners": [ + "DefaultActions": [ { - "InstancePort": "22", - "InstanceProtocol": "TCP", - "LoadBalancerPort": "22", - "Protocol": "TCP" + "Type": "forward", + "TargetGroupArn": { + "Ref": "AWSElasticLoadBalancingV2TargetGroupbastionprivateciliumexal2ms01" + } } ], - "SecurityGroups": [ + "LoadBalancerArn": { + "Ref": "AWSElasticLoadBalancingV2LoadBalancerbastionprivateciliumexamplecom" + }, + "Port": 22, + "Protocol": "TCP" + } + }, + "AWSElasticLoadBalancingV2LoadBalancerbastionprivateciliumexamplecom": { + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + "Properties": { + "Name": "bastion-privatecilium-exa-l2ms01", + "Scheme": "internet-facing", + "SubnetMappings": [ { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumexamplecom" + "SubnetId": { + "Ref": "AWSEC2Subnetutilityustest1aprivateciliumexamplecom" + } } ], - "Subnets": [ + "Type": "network", + "Tags": [ { - "Ref": "AWSEC2Subnetutilityustest1aprivateciliumexamplecom" + "Key": "KubernetesCluster", + "Value": "privatecilium.example.com" + }, + { + "Key": "Name", + "Value": "bastion.privatecilium.example.com" + }, + { + "Key": "kubernetes.io/cluster/privatecilium.example.com", + "Value": "owned" } - ], - "HealthCheck": { - "Target": "TCP:22", - "HealthyThreshold": "2", - "UnhealthyThreshold": "2", - "Interval": "10", - "Timeout": "5" - }, - "ConnectionSettings": { - "IdleTimeout": 300 + ] + } + }, + "AWSElasticLoadBalancingV2TargetGroupbastionprivateciliumexal2ms01": { + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", + "Properties": { + "Name": "bastion-privatecilium-exa-l2ms01", + "Port": 22, + "Protocol": "TCP", + "VpcId": { + "Ref": "AWSEC2VPCprivateciliumexamplecom" }, "Tags": [ { @@ -1499,13 +1497,16 @@ }, { "Key": "Name", - "Value": "bastion.privatecilium.example.com" + "Value": "bastion-privatecilium-exa-l2ms01" }, { "Key": "kubernetes.io/cluster/privatecilium.example.com", "Value": "owned" } - ] + ], + "HealthCheckProtocol": "TCP", + "HealthyThresholdCount": 2, + "UnhealthyThresholdCount": 2 } }, "AWSIAMInstanceProfilebastionsprivateciliumexamplecom": { diff --git a/tests/integration/update_cluster/privatecilium2/kubernetes.tf b/tests/integration/update_cluster/privatecilium2/kubernetes.tf index 0ce6c9573d7db..80dacc73cada5 100644 --- a/tests/integration/update_cluster/privatecilium2/kubernetes.tf +++ b/tests/integration/update_cluster/privatecilium2/kubernetes.tf @@ -121,7 +121,6 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" { id = aws_launch_template.bastion-privatecilium-example-com.id version = aws_launch_template.bastion-privatecilium-example-com.latest_version } - load_balancers = [aws_elb.bastion-privatecilium-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -158,6 +157,7 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privatecilium-exa-l2ms01.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privatecilium-example-com.id] } @@ -331,31 +331,6 @@ resource "aws_elb" "api-privatecilium-example-com" { } } -resource "aws_elb" "bastion-privatecilium-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privatecilium-exa-l2ms01" - security_groups = [aws_security_group.bastion-elb-privatecilium-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privatecilium-example-com.id] - tags = { - "KubernetesCluster" = "privatecilium.example.com" - "Name" = "bastion.privatecilium.example.com" - "kubernetes.io/cluster/privatecilium.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privatecilium-example-com" { name = "bastions.privatecilium.example.com" role = aws_iam_role.bastions-privatecilium-example-com.name @@ -672,6 +647,49 @@ resource "aws_launch_template" "nodes-privatecilium-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privatecilium.example.com_user_data") } +resource "aws_lb" "bastion-privatecilium-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privatecilium-exa-l2ms01" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privatecilium-example-com.id + } + tags = { + "KubernetesCluster" = "privatecilium.example.com" + "Name" = "bastion.privatecilium.example.com" + "kubernetes.io/cluster/privatecilium.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privatecilium-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privatecilium-exa-l2ms01.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privatecilium-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privatecilium-exa-l2ms01" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privatecilium-exa-l2ms01" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privatecilium.example.com" + "Name" = "bastion-privatecilium-exa-l2ms01" + "kubernetes.io/cluster/privatecilium.example.com" = "owned" + } + vpc_id = aws_vpc.privatecilium-example-com.id +} + resource "aws_nat_gateway" "us-test-1a-privatecilium-example-com" { allocation_id = aws_eip.us-test-1a-privatecilium-example-com.id subnet_id = aws_subnet.utility-us-test-1a-privatecilium-example-com.id @@ -920,17 +938,6 @@ resource "aws_security_group" "api-elb-privatecilium-example-com" { vpc_id = aws_vpc.privatecilium-example-com.id } -resource "aws_security_group" "bastion-elb-privatecilium-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privatecilium.example.com" - tags = { - "KubernetesCluster" = "privatecilium.example.com" - "Name" = "bastion-elb.privatecilium.example.com" - "kubernetes.io/cluster/privatecilium.example.com" = "owned" - } - vpc_id = aws_vpc.privatecilium-example-com.id -} - resource "aws_security_group" "bastion-privatecilium-example-com" { description = "Security group for bastion" name = "bastion.privatecilium.example.com" @@ -964,11 +971,11 @@ resource "aws_security_group" "nodes-privatecilium-example-com" { vpc_id = aws_vpc.privatecilium-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privatecilium-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privatecilium-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privatecilium-example-com.id + security_group_id = aws_security_group.bastion-privatecilium-example-com.id to_port = 22 type = "ingress" } @@ -982,6 +989,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privatecilium-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatecilium-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-privatecilium-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1000,33 +1016,6 @@ resource "aws_security_group_rule" "from-api-elb-privatecilium-example-com-egres type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privatecilium-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatecilium-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatecilium-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatecilium-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatecilium-example-com-ingress-tcp-22to22-bastion-privatecilium-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privatecilium-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privatecilium-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privatecilium-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1180,6 +1169,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatecilium-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatecilium-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privatecilium-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json b/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json index a35b78fcea9c8..1f471b34aa7d9 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json +++ b/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json @@ -74,9 +74,9 @@ ] } ], - "LoadBalancerNames": [ + "TargetGroupARNs": [ { - "Ref": "AWSElasticLoadBalancingLoadBalancerbastionprivateciliumadvancedexamplecom" + "Ref": "AWSElasticLoadBalancingV2TargetGroupbastionprivateciliumadva0jni40" } ] } @@ -836,30 +836,6 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupEgressfrombastionelbprivateciliumadvancedexamplecomegressall0to00": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumadvancedexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIpv6": "::/0" - } - }, - "AWSEC2SecurityGroupEgressfrombastionelbprivateciliumadvancedexamplecomegressall0to000000": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumadvancedexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, "AWSEC2SecurityGroupEgressfrombastionprivateciliumadvancedexamplecomegressall0to00": { "Type": "AWS::EC2::SecurityGroupEgress", "Properties": { @@ -932,11 +908,11 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22bastionelbprivateciliumadvancedexamplecom": { + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22bastionprivateciliumadvancedexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumadvancedexamplecom" + "Ref": "AWSEC2SecurityGroupbastionprivateciliumadvancedexamplecom" }, "FromPort": 22, "ToPort": 22, @@ -956,18 +932,16 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrombastionelbprivateciliumadvancedexamplecomingresstcp22to22bastionprivateciliumadvancedexamplecom": { + "AWSEC2SecurityGroupIngressfrom172204022ingresstcp22to22bastionprivateciliumadvancedexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { "Ref": "AWSEC2SecurityGroupbastionprivateciliumadvancedexamplecom" }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumadvancedexamplecom" - }, "FromPort": 22, "ToPort": 22, - "IpProtocol": "tcp" + "IpProtocol": "tcp", + "CidrIp": "172.20.4.0/22" } }, "AWSEC2SecurityGroupIngressfrombastionprivateciliumadvancedexamplecomingresstcp22to22mastersprivateciliumadvancedexamplecom": { @@ -1122,38 +1096,38 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupapielbprivateciliumadvancedexamplecom": { - "Type": "AWS::EC2::SecurityGroup", + "AWSEC2SecurityGroupIngressicmppmtusshnlb00000": { + "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { - "GroupName": "api-elb.privateciliumadvanced.example.com", - "VpcId": { - "Ref": "AWSEC2VPCprivateciliumadvancedexamplecom" + "GroupId": { + "Ref": "AWSEC2SecurityGroupbastionprivateciliumadvancedexamplecom" }, - "GroupDescription": "Security group for api ELB", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "privateciliumadvanced.example.com" - }, - { - "Key": "Name", - "Value": "api-elb.privateciliumadvanced.example.com" - }, - { - "Key": "kubernetes.io/cluster/privateciliumadvanced.example.com", - "Value": "owned" - } - ] + "FromPort": 3, + "ToPort": 4, + "IpProtocol": "icmp", + "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupbastionelbprivateciliumadvancedexamplecom": { + "AWSEC2SecurityGroupIngressicmppmtusshnlb172204022": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupbastionprivateciliumadvancedexamplecom" + }, + "FromPort": 3, + "ToPort": 4, + "IpProtocol": "icmp", + "CidrIp": "172.20.4.0/22" + } + }, + "AWSEC2SecurityGroupapielbprivateciliumadvancedexamplecom": { "Type": "AWS::EC2::SecurityGroup", "Properties": { - "GroupName": "bastion-elb.privateciliumadvanced.example.com", + "GroupName": "api-elb.privateciliumadvanced.example.com", "VpcId": { "Ref": "AWSEC2VPCprivateciliumadvancedexamplecom" }, - "GroupDescription": "Security group for bastion ELB", + "GroupDescription": "Security group for api ELB", "Tags": [ { "Key": "KubernetesCluster", @@ -1161,7 +1135,7 @@ }, { "Key": "Name", - "Value": "bastion-elb.privateciliumadvanced.example.com" + "Value": "api-elb.privateciliumadvanced.example.com" }, { "Key": "kubernetes.io/cluster/privateciliumadvanced.example.com", @@ -1545,37 +1519,61 @@ ] } }, - "AWSElasticLoadBalancingLoadBalancerbastionprivateciliumadvancedexamplecom": { - "Type": "AWS::ElasticLoadBalancing::LoadBalancer", + "AWSElasticLoadBalancingV2Listenerbastionprivateciliumadvancedexamplecom22": { + "Type": "AWS::ElasticLoadBalancingV2::Listener", "Properties": { - "LoadBalancerName": "bastion-privateciliumadva-0jni40", - "Listeners": [ + "DefaultActions": [ { - "InstancePort": "22", - "InstanceProtocol": "TCP", - "LoadBalancerPort": "22", - "Protocol": "TCP" + "Type": "forward", + "TargetGroupArn": { + "Ref": "AWSElasticLoadBalancingV2TargetGroupbastionprivateciliumadva0jni40" + } } ], - "SecurityGroups": [ + "LoadBalancerArn": { + "Ref": "AWSElasticLoadBalancingV2LoadBalancerbastionprivateciliumadvancedexamplecom" + }, + "Port": 22, + "Protocol": "TCP" + } + }, + "AWSElasticLoadBalancingV2LoadBalancerbastionprivateciliumadvancedexamplecom": { + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + "Properties": { + "Name": "bastion-privateciliumadva-0jni40", + "Scheme": "internet-facing", + "SubnetMappings": [ { - "Ref": "AWSEC2SecurityGroupbastionelbprivateciliumadvancedexamplecom" + "SubnetId": { + "Ref": "AWSEC2Subnetutilityustest1aprivateciliumadvancedexamplecom" + } } ], - "Subnets": [ + "Type": "network", + "Tags": [ { - "Ref": "AWSEC2Subnetutilityustest1aprivateciliumadvancedexamplecom" + "Key": "KubernetesCluster", + "Value": "privateciliumadvanced.example.com" + }, + { + "Key": "Name", + "Value": "bastion.privateciliumadvanced.example.com" + }, + { + "Key": "kubernetes.io/cluster/privateciliumadvanced.example.com", + "Value": "owned" } - ], - "HealthCheck": { - "Target": "TCP:22", - "HealthyThreshold": "2", - "UnhealthyThreshold": "2", - "Interval": "10", - "Timeout": "5" - }, - "ConnectionSettings": { - "IdleTimeout": 300 + ] + } + }, + "AWSElasticLoadBalancingV2TargetGroupbastionprivateciliumadva0jni40": { + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", + "Properties": { + "Name": "bastion-privateciliumadva-0jni40", + "Port": 22, + "Protocol": "TCP", + "VpcId": { + "Ref": "AWSEC2VPCprivateciliumadvancedexamplecom" }, "Tags": [ { @@ -1584,13 +1582,16 @@ }, { "Key": "Name", - "Value": "bastion.privateciliumadvanced.example.com" + "Value": "bastion-privateciliumadva-0jni40" }, { "Key": "kubernetes.io/cluster/privateciliumadvanced.example.com", "Value": "owned" } - ] + ], + "HealthCheckProtocol": "TCP", + "HealthyThresholdCount": 2, + "UnhealthyThresholdCount": 2 } }, "AWSIAMInstanceProfilebastionsprivateciliumadvancedexamplecom": { diff --git a/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf b/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf index 793b77ca83a8f..2877fa990de59 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf +++ b/tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf @@ -121,7 +121,6 @@ resource "aws_autoscaling_group" "bastion-privateciliumadvanced-example-com" { id = aws_launch_template.bastion-privateciliumadvanced-example-com.id version = aws_launch_template.bastion-privateciliumadvanced-example-com.latest_version } - load_balancers = [aws_elb.bastion-privateciliumadvanced-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -163,6 +162,7 @@ resource "aws_autoscaling_group" "bastion-privateciliumadvanced-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privateciliumadva-0jni40.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privateciliumadvanced-example-com.id] } @@ -367,31 +367,6 @@ resource "aws_elb" "api-privateciliumadvanced-example-com" { } } -resource "aws_elb" "bastion-privateciliumadvanced-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privateciliumadva-0jni40" - security_groups = [aws_security_group.bastion-elb-privateciliumadvanced-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privateciliumadvanced-example-com.id] - tags = { - "KubernetesCluster" = "privateciliumadvanced.example.com" - "Name" = "bastion.privateciliumadvanced.example.com" - "kubernetes.io/cluster/privateciliumadvanced.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privateciliumadvanced-example-com" { name = "bastions.privateciliumadvanced.example.com" role = aws_iam_role.bastions-privateciliumadvanced-example-com.name @@ -720,6 +695,49 @@ resource "aws_launch_template" "nodes-privateciliumadvanced-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data") } +resource "aws_lb" "bastion-privateciliumadvanced-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privateciliumadva-0jni40" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privateciliumadvanced-example-com.id + } + tags = { + "KubernetesCluster" = "privateciliumadvanced.example.com" + "Name" = "bastion.privateciliumadvanced.example.com" + "kubernetes.io/cluster/privateciliumadvanced.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privateciliumadvanced-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privateciliumadva-0jni40.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privateciliumadvanced-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privateciliumadva-0jni40" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privateciliumadva-0jni40" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privateciliumadvanced.example.com" + "Name" = "bastion-privateciliumadva-0jni40" + "kubernetes.io/cluster/privateciliumadvanced.example.com" = "owned" + } + vpc_id = aws_vpc.privateciliumadvanced-example-com.id +} + resource "aws_nat_gateway" "us-test-1a-privateciliumadvanced-example-com" { allocation_id = aws_eip.us-test-1a-privateciliumadvanced-example-com.id subnet_id = aws_subnet.utility-us-test-1a-privateciliumadvanced-example-com.id @@ -952,17 +970,6 @@ resource "aws_security_group" "api-elb-privateciliumadvanced-example-com" { vpc_id = aws_vpc.privateciliumadvanced-example-com.id } -resource "aws_security_group" "bastion-elb-privateciliumadvanced-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privateciliumadvanced.example.com" - tags = { - "KubernetesCluster" = "privateciliumadvanced.example.com" - "Name" = "bastion-elb.privateciliumadvanced.example.com" - "kubernetes.io/cluster/privateciliumadvanced.example.com" = "owned" - } - vpc_id = aws_vpc.privateciliumadvanced-example-com.id -} - resource "aws_security_group" "bastion-privateciliumadvanced-example-com" { description = "Security group for bastion" name = "bastion.privateciliumadvanced.example.com" @@ -996,11 +1003,11 @@ resource "aws_security_group" "nodes-privateciliumadvanced-example-com" { vpc_id = aws_vpc.privateciliumadvanced-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privateciliumadvanced-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privateciliumadvanced-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privateciliumadvanced-example-com.id + security_group_id = aws_security_group.bastion-privateciliumadvanced-example-com.id to_port = 22 type = "ingress" } @@ -1014,6 +1021,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privateciliumadvanced-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privateciliumadvanced-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-privateciliumadvanced-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1032,33 +1048,6 @@ resource "aws_security_group_rule" "from-api-elb-privateciliumadvanced-example-c type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privateciliumadvanced-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privateciliumadvanced-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privateciliumadvanced-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privateciliumadvanced-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privateciliumadvanced-example-com-ingress-tcp-22to22-bastion-privateciliumadvanced-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privateciliumadvanced-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privateciliumadvanced-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privateciliumadvanced-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1212,6 +1201,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privateciliumadvanced-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privateciliumadvanced-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privateciliumadvanced-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/privatedns1/kubernetes.tf b/tests/integration/update_cluster/privatedns1/kubernetes.tf index 1217ddb88aecb..d1e54c4a3af3c 100644 --- a/tests/integration/update_cluster/privatedns1/kubernetes.tf +++ b/tests/integration/update_cluster/privatedns1/kubernetes.tf @@ -121,7 +121,6 @@ resource "aws_autoscaling_group" "bastion-privatedns1-example-com" { id = aws_launch_template.bastion-privatedns1-example-com.id version = aws_launch_template.bastion-privatedns1-example-com.latest_version } - load_balancers = [aws_elb.bastion-privatedns1-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -173,6 +172,7 @@ resource "aws_autoscaling_group" "bastion-privatedns1-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privatedns1-examp-mbgbef.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privatedns1-example-com.id] } @@ -389,33 +389,6 @@ resource "aws_elb" "api-privatedns1-example-com" { } } -resource "aws_elb" "bastion-privatedns1-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privatedns1-examp-mbgbef" - security_groups = [aws_security_group.bastion-elb-privatedns1-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privatedns1-example-com.id] - tags = { - "KubernetesCluster" = "privatedns1.example.com" - "Name" = "bastion.privatedns1.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/privatedns1.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privatedns1-example-com" { name = "bastions.privatedns1.example.com" role = aws_iam_role.bastions-privatedns1-example-com.name @@ -778,6 +751,53 @@ resource "aws_launch_template" "nodes-privatedns1-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privatedns1.example.com_user_data") } +resource "aws_lb" "bastion-privatedns1-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privatedns1-examp-mbgbef" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privatedns1-example-com.id + } + tags = { + "KubernetesCluster" = "privatedns1.example.com" + "Name" = "bastion.privatedns1.example.com" + "Owner" = "John Doe" + "foo/bar" = "fib+baz" + "kubernetes.io/cluster/privatedns1.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privatedns1-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privatedns1-examp-mbgbef.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privatedns1-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privatedns1-examp-mbgbef" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privatedns1-examp-mbgbef" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privatedns1.example.com" + "Name" = "bastion-privatedns1-examp-mbgbef" + "Owner" = "John Doe" + "foo/bar" = "fib+baz" + "kubernetes.io/cluster/privatedns1.example.com" = "owned" + } + vpc_id = aws_vpc.privatedns1-example-com.id +} + resource "aws_nat_gateway" "us-test-1a-privatedns1-example-com" { allocation_id = aws_eip.us-test-1a-privatedns1-example-com.id subnet_id = aws_subnet.utility-us-test-1a-privatedns1-example-com.id @@ -1007,19 +1027,6 @@ resource "aws_security_group" "api-elb-privatedns1-example-com" { vpc_id = aws_vpc.privatedns1-example-com.id } -resource "aws_security_group" "bastion-elb-privatedns1-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privatedns1.example.com" - tags = { - "KubernetesCluster" = "privatedns1.example.com" - "Name" = "bastion-elb.privatedns1.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/privatedns1.example.com" = "owned" - } - vpc_id = aws_vpc.privatedns1-example-com.id -} - resource "aws_security_group" "bastion-privatedns1-example-com" { description = "Security group for bastion" name = "bastion.privatedns1.example.com" @@ -1059,11 +1066,11 @@ resource "aws_security_group" "nodes-privatedns1-example-com" { vpc_id = aws_vpc.privatedns1-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privatedns1-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privatedns1-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privatedns1-example-com.id + security_group_id = aws_security_group.bastion-privatedns1-example-com.id to_port = 22 type = "ingress" } @@ -1077,6 +1084,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privatedns1-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatedns1-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-privatedns1-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1095,33 +1111,6 @@ resource "aws_security_group_rule" "from-api-elb-privatedns1-example-com-egress- type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privatedns1-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatedns1-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatedns1-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatedns1-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatedns1-example-com-ingress-tcp-22to22-bastion-privatedns1-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privatedns1-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privatedns1-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privatedns1-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1275,6 +1264,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatedns1-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatedns1-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privatedns1-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/privatedns2/kubernetes.tf b/tests/integration/update_cluster/privatedns2/kubernetes.tf index 514bf4846fad2..757c11f92a896 100644 --- a/tests/integration/update_cluster/privatedns2/kubernetes.tf +++ b/tests/integration/update_cluster/privatedns2/kubernetes.tf @@ -116,7 +116,6 @@ resource "aws_autoscaling_group" "bastion-privatedns2-example-com" { id = aws_launch_template.bastion-privatedns2-example-com.id version = aws_launch_template.bastion-privatedns2-example-com.latest_version } - load_balancers = [aws_elb.bastion-privatedns2-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -158,6 +157,7 @@ resource "aws_autoscaling_group" "bastion-privatedns2-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privatedns2-examp-e704o2.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privatedns2-example-com.id] } @@ -346,31 +346,6 @@ resource "aws_elb" "api-privatedns2-example-com" { } } -resource "aws_elb" "bastion-privatedns2-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privatedns2-examp-e704o2" - security_groups = [aws_security_group.bastion-elb-privatedns2-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privatedns2-example-com.id] - tags = { - "KubernetesCluster" = "privatedns2.example.com" - "Name" = "bastion.privatedns2.example.com" - "kubernetes.io/cluster/privatedns2.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privatedns2-example-com" { name = "bastions.privatedns2.example.com" role = aws_iam_role.bastions-privatedns2-example-com.name @@ -690,6 +665,49 @@ resource "aws_launch_template" "nodes-privatedns2-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privatedns2.example.com_user_data") } +resource "aws_lb" "bastion-privatedns2-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privatedns2-examp-e704o2" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privatedns2-example-com.id + } + tags = { + "KubernetesCluster" = "privatedns2.example.com" + "Name" = "bastion.privatedns2.example.com" + "kubernetes.io/cluster/privatedns2.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privatedns2-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privatedns2-examp-e704o2.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privatedns2-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privatedns2-examp-e704o2" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privatedns2-examp-e704o2" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privatedns2.example.com" + "Name" = "bastion-privatedns2-examp-e704o2" + "kubernetes.io/cluster/privatedns2.example.com" = "owned" + } + vpc_id = "vpc-12345678" +} + resource "aws_nat_gateway" "us-test-1a-privatedns2-example-com" { allocation_id = aws_eip.us-test-1a-privatedns2-example-com.id subnet_id = aws_subnet.utility-us-test-1a-privatedns2-example-com.id @@ -898,17 +916,6 @@ resource "aws_security_group" "api-elb-privatedns2-example-com" { vpc_id = "vpc-12345678" } -resource "aws_security_group" "bastion-elb-privatedns2-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privatedns2.example.com" - tags = { - "KubernetesCluster" = "privatedns2.example.com" - "Name" = "bastion-elb.privatedns2.example.com" - "kubernetes.io/cluster/privatedns2.example.com" = "owned" - } - vpc_id = "vpc-12345678" -} - resource "aws_security_group" "bastion-privatedns2-example-com" { description = "Security group for bastion" name = "bastion.privatedns2.example.com" @@ -942,11 +949,11 @@ resource "aws_security_group" "nodes-privatedns2-example-com" { vpc_id = "vpc-12345678" } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privatedns2-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privatedns2-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privatedns2-example-com.id + security_group_id = aws_security_group.bastion-privatedns2-example-com.id to_port = 22 type = "ingress" } @@ -960,6 +967,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privatedns2-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatedns2-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-privatedns2-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -978,33 +994,6 @@ resource "aws_security_group_rule" "from-api-elb-privatedns2-example-com-egress- type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privatedns2-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatedns2-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatedns2-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatedns2-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privatedns2-example-com-ingress-tcp-22to22-bastion-privatedns2-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privatedns2-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privatedns2-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privatedns2-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1158,6 +1147,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatedns2-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatedns2-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privatedns2-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/privateflannel/kubernetes.tf b/tests/integration/update_cluster/privateflannel/kubernetes.tf index 2f91bbfb3d4fd..2276a060c19e9 100644 --- a/tests/integration/update_cluster/privateflannel/kubernetes.tf +++ b/tests/integration/update_cluster/privateflannel/kubernetes.tf @@ -121,7 +121,6 @@ resource "aws_autoscaling_group" "bastion-privateflannel-example-com" { id = aws_launch_template.bastion-privateflannel-example-com.id version = aws_launch_template.bastion-privateflannel-example-com.latest_version } - load_balancers = [aws_elb.bastion-privateflannel-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -158,6 +157,7 @@ resource "aws_autoscaling_group" "bastion-privateflannel-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privateflannel-ex-753531.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privateflannel-example-com.id] } @@ -331,31 +331,6 @@ resource "aws_elb" "api-privateflannel-example-com" { } } -resource "aws_elb" "bastion-privateflannel-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privateflannel-ex-753531" - security_groups = [aws_security_group.bastion-elb-privateflannel-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privateflannel-example-com.id] - tags = { - "KubernetesCluster" = "privateflannel.example.com" - "Name" = "bastion.privateflannel.example.com" - "kubernetes.io/cluster/privateflannel.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privateflannel-example-com" { name = "bastions.privateflannel.example.com" role = aws_iam_role.bastions-privateflannel-example-com.name @@ -672,6 +647,49 @@ resource "aws_launch_template" "nodes-privateflannel-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privateflannel.example.com_user_data") } +resource "aws_lb" "bastion-privateflannel-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privateflannel-ex-753531" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privateflannel-example-com.id + } + tags = { + "KubernetesCluster" = "privateflannel.example.com" + "Name" = "bastion.privateflannel.example.com" + "kubernetes.io/cluster/privateflannel.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privateflannel-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privateflannel-ex-753531.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privateflannel-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privateflannel-ex-753531" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privateflannel-ex-753531" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privateflannel.example.com" + "Name" = "bastion-privateflannel-ex-753531" + "kubernetes.io/cluster/privateflannel.example.com" = "owned" + } + vpc_id = aws_vpc.privateflannel-example-com.id +} + resource "aws_nat_gateway" "us-test-1a-privateflannel-example-com" { allocation_id = aws_eip.us-test-1a-privateflannel-example-com.id subnet_id = aws_subnet.utility-us-test-1a-privateflannel-example-com.id @@ -912,17 +930,6 @@ resource "aws_security_group" "api-elb-privateflannel-example-com" { vpc_id = aws_vpc.privateflannel-example-com.id } -resource "aws_security_group" "bastion-elb-privateflannel-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privateflannel.example.com" - tags = { - "KubernetesCluster" = "privateflannel.example.com" - "Name" = "bastion-elb.privateflannel.example.com" - "kubernetes.io/cluster/privateflannel.example.com" = "owned" - } - vpc_id = aws_vpc.privateflannel-example-com.id -} - resource "aws_security_group" "bastion-privateflannel-example-com" { description = "Security group for bastion" name = "bastion.privateflannel.example.com" @@ -956,11 +963,11 @@ resource "aws_security_group" "nodes-privateflannel-example-com" { vpc_id = aws_vpc.privateflannel-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privateflannel-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privateflannel-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privateflannel-example-com.id + security_group_id = aws_security_group.bastion-privateflannel-example-com.id to_port = 22 type = "ingress" } @@ -974,6 +981,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privateflannel-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privateflannel-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-privateflannel-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -992,33 +1008,6 @@ resource "aws_security_group_rule" "from-api-elb-privateflannel-example-com-egre type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privateflannel-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privateflannel-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privateflannel-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privateflannel-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privateflannel-example-com-ingress-tcp-22to22-bastion-privateflannel-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privateflannel-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privateflannel-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privateflannel-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1172,6 +1161,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privateflannel-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privateflannel-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privateflannel-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/privatekopeio/kubernetes.tf b/tests/integration/update_cluster/privatekopeio/kubernetes.tf index 07281d1f0cfd1..ce9916aafbfa4 100644 --- a/tests/integration/update_cluster/privatekopeio/kubernetes.tf +++ b/tests/integration/update_cluster/privatekopeio/kubernetes.tf @@ -136,7 +136,6 @@ resource "aws_autoscaling_group" "bastion-privatekopeio-example-com" { id = aws_launch_template.bastion-privatekopeio-example-com.id version = aws_launch_template.bastion-privatekopeio-example-com.latest_version } - load_balancers = [aws_elb.bastion-privatekopeio-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -178,6 +177,7 @@ resource "aws_autoscaling_group" "bastion-privatekopeio-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privatekopeio-exa-d8ef8e.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privatekopeio-example-com.id] } @@ -357,31 +357,6 @@ resource "aws_elb" "api-privatekopeio-example-com" { } } -resource "aws_elb" "bastion-privatekopeio-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privatekopeio-exa-d8ef8e" - security_groups = [aws_security_group.bastion-elb-privatekopeio-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privatekopeio-example-com.id, aws_subnet.utility-us-test-1b-privatekopeio-example-com.id] - tags = { - "KubernetesCluster" = "privatekopeio.example.com" - "Name" = "bastion.privatekopeio.example.com" - "kubernetes.io/cluster/privatekopeio.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privatekopeio-example-com" { name = "bastions.privatekopeio.example.com" role = aws_iam_role.bastions-privatekopeio-example-com.name @@ -710,6 +685,52 @@ resource "aws_launch_template" "nodes-privatekopeio-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privatekopeio.example.com_user_data") } +resource "aws_lb" "bastion-privatekopeio-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privatekopeio-exa-d8ef8e" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privatekopeio-example-com.id + } + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1b-privatekopeio-example-com.id + } + tags = { + "KubernetesCluster" = "privatekopeio.example.com" + "Name" = "bastion.privatekopeio.example.com" + "kubernetes.io/cluster/privatekopeio.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privatekopeio-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privatekopeio-exa-d8ef8e.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privatekopeio-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privatekopeio-exa-d8ef8e" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privatekopeio-exa-d8ef8e" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privatekopeio.example.com" + "Name" = "bastion-privatekopeio-exa-d8ef8e" + "kubernetes.io/cluster/privatekopeio.example.com" = "owned" + } + vpc_id = aws_vpc.privatekopeio-example-com.id +} + resource "aws_route" "route-0-0-0-0--0" { destination_cidr_block = "0.0.0.0/0" gateway_id = aws_internet_gateway.privatekopeio-example-com.id @@ -942,17 +963,6 @@ resource "aws_security_group" "api-elb-privatekopeio-example-com" { vpc_id = aws_vpc.privatekopeio-example-com.id } -resource "aws_security_group" "bastion-elb-privatekopeio-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privatekopeio.example.com" - tags = { - "KubernetesCluster" = "privatekopeio.example.com" - "Name" = "bastion-elb.privatekopeio.example.com" - "kubernetes.io/cluster/privatekopeio.example.com" = "owned" - } - vpc_id = aws_vpc.privatekopeio-example-com.id -} - resource "aws_security_group" "bastion-privatekopeio-example-com" { description = "Security group for bastion" name = "bastion.privatekopeio.example.com" @@ -986,11 +996,11 @@ resource "aws_security_group" "nodes-privatekopeio-example-com" { vpc_id = aws_vpc.privatekopeio-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privatekopeio-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privatekopeio-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privatekopeio-example-com.id + security_group_id = aws_security_group.bastion-privatekopeio-example-com.id to_port = 22 type = "ingress" } @@ -1004,51 +1014,42 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } -resource "aws_security_group_rule" "from-api-elb-privatekopeio-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.api-elb-privatekopeio-example-com.id - to_port = 0 - type = "egress" +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privatekopeio-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatekopeio-example-com.id + to_port = 22 + type = "ingress" } -resource "aws_security_group_rule" "from-api-elb-privatekopeio-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.api-elb-privatekopeio-example-com.id - to_port = 0 - type = "egress" +resource "aws_security_group_rule" "from-172-20-8-0--22-ingress-tcp-22to22-bastion-privatekopeio-example-com" { + cidr_blocks = ["172.20.8.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatekopeio-example-com.id + to_port = 22 + type = "ingress" } -resource "aws_security_group_rule" "from-bastion-elb-privatekopeio-example-com-egress-all-0to0-0-0-0-0--0" { +resource "aws_security_group_rule" "from-api-elb-privatekopeio-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatekopeio-example-com.id + security_group_id = aws_security_group.api-elb-privatekopeio-example-com.id to_port = 0 type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privatekopeio-example-com-egress-all-0to0-__--0" { +resource "aws_security_group_rule" "from-api-elb-privatekopeio-example-com-egress-all-0to0-__--0" { from_port = 0 ipv6_cidr_blocks = ["::/0"] protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privatekopeio-example-com.id + security_group_id = aws_security_group.api-elb-privatekopeio-example-com.id to_port = 0 type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privatekopeio-example-com-ingress-tcp-22to22-bastion-privatekopeio-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privatekopeio-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privatekopeio-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privatekopeio-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1202,6 +1203,33 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatekopeio-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatekopeio-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-8-0--22" { + cidr_blocks = ["172.20.8.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatekopeio-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privatekopeio-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/privateweave/kubernetes.tf b/tests/integration/update_cluster/privateweave/kubernetes.tf index ad3cf61c410fb..c321dd87573d9 100644 --- a/tests/integration/update_cluster/privateweave/kubernetes.tf +++ b/tests/integration/update_cluster/privateweave/kubernetes.tf @@ -121,7 +121,6 @@ resource "aws_autoscaling_group" "bastion-privateweave-example-com" { id = aws_launch_template.bastion-privateweave-example-com.id version = aws_launch_template.bastion-privateweave-example-com.latest_version } - load_balancers = [aws_elb.bastion-privateweave-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -163,6 +162,7 @@ resource "aws_autoscaling_group" "bastion-privateweave-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-privateweave-exam-fdb6ge.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privateweave-example-com.id] } @@ -351,31 +351,6 @@ resource "aws_elb" "api-privateweave-example-com" { } } -resource "aws_elb" "bastion-privateweave-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-privateweave-exam-fdb6ge" - security_groups = [aws_security_group.bastion-elb-privateweave-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-privateweave-example-com.id] - tags = { - "KubernetesCluster" = "privateweave.example.com" - "Name" = "bastion.privateweave.example.com" - "kubernetes.io/cluster/privateweave.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-privateweave-example-com" { name = "bastions.privateweave.example.com" role = aws_iam_role.bastions-privateweave-example-com.name @@ -704,6 +679,49 @@ resource "aws_launch_template" "nodes-privateweave-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privateweave.example.com_user_data") } +resource "aws_lb" "bastion-privateweave-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privateweave-exam-fdb6ge" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privateweave-example-com.id + } + tags = { + "KubernetesCluster" = "privateweave.example.com" + "Name" = "bastion.privateweave.example.com" + "kubernetes.io/cluster/privateweave.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privateweave-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privateweave-exam-fdb6ge.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privateweave-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privateweave-exam-fdb6ge" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privateweave-exam-fdb6ge" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privateweave.example.com" + "Name" = "bastion-privateweave-exam-fdb6ge" + "kubernetes.io/cluster/privateweave.example.com" = "owned" + } + vpc_id = aws_vpc.privateweave-example-com.id +} + resource "aws_nat_gateway" "us-test-1a-privateweave-example-com" { allocation_id = aws_eip.us-test-1a-privateweave-example-com.id subnet_id = aws_subnet.utility-us-test-1a-privateweave-example-com.id @@ -920,17 +938,6 @@ resource "aws_security_group" "api-elb-privateweave-example-com" { vpc_id = aws_vpc.privateweave-example-com.id } -resource "aws_security_group" "bastion-elb-privateweave-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.privateweave.example.com" - tags = { - "KubernetesCluster" = "privateweave.example.com" - "Name" = "bastion-elb.privateweave.example.com" - "kubernetes.io/cluster/privateweave.example.com" = "owned" - } - vpc_id = aws_vpc.privateweave-example-com.id -} - resource "aws_security_group" "bastion-privateweave-example-com" { description = "Security group for bastion" name = "bastion.privateweave.example.com" @@ -964,11 +971,11 @@ resource "aws_security_group" "nodes-privateweave-example-com" { vpc_id = aws_vpc.privateweave-example-com.id } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privateweave-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-privateweave-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-privateweave-example-com.id + security_group_id = aws_security_group.bastion-privateweave-example-com.id to_port = 22 type = "ingress" } @@ -982,6 +989,15 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-privateweave-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privateweave-example-com.id + to_port = 22 + type = "ingress" +} + resource "aws_security_group_rule" "from-api-elb-privateweave-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1000,33 +1016,6 @@ resource "aws_security_group_rule" "from-api-elb-privateweave-example-com-egress type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-privateweave-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privateweave-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privateweave-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.bastion-elb-privateweave-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-bastion-elb-privateweave-example-com-ingress-tcp-22to22-bastion-privateweave-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-privateweave-example-com.id - source_security_group_id = aws_security_group.bastion-elb-privateweave-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-privateweave-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1180,6 +1169,24 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privateweave-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privateweave-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-privateweave-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/tests/integration/update_cluster/unmanaged/kubernetes.tf b/tests/integration/update_cluster/unmanaged/kubernetes.tf index a02f5f0b9c665..5d9c5c77c652a 100644 --- a/tests/integration/update_cluster/unmanaged/kubernetes.tf +++ b/tests/integration/update_cluster/unmanaged/kubernetes.tf @@ -116,7 +116,6 @@ resource "aws_autoscaling_group" "bastion-unmanaged-example-com" { id = aws_launch_template.bastion-unmanaged-example-com.id version = aws_launch_template.bastion-unmanaged-example-com.latest_version } - load_balancers = [aws_elb.bastion-unmanaged-example-com.id] max_instance_lifetime = 0 max_size = 1 metrics_granularity = "1Minute" @@ -158,6 +157,7 @@ resource "aws_autoscaling_group" "bastion-unmanaged-example-com" { propagate_at_launch = true value = "owned" } + target_group_arns = [aws_lb_target_group.bastion-unmanaged-example-d7bn3d.id] vpc_zone_identifier = [aws_subnet.utility-us-test-1a-unmanaged-example-com.id] } @@ -337,31 +337,6 @@ resource "aws_elb" "api-unmanaged-example-com" { } } -resource "aws_elb" "bastion-unmanaged-example-com" { - health_check { - healthy_threshold = 2 - interval = 10 - target = "TCP:22" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 22 - instance_protocol = "TCP" - lb_port = 22 - lb_protocol = "TCP" - } - name = "bastion-unmanaged-example-d7bn3d" - security_groups = [aws_security_group.bastion-elb-unmanaged-example-com.id] - subnets = [aws_subnet.utility-us-test-1a-unmanaged-example-com.id, aws_subnet.utility-us-test-1b-unmanaged-example-com.id] - tags = { - "KubernetesCluster" = "unmanaged.example.com" - "Name" = "bastion.unmanaged.example.com" - "kubernetes.io/cluster/unmanaged.example.com" = "owned" - } -} - resource "aws_iam_instance_profile" "bastions-unmanaged-example-com" { name = "bastions.unmanaged.example.com" role = aws_iam_role.bastions-unmanaged-example-com.name @@ -681,6 +656,52 @@ resource "aws_launch_template" "nodes-unmanaged-example-com" { user_data = filebase64("${path.module}/data/aws_launch_template_nodes.unmanaged.example.com_user_data") } +resource "aws_lb" "bastion-unmanaged-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-unmanaged-example-d7bn3d" + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-unmanaged-example-com.id + } + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1b-unmanaged-example-com.id + } + tags = { + "KubernetesCluster" = "unmanaged.example.com" + "Name" = "bastion.unmanaged.example.com" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-unmanaged-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-unmanaged-example-d7bn3d.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-unmanaged-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-unmanaged-example-d7bn3d" { + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-unmanaged-example-d7bn3d" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "unmanaged.example.com" + "Name" = "bastion-unmanaged-example-d7bn3d" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + } + vpc_id = "vpc-12345678" +} + resource "aws_route53_record" "api-unmanaged-example-com" { alias { evaluate_target_health = false @@ -831,17 +852,6 @@ resource "aws_security_group" "api-elb-unmanaged-example-com" { vpc_id = "vpc-12345678" } -resource "aws_security_group" "bastion-elb-unmanaged-example-com" { - description = "Security group for bastion ELB" - name = "bastion-elb.unmanaged.example.com" - tags = { - "KubernetesCluster" = "unmanaged.example.com" - "Name" = "bastion-elb.unmanaged.example.com" - "kubernetes.io/cluster/unmanaged.example.com" = "owned" - } - vpc_id = "vpc-12345678" -} - resource "aws_security_group" "bastion-unmanaged-example-com" { description = "Security group for bastion" name = "bastion.unmanaged.example.com" @@ -875,11 +885,11 @@ resource "aws_security_group" "nodes-unmanaged-example-com" { vpc_id = "vpc-12345678" } -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-unmanaged-example-com" { +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-unmanaged-example-com" { cidr_blocks = ["0.0.0.0/0"] from_port = 22 protocol = "tcp" - security_group_id = aws_security_group.bastion-elb-unmanaged-example-com.id + security_group_id = aws_security_group.bastion-unmanaged-example-com.id to_port = 22 type = "ingress" } @@ -893,51 +903,42 @@ resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb type = "ingress" } -resource "aws_security_group_rule" "from-api-elb-unmanaged-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.api-elb-unmanaged-example-com.id - to_port = 0 - type = "egress" +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-unmanaged-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-unmanaged-example-com.id + to_port = 22 + type = "ingress" } -resource "aws_security_group_rule" "from-api-elb-unmanaged-example-com-egress-all-0to0-__--0" { - from_port = 0 - ipv6_cidr_blocks = ["::/0"] - protocol = "-1" - security_group_id = aws_security_group.api-elb-unmanaged-example-com.id - to_port = 0 - type = "egress" +resource "aws_security_group_rule" "from-172-20-8-0--22-ingress-tcp-22to22-bastion-unmanaged-example-com" { + cidr_blocks = ["172.20.8.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-unmanaged-example-com.id + to_port = 22 + type = "ingress" } -resource "aws_security_group_rule" "from-bastion-elb-unmanaged-example-com-egress-all-0to0-0-0-0-0--0" { +resource "aws_security_group_rule" "from-api-elb-unmanaged-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 protocol = "-1" - security_group_id = aws_security_group.bastion-elb-unmanaged-example-com.id + security_group_id = aws_security_group.api-elb-unmanaged-example-com.id to_port = 0 type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-unmanaged-example-com-egress-all-0to0-__--0" { +resource "aws_security_group_rule" "from-api-elb-unmanaged-example-com-egress-all-0to0-__--0" { from_port = 0 ipv6_cidr_blocks = ["::/0"] protocol = "-1" - security_group_id = aws_security_group.bastion-elb-unmanaged-example-com.id + security_group_id = aws_security_group.api-elb-unmanaged-example-com.id to_port = 0 type = "egress" } -resource "aws_security_group_rule" "from-bastion-elb-unmanaged-example-com-ingress-tcp-22to22-bastion-unmanaged-example-com" { - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.bastion-unmanaged-example-com.id - source_security_group_id = aws_security_group.bastion-elb-unmanaged-example-com.id - to_port = 22 - type = "ingress" -} - resource "aws_security_group_rule" "from-bastion-unmanaged-example-com-egress-all-0to0-0-0-0-0--0" { cidr_blocks = ["0.0.0.0/0"] from_port = 0 @@ -1091,6 +1092,33 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { type = "ingress" } +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-unmanaged-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-unmanaged-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-8-0--22" { + cidr_blocks = ["172.20.8.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-unmanaged-example-com.id + to_port = 4 + type = "ingress" +} + resource "aws_subnet" "us-test-1a-unmanaged-example-com" { availability_zone = "us-test-1a" cidr_block = "172.20.32.0/19" diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec_test.go b/upup/pkg/fi/cloudup/populate_cluster_spec_test.go index 21e1c62f2c088..a3a7e514fd553 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec_test.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec_test.go @@ -345,16 +345,6 @@ func TestPopulateCluster_BastionInvalidMatchingValues_Required(t *testing.T) { expectErrorFromPopulateCluster(t, c, cloud, "bastion") } -func TestPopulateCluster_BastionIdleTimeoutInvalidNegative_Required(t *testing.T) { - cloud, c := buildMinimalCluster() - - c.Spec.Topology.Masters = kopsapi.TopologyPrivate - c.Spec.Topology.Nodes = kopsapi.TopologyPrivate - c.Spec.Topology.Bastion = &kopsapi.BastionSpec{} - c.Spec.Topology.Bastion.IdleTimeoutSeconds = fi.Int64(-1) - expectErrorFromPopulateCluster(t, c, cloud, "bastion") -} - func expectErrorFromPopulateCluster(t *testing.T, c *kopsapi.Cluster, cloud fi.Cloud, message string) { _, err := mockedPopulateClusterSpec(c, cloud) if err == nil {